Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

Reapply PR #205 (HCC readiness decoupling)

Alfredo Lopez5d2ea4c78619
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 18daf933c..08541e5bc 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -390,6 +390,21 @@ jobs:
- name: Test HCC watch-recovery gate coordination
run: bash scripts/tests/test-hcc-watch-recovery-gate.sh
+ - name: Test HCC API gateway readiness separation
+ run: bash scripts/tests/test-hcc-api-gateway-readiness.sh
+
+ - name: Test host-storm metric adjudication
+ run: bash scripts/tests/test-host-storm-metric-gate.sh
+
+ - name: Test WRC-to-HCC contracts
+ run: bash scripts/tests/test-wrc-hcc-contracts-gate.sh
+
+ - name: Test WRC internal dependency NetworkPolicy contracts
+ run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
+
+ - name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)
+ run: bash scripts/tests/test-lint-networkpolicies.sh
+
- name: Test Plugin Workload SDK auth bootstrap policy
run: bash scripts/tests/test-plugin-workload-sdk-auth-bootstrap.sh
@@ -399,6 +414,24 @@ jobs:
- name: Test minikube detect-k8s-api-ip template rendering
run: bash scripts/tests/test-minikube-detect-k8s-api-ip.sh
+ - name: Validate single-writer rollout invariant on rendered overlays
+ run: |
+ # The k8s-api-ip patch is rendered from a gitignored template at setup
+ # time; render it with a dummy IP so `kubectl kustomize` can build the
+ # minikube overlay here. The gcp-* overlays live in evenfire-infra, so
+ # this repo validates only the minikube overlay it actually ships.
+ sed 's#__K8S_API_IP__#10.96.0.1#g' \
+ deploy/overlays/minikube/patches/k8s-api-ip.yaml.template \
+ > deploy/overlays/minikube/patches/k8s-api-ip.yaml
+ SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" \
+ bash deploy/scripts/validate-postgres-single-writer-strategy.sh
+
+ - name: Test single-writer validator fails loud on an absent local deploy
+ run: bash scripts/tests/test-postgres-single-writer-strategy.sh
+
+ - name: Test post-reset convergence isolates DB-dependent controllers
+ run: bash scripts/tests/test-converge-control-db-after-reset.sh
+
- name: Test release coordinates (writer, checker, floor decoupling)
run: bash scripts/tests/test-release-coordinates.sh
diff --git a/Makefile b/Makefile
index 57f47ec5a..107633d70 100644
--- a/Makefile
+++ b/Makefile
@@ -934,7 +934,20 @@ test-e2e-stateless-wake-recovery: ## Run stateless wake-recovery latency gate (R
.PHONY: test-e2e-hcc-communicationchannel-watch-recovery
test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC watch-recovery fault-injection gate
@echo "Running HCC CommunicationChannel watch-recovery gate..."
- E2E_HCC_WATCH_FAULT_INJECTION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
+ E2E_HCC_WATCH_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
+
+.PHONY: test-e2e-hcc-readiness-bootstrap
+test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active
+ @echo "Running HCC initial-fleet readiness gate..."
+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
+ E2E_HCC_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-readiness-bootstrap.sh
+
+.PHONY: test-e2e-hcc-mcp-context-readiness
+test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Context/NetworkPolicy initial convergence
+ @echo "Running HCC MCP/Context/NetworkPolicy readiness gate..."
+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
+ E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-mcp-context-readiness.sh
.PHONY: test-e2e-stateless-multinode
test-e2e-stateless-multinode: ## Run stateless multi-node lane (opt-in: STATELESS_MULTINODE_GATE=1; needs >=2 schedulable nodes; exit 3 = cross-node UNVERIFIED)
diff --git a/deploy/base/control-plane/host-context-controller-api-gateway.yaml b/deploy/base/control-plane/host-context-controller-api-gateway.yaml
index 961cebe48..1c30ccfbc 100644
--- a/deploy/base/control-plane/host-context-controller-api-gateway.yaml
+++ b/deploy/base/control-plane/host-context-controller-api-gateway.yaml
@@ -43,7 +43,7 @@ spec:
timeoutSeconds: 5
readinessProbe:
httpGet:
- path: /ready
+ path: /health
port: http
scheme: HTTP
initialDelaySeconds: 3
diff --git a/deploy/base/control-plane/host-context-controller.yaml b/deploy/base/control-plane/host-context-controller.yaml
index df8126a53..24a4c21dd 100644
--- a/deploy/base/control-plane/host-context-controller.yaml
+++ b/deploy/base/control-plane/host-context-controller.yaml
@@ -13,21 +13,33 @@ metadata:
app: host-context-controller
spec:
replicas: 1
- # HCC's /ready stays 503 until its initial fleet reconciliation finishes, so time-to-Ready
- # scales with the McpServer/Context fleet rather than with process start. clerum-dev took
- # 651s on 2026-08-05 at 108 McpServers / 22 Contexts.
+ # HCC's safety model assumes a single NetworkPolicy writer: replicas:1 with no
+ # leader election (see statelessLifecycleExecutor.ts). RollingUpdate (the
+ # default) starts the new pod before terminating the old one, so every rollout
+ # would overlap two writers — during which the outgoing replica can win a
+ # conflict-retry replace and land its wider spec back over a policy the new
+ # replica just hardened and certified, with no NetworkPolicy resync to heal it
+ # and /ready still 200. Recreate terminates the old pod first, honouring the
+ # single-writer assumption through the rollout too.
+ strategy:
+ type: Recreate
+ # Recreate + replicas:1 means a botched rollout (new pod never Ready) leaves
+ # the control plane down until a manual `kubectl rollout undo` — the single-
+ # writer invariant forbids any overlap-based auto-recovery. This deadline is
+ # DETECTION ONLY: it flips Progressing=False (ProgressDeadlineExceeded) and
+ # fails `kubectl rollout status` for callers without their own --timeout; it
+ # does not roll back.
#
- # Unset, this defaults to 600s, and that default is what actually governed the deploy:
- # `kubectl rollout status` aborts the instant the controller sets ProgressDeadlineExceeded,
- # so provision-gfs-runtime.sh's --timeout could never exceed the deadline remaining. The
- # clock also starts several steps earlier, at apply-inter-service-tokens.sh's `rollout
- # restart` of HCC, so the wait inherits only what is left. On 2026-08-05 that was 600-160
- # = 440s against a 480s timeout: the timeout raised in #202 never bound, and the deploy
- # failed 50s before HCC came Ready.
- #
- # 1200s covers the 900s wait plus a 300s head-start allowance. This is a mitigation, not
- # the fix — evenfire-ai/evenfire#205 decouples readiness from fleet convergence, after
- # which both numbers should come back down.
+ # Value (origin/dev, 2026-08-05): HCC's /ready stays 503 until its initial fleet
+ # reconciliation finishes, so time-to-Ready scales with the McpServer/Context
+ # fleet — clerum-dev took 651s at 108 McpServers / 22 Contexts. The unset default
+ # (600s) governed the deploy because `kubectl rollout status` aborts the instant
+ # the deadline is exceeded, and the clock starts at apply-inter-service-tokens.sh's
+ # `rollout restart`, so the wait inherits only the remaining budget (600-160=440s
+ # vs a 480s timeout on 2026-08-05: #202's timeout never bound, deploy failed 50s
+ # short). 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the
+ # fix — #205 decouples readiness from fleet convergence, after which this should
+ # come back down.
progressDeadlineSeconds: 1200
selector:
matchLabels:
@@ -91,6 +103,8 @@ spec:
# now fails closed at startup on any interval > overlap/2.
- name: HCC_EXTERNAL_EGRESS_RESYNC_SEC
value: "60"
+ - name: HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS
+ value: "5000"
- name: HCC_HOST_K8S_REQUEST_TIMEOUT_MS
value: "30000"
# Bounded cross-Host concurrency for a full/lifecycle Host fleet pass.
diff --git a/deploy/scripts/converge-control-db-after-reset.sh b/deploy/scripts/converge-control-db-after-reset.sh
index 99f50d8bf..404d61076 100755
--- a/deploy/scripts/converge-control-db-after-reset.sh
+++ b/deploy/scripts/converge-control-db-after-reset.sh
@@ -183,7 +183,7 @@ if [ "$STATE_PHASE" = converged ]; then
fi
kc -n control-plane scale deployment/host-context-controller --replicas="$HCC_REPLICAS" >/dev/null
if [ "$HCC_REPLICAS" -gt 0 ]; then
- kc -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/null
+ kc -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null
fi
assert_replacement_uid
delete_reset_state
@@ -264,7 +264,7 @@ kc -n control-plane scale deployment/host-context-controller \
--replicas="$HCC_REPLICAS" >/dev/null
if [ "$HCC_REPLICAS" -gt 0 ]; then
kc -n control-plane rollout status \
- deployment/host-context-controller --timeout=180s >/dev/null
+ deployment/host-context-controller --timeout=900s >/dev/null
fi
assert_replacement_uid
read_reset_state || fail "cannot reload reset state before completion"
diff --git a/deploy/scripts/validate-postgres-single-writer-strategy.sh b/deploy/scripts/validate-postgres-single-writer-strategy.sh
index f1a77091c..ee90b96be 100755
--- a/deploy/scripts/validate-postgres-single-writer-strategy.sh
+++ b/deploy/scripts/validate-postgres-single-writer-strategy.sh
@@ -2,40 +2,86 @@
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-OVERLAYS=(
- "deploy/overlays/minikube"
- "deploy/overlays/gcp-dev"
- "deploy/overlays/gcp-prod"
-)
+# Overlays to render. Defaults to the full production set; the gcp-* overlays
+# live in keyper-labs/evenfire-infra, so this script's full coverage runs there.
+# Override (space-separated) to validate only the overlays present in a given
+# checkout — e.g. SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" in this repo.
+read -r -a OVERLAYS <<<"${SINGLE_WRITER_OVERLAYS:-deploy/overlays/minikube deploy/overlays/gcp-dev deploy/overlays/gcp-prod}"
DATABASE_DEPLOYS=(
"control-plane/control-postgres"
"registry/registry-postgres"
)
+# Deployments whose safety model assumes a single writer with NO leader election
+# (see host-context-controller/src/statelessLifecycleExecutor.ts). For these the
+# RENDERED manifest — not just deploy/base — must keep replicas<=1 and
+# strategy=Recreate with no rollingUpdate residue, or a rollout reopens the
+# two-writer NetworkPolicy overlap closed in PR #205. Unlike DATABASE_DEPLOYS
+# these mount no PVC, so they need their own list rather than the PVC heuristic.
+SINGLE_WRITER_DEPLOYS=(
+ "control-plane/host-context-controller"
+)
fail() {
echo "ERROR: $*" >&2
exit 1
}
+# A Deployment id ("namespace/name") is DEFINED in this repo when its namespace
+# has a base manifest tree here (deploy/base/<namespace>/). For such a LOCAL
+# deploy the rendered overlay MUST contain it: a silent disappearance would let
+# the loops below validate the single-writer/database rollout invariant over an
+# absent deploy (a fail-open). Deploys whose base lives only in the sibling
+# evenfire-infra repo (e.g. registry/*) are FOREIGN here and are validated where
+# they are defined, so for them an absence is a NOTICE-and-skip, not a failure.
+deploy_defined_in_this_repo() {
+ local id="$1"
+ local namespace="${id%%/*}"
+ [ -d "${ROOT_DIR}/deploy/base/${namespace}" ]
+}
+
command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"
command -v jq >/dev/null 2>&1 || fail "jq is required"
+command -v yq >/dev/null 2>&1 || fail "yq is required"
+
+# Zero overlays means the loop below runs zero iterations and the script exits 0
+# having validated nothing — a fail-open. An empty or whitespace-only
+# SINGLE_WRITER_OVERLAYS must fail loudly, not pass vacuously.
+[ "${#OVERLAYS[@]}" -gt 0 ] ||
+ fail "SINGLE_WRITER_OVERLAYS resolved to zero overlays — refusing to pass vacuously"
for overlay in "${OVERLAYS[@]}"; do
+ # Render OFFLINE: `kubectl kustomize` builds the overlay client-side, and yq
+ # converts the multi-document YAML into a single JSON array for jq. The
+ # previous `kubectl create --dry-run=client` performed API discovery
+ # (kind -> resource), which needs a live API server and fails in CI with
+ # "unable to recognize STDIN: connection refused". This validation is a pure
+ # rendered-manifest lint and must never touch a cluster.
rendered_json="$(
cd "${ROOT_DIR}" &&
kubectl kustomize "${overlay}" |
- kubectl create --dry-run=client --validate=false -f - -o json |
- jq -s '.'
+ yq ea -o=json '[.]'
)"
for id in "${DATABASE_DEPLOYS[@]}"; do
namespace="${id%%/*}"
name="${id#*/}"
- deployment="$(
+ if ! deployment="$(
jq -e --arg namespace "${namespace}" --arg name "${name}" \
'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \
<<<"${rendered_json}"
- )" || fail "${overlay}: missing Deployment ${id}"
+ )"; then
+ # This overlay does not render ${id}. A deploy this repo DEFINES
+ # (deploy/base/<namespace>/ exists here) MUST be rendered by a local
+ # overlay: a silent disappearance would validate the rollout invariant over
+ # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and
+ # the registry Deployment live in the sibling evenfire-infra repo) are
+ # validated where they are defined, so for them a NOTICE-and-skip is right.
+ if deploy_defined_in_this_repo "${id}"; then
+ fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"
+ fi
+ echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2
+ continue
+ fi
pvc_count="$(
jq '[.spec.template.spec.volumes[]? | select(.persistentVolumeClaim.claimName != null)] | length' \
@@ -54,6 +100,42 @@ for overlay in "${OVERLAYS[@]}"; do
fail "${overlay}: Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"
fi
done
+
+ for id in "${SINGLE_WRITER_DEPLOYS[@]}"; do
+ namespace="${id%%/*}"
+ name="${id#*/}"
+ if ! deployment="$(
+ jq -e --arg namespace "${namespace}" --arg name "${name}" \
+ 'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \
+ <<<"${rendered_json}"
+ )"; then
+ # This overlay does not render ${id}. A deploy this repo DEFINES
+ # (deploy/base/<namespace>/ exists here) MUST be rendered by a local
+ # overlay: a silent disappearance would validate the rollout invariant over
+ # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and
+ # the registry Deployment live in the sibling evenfire-infra repo) are
+ # validated where they are defined, so for them a NOTICE-and-skip is right.
+ if deploy_defined_in_this_repo "${id}"; then
+ fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"
+ fi
+ echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2
+ continue
+ fi
+
+ replicas="$(jq -r '.spec.replicas // 1' <<<"${deployment}")"
+ if [ "${replicas}" -gt 1 ]; then
+ fail "${overlay}: single-writer Deployment ${id} must keep replicas<=1 (no leader election), got ${replicas}"
+ fi
+
+ strategy="$(jq -r '.spec.strategy.type // "RollingUpdate"' <<<"${deployment}")"
+ if [ "${strategy}" != "Recreate" ]; then
+ fail "${overlay}: single-writer Deployment ${id} must use strategy.type=Recreate, got ${strategy}"
+ fi
+
+ if jq -e '.spec.strategy.rollingUpdate != null' >/dev/null <<<"${deployment}"; then
+ fail "${overlay}: single-writer Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"
+ fi
+ done
done
-echo "Postgres single-writer rollout strategy validation passed"
+echo "Single-writer rollout strategy validation passed"
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx
index 0b1007986..ff8667aa0 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx
@@ -52,9 +52,7 @@ function createDeferred<T>(): Deferred<T> {
}
function createSessionState(authenticated: boolean, me = HARNESS_ME) {
- return authenticated
- ? { authenticated: true, me }
- : { authenticated: false, me: null }
+ return authenticated ? { authenticated: true, me } : { authenticated: false, me: null }
}
function createCatalog(agentNames: string[]) {
diff --git a/docs/architecture/hcc-readiness-flows.md b/docs/architecture/hcc-readiness-flows.md
new file mode 100644
index 000000000..6a4e89ca9
--- /dev/null
+++ b/docs/architecture/hcc-readiness-flows.md
@@ -0,0 +1,90 @@
+# HCC readiness & external-egress — design note (PR #205)
+
+> Design note for the readiness-decoupling and single-writer NetworkPolicy work in
+> `host-context-controller` (HCC). Addresses the "no design/decision doc for the
+> 500+-line external-egress coordinator and the certification barrier" review gap.
+
+![HCC flows resolved by PR #205](./pr205-hcc-flows.png)
+
+## 1. Readiness lifecycle — decoupled from initial fleet convergence
+
+Before this work, `McpServerWatcher.start()` awaited full initial fleet
+reconciliation, so a large MCP/Context/Host fleet could keep `provider.start()`
+and `/ready` unavailable until every individual reconcile finished.
+
+Now readiness is gated on an **authoritative, fail-closed safety boundary**, not on
+full convergence:
+
+1. **HCC start** — four watch lanes (Context ingress, mcp-host egress, rpc-proxy
+ egress, external egress) start. A transient LIST/WATCH failure on any lane enters
+ in-process recovery and **does not kill the process**; HCC simply stays unready.
+2. **4-LIST inventory** — bounded inventory across the four lanes.
+3. **Certify safety inventory** — the authoritative pass revokes any stale /
+ unprovable allow **before** recording a certificate (fail-closed).
+4. **`/ready = 200`** — served **only** when the inventory is certified and no stale
+ allow survives.
+5. **Additive convergence** — the remaining fleet converges **after** ready,
+ asynchronously, without blocking it.
+
+## 2. Fail-closed fences
+
+| Fence | Rule | Source |
+|-------|------|--------|
+| Readiness gate | `/ready = ready && providerAuthoritativeFn()`; constructor default `() => false`; a throwing gate → 503 | `server.ts`, `readinessGate.ts` |
+| G1 — unconfigured authority | `currentWhenUnconfigured()` returns `false` + warns once; prod always wires the source, dev never constructs the reconciler | `reconciler.ts` |
+| F1 — readiness-poll rebind | rebinds `isCurrent`/`server` on a superseding reconcile so the poll window does not self-destruct on a stale fence | `reconciler.ts` |
+| Authority fence — `isCurrent` | **required** (not an optional/defaulted arg) on all three allow-granting reconcilers — `bindingPolicyReconciler`, `reconcileContext` (L2 context-allow), `reconcileExternalEgress` (L3 external-egress ALLOW). No `?? (() => true)` fail-open default: a caller cannot compile without supplying the fence (TS2554/TS2345) | `networkPolicyReconciler.ts`, `bindingPolicyReconciler.ts` |
+
+The affirmative literal (`() => true`) is reachable only from the dev/test wiring;
+every production path resolves the real authority function, and — because the fence
+is a **required argument** on every allow-granting reconciler rather than a defaulted
+option — no production caller can silently fall back to it.
+
+## 3. DNS external-egress contract (B3) — retain / revoke decision table
+
+A DNS-derived external-egress allow is compared against the desired policy
+reconstructed from the **live** resolved CIDRs ("modulo cidr"). The decision:
+
+| Observed change | Classification | Action | Rationale |
+|-----------------|----------------|--------|-----------|
+| identity unchanged (same protocol, port, selector, ownership; cidrs only) | identity-stable | **RETAIN** while Ready; refresh cidrs in the additive lane | revoking a healthy binding every pass created a guaranteed deny window |
+| protocol drift (TCP→UDP) | identity-changed | **REVOKE before readiness** | old allow is unprovable/stale |
+| spec-level port drift | identity-changed | **REVOKE before readiness** | " |
+| ownership drift | identity-changed | **REVOKE before readiness** | " |
+| zero-cidr (deny-in-disguise) | never retained | **REVOKE** | an empty cidr set is never a valid retained allow |
+
+Revocation of an unprovable policy runs immediately (`cleanupExternalEgress`) under
+the `isCurrent()` fence, **before** the safety inventory is certified — so `/ready`
+never reports 200 with a stale divergent policy present. The unit suite pins both
+directions (retain identity-stable; revoke on each drift), and the branch-owned
+Minikube gate `e2e-hcc-mcp-context-readiness.sh` exercises both phases live.
+
+## 4. ExternalEgressConvergenceCoordinator
+
+The coordinator is the **single owner** of external-egress reconcile cadence
+(replacing an inline timer in `k8sClient`):
+
+- **Per-server FIFO + concurrency limiter** — a slow or never-resolving binding
+ cannot head-of-line block the control-plane event loop.
+- **Sliding-window CIDR accumulation (#299)** — resolved IPs live for `TTL + overlap`;
+ large provider pools are covered by provider CIDRs rather than per-IP `/32`s. A
+ failed refresh deletes the allow (fail-closed), never widens it.
+- **TTL-aware periodic resync (H2)** — `startPeriodicResync` re-arms after each pass
+ with a delay of `min(interval, TTL/2)` bounded below by a floor, so low-TTL hosts
+ refresh promptly. The timer is `.unref()`'d and re-programs **only after a pass
+ completes** (no overlap).
+- **Fence** — startup rejects `interval > overlap/2`, so the accumulated window can
+ never lapse regardless of the configured cadence.
+- **Observability** — `clerum_hcc_external_egress_retries_at_cap` gauges how many
+ servers are stuck retrying at the delay cap (a never-resolving DNS binding is
+ otherwise invisible: denied with `/ready` = 200 and no signal).
+
+## 5. Deploy invariant — single writer
+
+The HCC Deployment ships `strategy: Recreate`, `replicas: 1`, and
+`progressDeadlineSeconds: 1200`: the stateless single-writer safety model has no
+leader election, so two concurrently-running controllers must be impossible during a
+rollout. The trade-off (full HCC control-plane downtime per rollout; a rollout whose
+new pod never becomes Ready needs a manual `kubectl rollout undo`) is declared in the
+manifest and asserted by the bootstrap gate. `validate-postgres-single-writer-strategy.sh`
+enforces the invariant on the rendered manifest in CI.
diff --git a/docs/architecture/pr205-before-after.png b/docs/architecture/pr205-before-after.png
new file mode 100644
index 000000000..524481e1f
Binary files /dev/null and b/docs/architecture/pr205-before-after.png differ
diff --git a/docs/architecture/pr205-before-after.svg b/docs/architecture/pr205-before-after.svg
new file mode 100644
index 000000000..471f74cc3
--- /dev/null
+++ b/docs/architecture/pr205-before-after.svg
@@ -0,0 +1,71 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1620 880" font-family="Helvetica, Arial, sans-serif">
+<defs>
+<marker id="arr_2563eb" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#2563eb"/></marker>
+<marker id="arr_dc2626" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#dc2626"/></marker>
+<marker id="arr_16a34a" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#16a34a"/></marker>
+<marker id="arr_d97706" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#d97706"/></marker>
+<marker id="arr_64748b" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#64748b"/></marker>
+</defs>
+<style>text{font-family:Helvetica,Arial,sans-serif;}</style>
+<rect x="0" y="0" width="1620" height="880" rx="0" fill="#ffffff" stroke="#ffffff" stroke-width="0" opacity="1.0"/>
+<text x="810.0" y="46" font-size="27" fill="#0f172a" font-weight="700" text-anchor="middle">PR #205 — Decoupling HCC readiness: before vs after</text>
+<text x="810.0" y="76" font-size="15" fill="#475569" font-weight="400" text-anchor="middle">The Host Context Controller's /ready endpoint stops waiting for full fleet convergence</text>
+<rect x="30" y="100" width="760" height="660" rx="18" fill="#fff7f6" stroke="#fecaca" stroke-width="2" opacity="1.0"/>
+<rect x="830" y="100" width="760" height="660" rx="18" fill="#f5fbf7" stroke="#bbf7d0" stroke-width="2" opacity="1.0"/>
+<text x="410" y="136" font-size="19" fill="#dc2626" font-weight="700" text-anchor="middle">BEFORE · origin/dev (issue #204)</text>
+<text x="1210" y="136" font-size="19" fill="#16a34a" font-weight="700" text-anchor="middle">AFTER · PR #205</text>
+<rect x="100.0" y="168" width="600" height="58" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
+<text x="400" y="201.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>
+<line x1="400" y1="226" x2="400" y2="279" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>
+<rect x="100.0" y="282" width="600" height="128" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>
+<text x="400" y="330.05" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">Waits to reconcile the ENTIRE initial fleet</text>
+<text x="400" y="357.35" font-size="14" fill="#475569" font-weight="400" text-anchor="middle">108 McpServers · 22 Contexts · Hosts</text>
+<text x="400" y="383.94" font-size="15" fill="#dc2626" font-weight="700" text-anchor="middle">/ready = 503 for the entire wait</text>
+<rect x="636" y="300" width="132" height="92" rx="12" fill="#dc2626" stroke="#dc2626" stroke-width="0" opacity="1.0"/>
+<text x="702" y="336" font-size="15" fill="#ffffff" font-weight="700" text-anchor="middle">BLOCKED</text>
+<text x="702" y="362" font-size="22" fill="#ffffff" font-weight="700" text-anchor="middle">~651 s</text>
+<line x1="400" y1="410" x2="400" y2="467" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>
+<rect x="100.0" y="470" width="600" height="58" rx="12" fill="#f1f5f9" stroke="#64748b" stroke-width="2" opacity="1.0"/>
+<text x="400" y="503.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">/ready = 200 (only after the ENTIRE fleet converges)</text>
+<line x1="400" y1="528" x2="400" y2="585" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>
+<rect x="100.0" y="588" width="600" height="76" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>
+<text x="400" y="621.7" font-size="17" fill="#dc2626" font-weight="700" text-anchor="middle">rollout status TIMEOUT → deploy FAILS</text>
+<text x="400" y="649.0" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">even though the HCC is healthy and progressing</text>
+<text x="410" y="700" font-size="14" fill="#dc2626" font-weight="700" text-anchor="middle" font-style="italic">Readiness COUPLED to full fleet convergence</text>
+<rect x="890.0" y="168" width="520" height="54" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
+<text x="1150" y="199.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>
+<line x1="1150" y1="222" x2="1150" y2="265" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
+<rect x="890.0" y="268" width="520" height="50" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
+<text x="1150" y="297.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">BOUNDED inventory (bounded 4-LIST)</text>
+<line x1="1150" y1="318" x2="1150" y2="357" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
+<rect x="890.0" y="360" width="520" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>
+<text x="1150" y="391.89" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">Certifies authoritative fail-closed BARRIER</text>
+<text x="1150" y="417.77" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">revokes stale / unprovable NetworkPolicies</text>
+<text x="1150" y="442.22999999999996" font-size="13" fill="#d97706" font-weight="700" text-anchor="middle">BEFORE declaring Ready</text>
+<line x1="1150" y1="452" x2="1150" y2="493" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
+<rect x="890.0" y="496" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>
+<text x="1150" y="528.9300000000001" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">/ready = 200 (seconds)</text>
+<line x1="1150" y1="552" x2="1150" y2="593" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
+<rect x="890.0" y="596" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>
+<text x="1150" y="628.93" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">rollout OK → deploy OK</text>
+<path d="M 1410.0 524 H 1452 V 596" fill="none" stroke="#d97706" stroke-width="3" stroke-dasharray="7 5" marker-end="url(#arr_d97706)"/>
+<rect x="1336.0" y="596" width="232" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>
+<text x="1452" y="629.44" font-size="13" fill="#0f172a" font-weight="700" text-anchor="middle">ADDITIVE fleet convergence</text>
+<text x="1452" y="653.19" font-size="12" fill="#475569" font-weight="400" text-anchor="middle">async, AFTER Ready</text>
+<text x="1452" y="675.5200000000001" font-size="11" fill="#475569" font-weight="400" text-anchor="middle">(the rest of the policies converge)</text>
+<rect x="1362.0" y="357" width="200.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>
+<text x="1462" y="373" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">isCurrent required (J3)</text>
+<rect x="1382.0" y="389" width="160.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>
+<text x="1462" y="405" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">B3 retain / revoke</text>
+<rect x="1358.0" y="421" width="208.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>
+<text x="1462" y="437" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">single-writer Recreate·1</text>
+<text x="1150" y="700" font-size="14" fill="#16a34a" font-weight="700" text-anchor="middle" font-style="italic">Readiness DECOUPLED: gated on the safety barrier, not the fleet</text>
+<rect x="300" y="812" width="26" height="20" rx="5" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
+<text x="336" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Startup step</text>
+<rect x="506" y="812" width="26" height="20" rx="5" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>
+<text x="542" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Block / failure</text>
+<rect x="742" y="812" width="26" height="20" rx="5" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>
+<text x="778" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Fast Ready / OK</text>
+<rect x="978" y="812" width="26" height="20" rx="5" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>
+<text x="1014" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Barrier / async branch</text>
+</svg>
\ No newline at end of file
diff --git a/docs/architecture/pr205-hcc-flows.png b/docs/architecture/pr205-hcc-flows.png
new file mode 100644
index 000000000..d0bbfa3d7
Binary files /dev/null and b/docs/architecture/pr205-hcc-flows.png differ
diff --git a/docs/architecture/pr205-hcc-flows.svg b/docs/architecture/pr205-hcc-flows.svg
new file mode 100644
index 000000000..adb22120f
--- /dev/null
+++ b/docs/architecture/pr205-hcc-flows.svg
@@ -0,0 +1,126 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 880" font-family="Inter, Segoe UI, Helvetica, Arial, sans-serif">
+<defs>
+<marker id="aBlue" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#2563eb"/></marker>
+<marker id="aRose" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#dc2626"/></marker>
+<marker id="aViolet" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#7c3aed"/></marker>
+<marker id="aAmber" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#d97706"/></marker>
+<marker id="aGreen" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#059669"/></marker>
+<marker id="aGray" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#64748b"/></marker>
+<filter id="sh" x="-5%" y="-6%" width="110%" height="122%"><feDropShadow dx="0" dy="1.5" stdDeviation="1.8" flood-color="#0f172a" flood-opacity="0.16"/></filter>
+</defs>
+<rect width="1280" height="880" fill="#ffffff"/>
+<text x="40" y="36" font-size="20" font-weight="800" fill="#0f172a">host-context-controller — flujos que resuelve el PR #205</text>
+<text x="40" y="57" font-size="12.5" fill="#64748b">decouple readiness from initial fleet convergence · seguridad fail-closed · gobernanza single-writer de NetworkPolicy</text>
+<text x="40" y="92" font-size="13" font-weight="700" fill="#1d4ed8">1 · Lifecycle de readiness — DESACOPLADO de la convergencia de la flota</text>
+<rect x="40.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
+<text x="147.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">HCC start</text>
+<text x="147.0" y="146" text-anchor="middle" font-size="11" fill="#475569">4 watch lanes arrancan;</text>
+<text x="147.0" y="162" text-anchor="middle" font-size="11" fill="#475569">una lane que falla → recovery</text>
+<text x="147.0" y="178" text-anchor="middle" font-size="11" fill="#475569">in-process, NO mata el proceso</text>
+<rect x="286.5" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
+<text x="393.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">4-LIST inventory</text>
+<text x="393.5" y="146" text-anchor="middle" font-size="11" fill="#475569">Context ingress · mcp-host</text>
+<text x="393.5" y="162" text-anchor="middle" font-size="11" fill="#475569">egress · rpc-proxy egress</text>
+<text x="393.5" y="178" text-anchor="middle" font-size="11" fill="#475569">· external egress</text>
+<rect x="533.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
+<text x="640.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Certificar safety inventory</text>
+<text x="640.0" y="146" text-anchor="middle" font-size="11" fill="#475569">revoca allows stale/</text>
+<text x="640.0" y="162" text-anchor="middle" font-size="11" fill="#475569">improbables ANTES de</text>
+<text x="640.0" y="178" text-anchor="middle" font-size="11" fill="#475569">certificar (fail-closed)</text>
+<rect x="779.5" y="104" width="214" height="92" rx="11" fill="#dbeafe" stroke="#1d4ed8" stroke-width="1.8" filter="url(#sh)"/>
+<text x="886.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">/ready = 200</text>
+<text x="886.5" y="146" text-anchor="middle" font-size="11" fill="#475569">solo si el inventory está</text>
+<text x="886.5" y="162" text-anchor="middle" font-size="11" fill="#475569">certificado y ningún allow</text>
+<text x="886.5" y="178" text-anchor="middle" font-size="11" fill="#475569">stale sobrevive</text>
+<rect x="1026.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
+<text x="1133.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Additive convergence</text>
+<text x="1133.0" y="146" text-anchor="middle" font-size="11" fill="#475569">la flota converge DESPUÉS</text>
+<text x="1133.0" y="162" text-anchor="middle" font-size="11" fill="#475569">del ready — asíncrono,</text>
+<text x="1133.0" y="178" text-anchor="middle" font-size="11" fill="#475569">sin bloquearlo</text>
+<line x1="254.0" y1="150.0" x2="286.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>
+<rect x="229.05" y="130.0" width="82.4" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="270.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">inventariar</text>
+<line x1="500.5" y1="150.0" x2="533.0" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>
+<rect x="478.75" y="130.0" width="76.0" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="516.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">certificar</text>
+<line x1="747.0" y1="150.0" x2="779.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>
+<rect x="738.05" y="130.0" width="50.400000000000006" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="763.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">/ready</text>
+<line x1="993.5" y1="150.0" x2="1026.0" y2="150.0" stroke="#64748b" stroke-width="2.4" stroke-dasharray="6,4" marker-end="url(#aGray)" stroke-linecap="round"/>
+<rect x="946.15" y="130.0" width="127.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="1009.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#475569">async · post-ready</text>
+<text x="40" y="275" font-size="13" font-weight="700" fill="#b91c1c">2 · Fences fail-closed (R3 · R4 · J3)</text>
+<rect x="40.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
+<text x="230.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Readiness gate — default () =&gt; false</text>
+<text x="230.0" y="326" text-anchor="middle" font-size="11" fill="#475569">/ready = ready &amp;&amp; providerAuthoritativeFn()</text>
+<text x="230.0" y="342" text-anchor="middle" font-size="11" fill="#475569">throw → 503 · nunca un 200 equivocado</text>
+<rect x="450.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
+<text x="640.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">G1 — deny sin inventory-authority</text>
+<text x="640.0" y="326" text-anchor="middle" font-size="11" fill="#475569">currentWhenUnconfigured() → false + warn</text>
+<text x="640.0" y="342" text-anchor="middle" font-size="11" fill="#475569">dev nunca construye el reconciler</text>
+<rect x="860.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
+<text x="1050.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">F1 — rebind del readiness-poll</text>
+<text x="1050.0" y="326" text-anchor="middle" font-size="11" fill="#475569">re-liga isCurrent/server en reconciles</text>
+<text x="1050.0" y="342" text-anchor="middle" font-size="11" fill="#475569">superpuestos → la ventana no se autodestruye</text>
+<text x="40" y="421" font-size="13" font-weight="700" fill="#6d28d9">3 · Egress externo DNS — contrato B3 (retener/revocar) + sliding-window #299</text>
+<rect x="40" y="438" width="224" height="80" rx="11" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>
+<text x="152.0" y="461" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">DNS binding observado</text>
+<text x="152.0" y="480" text-anchor="middle" font-size="11" fill="#475569">resolveExternalEgressDns</text>
+<text x="152.0" y="496" text-anchor="middle" font-size="11" fill="#475569">timeout 5s + fence isCurrent</text>
+<polygon points="372,436 472,478 372,520 272,478" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>
+<text x="372" y="475" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">¿identity</text>
+<text x="372" y="491" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">unchanged?</text>
+<line x1="264" y1="478" x2="272" y2="478" stroke="#7c3aed" stroke-width="2.6" marker-end="url(#aViolet)" stroke-linecap="round"/>
+<rect x="540" y="414" width="306" height="60" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
+<text x="693.0" y="437" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">RETAIN — allow identity-stable</text>
+<text x="693.0" y="456" text-anchor="middle" font-size="11" fill="#475569">mismo protocolo/puerto/ownership/cidrs</text>
+<text x="693.0" y="472" text-anchor="middle" font-size="11" fill="#475569">· HCC sigue Ready (módulo cidr)</text>
+<rect x="540" y="496" width="306" height="60" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
+<text x="693.0" y="519" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">REVOKE antes de readiness</text>
+<text x="693.0" y="538" text-anchor="middle" font-size="11" fill="#475569">drift TCP→UDP · port · ownership · zero-cidr</text>
+<text x="693.0" y="554" text-anchor="middle" font-size="11" fill="#475569">→ unprovable → cleanup fail-closed</text>
+<path d="M 472,470 H 506.0 V 444 H 540" fill="none" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linejoin="round"/>
+<rect x="423.6" y="415" width="88.80000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="468" y="427" text-anchor="middle" font-size="10" font-weight="700" fill="#047857">sí · retiene</text>
+<path d="M 472,486 H 506.0 V 526 H 540" fill="none" stroke="#dc2626" stroke-width="2.6" marker-end="url(#aRose)" stroke-linejoin="round"/>
+<rect x="417.2" y="523" width="101.60000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="468" y="535" text-anchor="middle" font-size="10" font-weight="700" fill="#b91c1c">drift · revoca</text>
+<rect x="892" y="430" width="348" height="88" rx="11" fill="#fffbeb" stroke="#d97706" stroke-width="1.8" filter="url(#sh)"/>
+<text x="1066.0" y="453" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ExternalEgressConvergenceCoordinator</text>
+<text x="1066.0" y="472" text-anchor="middle" font-size="11" fill="#475569">resync periódico TTL-aware (≤ TTL/2, floor)</text>
+<text x="1066.0" y="488" text-anchor="middle" font-size="11" fill="#475569">sliding-window accumulate · provider CIDRs</text>
+<text x="1066.0" y="504" text-anchor="middle" font-size="11" fill="#475569">fence interval ≤ overlap/2 · limiter</text>
+<line x1="846" y1="444" x2="892" y2="460" stroke="#d97706" stroke-width="2.6" marker-end="url(#aAmber)" stroke-linecap="round"/>
+<rect x="821.4" y="425" width="95.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
+<text x="869" y="437" text-anchor="middle" font-size="10" font-weight="700" fill="#b45309">add (aditiva)</text>
+<text x="40" y="575" font-size="13" font-weight="700" fill="#047857">4 · Seam de rotación #223 (preservado) &amp; invariante de deploy single-writer</text>
+<rect x="40" y="584" width="236" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
+<text x="158.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">validateSecret()</text>
+<text x="158.0" y="626" text-anchor="middle" font-size="11" fill="#475569">sha256(canonicalJSON(secret.data))</text>
+<rect x="306" y="584" width="306" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
+<text x="459.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ensureDeployment(rev, isCurrent)</text>
+<text x="459.0" y="626" text-anchor="middle" font-size="11" fill="#475569">el fence de #205 se propaga aquí</text>
+<rect x="642" y="584" width="300" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
+<text x="792.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">pod annotation</text>
+<text x="792.0" y="626" text-anchor="middle" font-size="11" fill="#475569">clerum.io/credentials-revision → rueda 1 vez</text>
+<line x1="276" y1="617" x2="306" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>
+<line x1="612" y1="617" x2="642" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>
+<rect x="972" y="584" width="268" height="66" rx="11" fill="#f8fafc" stroke="#475569" stroke-width="1.8" filter="url(#sh)"/>
+<text x="1106.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Deploy single-writer</text>
+<text x="1106.0" y="626" text-anchor="middle" font-size="11" fill="#475569">strategy: Recreate · replicas 1</text>
+<text x="1106.0" y="642" text-anchor="middle" font-size="11" fill="#475569">· progressDeadlineSeconds 1200</text>
+<text x="40" y="700" font-size="11.5" font-weight="700" fill="#334155">Leyenda</text>
+<rect x="118" y="688" width="14" height="14" rx="3" fill="#2563eb" opacity="0.9"/>
+<text x="137" y="700" font-size="11" fill="#475569">lifecycle readiness</text>
+<rect x="282.6" y="688" width="14" height="14" rx="3" fill="#dc2626" opacity="0.9"/>
+<text x="301.6" y="700" font-size="11" fill="#475569">fence fail-closed / revoke</text>
+<rect x="492.0" y="688" width="14" height="14" rx="3" fill="#7c3aed" opacity="0.9"/>
+<text x="511.0" y="700" font-size="11" fill="#475569">egress DNS (B3)</text>
+<rect x="631.0" y="688" width="14" height="14" rx="3" fill="#d97706" opacity="0.9"/>
+<text x="650.0" y="700" font-size="11" fill="#475569">coordinator #299</text>
+<rect x="776.4" y="688" width="14" height="14" rx="3" fill="#059669" opacity="0.9"/>
+<text x="795.4" y="700" font-size="11" fill="#475569">retain / rotación #223</text>
+<rect x="960.2" y="688" width="14" height="14" rx="3" fill="#475569" opacity="0.9"/>
+<text x="979.2" y="700" font-size="11" fill="#475569">invariante deploy</text>
+<text x="40" y="858" font-size="10.5" fill="#94a3b8">PR #205 · evenfire-ai/evenfire · el /ready certifica una frontera de seguridad autoritativa ANTES de que la flota converja — la convergencia restante es aditiva y asíncrona</text>
+</svg>
\ No newline at end of file
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index e1227184a..0ae6125bb 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.7.0',
externalRestApiVersion: '0.1.80',
- rpcProxyVersion: '0.1.64',
+ rpcProxyVersion: '0.1.65',
desktopVersion: '0.7.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/host-context-controller/README.md b/host-context-controller/README.md
index 8ee40e981..169458690 100644
--- a/host-context-controller/README.md
+++ b/host-context-controller/README.md
@@ -39,6 +39,8 @@ When an `McpServer` CRD is created, modified, or deleted, the host-context-contr
- **MODIFIED** — Updates the Deployment and Service to match the CRD spec (env vars, image, ports, resources, etc.).
- **DELETED** — Removes the Deployment and Service.
+Remote egress-proxy McpServers keep the author's `spec.image` as desired-state input; HCC no longer rewrites it to the platform proxy image. The former `canonicalizeRemoteEgressProxyImage` step and its `ImageCanonicalized` status condition were removed (PR #205 readiness-safety cleanup): canonicalization mutated the observed CRD cache object and bumped the resource generation, retiring the in-flight reconcile before it could create the runtime resources. The platform-owned proxy image is instead selected at Deployment build time (`buildDeployment`), leaving the CRD spec untouched.
+
The reconciler builds Deployments with labels used for pod selection and network policy targeting:
| Label | Value | Purpose |
@@ -97,7 +99,21 @@ Servers removed from a Context lose their policies on the next reconcile; deleti
- **`exact-host`** (default) — a CIDR binding, or a public DNS hostname resolved to `/32` ipBlocks, on one declared port/protocol. CIDRs and resolved IPs are rejected if they overlap `PUBLIC_EGRESS_EXCEPT_CIDRS` (RFC1918, CGNAT, loopback, link-local incl. cloud metadata, documentation, benchmarking, multicast, reserved ranges). Hostnames like `localhost`, `metadata.goog`, `*.internal`, `*.svc`, `*.cluster.local` are rejected outright.
- **`public-web`** — `0.0.0.0/0` on TCP 80/443 with `PUBLIC_EGRESS_EXCEPT_CIDRS` carved out via `except`.
-Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS answers are re-resolved periodically (`HCC_EXTERNAL_EGRESS_RESYNC_SEC`, default 300s).
+Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS-derived allows are never retained across a failed refresh: HCC revokes the unprovable policy before certifying global readiness, while only the affected runtime remains blocked until DNS recovers. Each lookup has a bounded deadline.
+
+HCC readiness certifies authoritative inventory and revocation safety, not
+completion of positive policy creation. A renamed or newly requested allow may
+therefore remain unavailable after HCC becomes Ready until its additive
+reconciliation succeeds. HCC never keeps an obsolete allow merely to avoid
+that availability gap: when old and new ownership cannot be proven equivalent,
+revocation wins and only the affected runtime remains fail-closed.
+
+During an in-place release, HCC is also the controller that stamps
+`CONTEXT_MAPPER_HOST_IMAGE` onto managed mcp-host workloads. The new HCC
+producer is consequently running before it replaces those consumers with the
+new mcp-host image. For compatibility, an omitted `status.authoritative` field
+retains the legacy fail-closed meaning; only an explicit `authoritative: false`
+asks a new mcp-host to preserve an unchanged live connection.
### Recipe binding policies
@@ -105,7 +121,8 @@ McpServers carrying a `clerum.io/recipe-bindings` annotation (set for WorkflowRe
### Reconciliation Lifecycle
-- **Startup** — Ensures L0/L1 defaults, reconciles all Contexts and all McpServer egress bindings, then deletes orphaned context-allow policies in the MCP server namespace, plus orphaned rpc-proxy-egress and external-egress policies, whose owning CRD no longer exists (`fullReconcile`).
+- **Startup safety** — Ensures L0/L1 defaults, revokes stale or orphaned context-allow, rpc-proxy-egress, and external-egress policies from authoritative Context/McpServer inventories, then certifies readiness safety.
+- **Startup additive convergence** — Reconciles Context allows after certification. The dedicated external-egress coordinator is the sole owner of McpServer egress creation, per-server startup gating, retry, and periodic DNS refresh; a failed binding blocks only that server's runtime.
- **Context ADDED/MODIFIED/DELETED** — Creates, updates, or removes the L2 policy set for that context.
- **McpServer ADDED/MODIFIED/DELETED** — Reconciles L3 external egress (before the workload Deployment is created) and recipe binding policies.
@@ -267,16 +284,17 @@ a permissive policy — see the NetworkPolicy and GFS tables:
### MCP server provisioning
-| Variable | Default | Description |
-| -------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------- |
-| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |
-| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |
-| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync (seconds), so DNS changes converge without a watch event. `0` disables. |
+| Variable | Default | Description |
+| -------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------- |
+| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |
+| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |
+| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync interval (seconds). `0` disables periodic refresh; watch events still reconcile. |
+| `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS` | `5000` | Per-hostname DNS lookup deadline in milliseconds. |
### mcp-host provisioning
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index e9b28c157..9bcfc9c0f 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.170",
+ "version": "0.1.223",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.170",
+ "version": "0.1.223",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 17c98b81a..42fe8f398 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.170",
+ "version": "0.1.223",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
index f8d6857cb..f4e15bf35 100644
--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
@@ -119,6 +119,7 @@ describe('connector credentials revision (issue #223)', () => {
data: { 'api-key': stored('old-key') },
})
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -220,6 +221,7 @@ describe('generation-aware rollout readiness (issue #223)', () => {
vi.clearAllMocks()
coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -322,6 +324,7 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
vi.useFakeTimers()
coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -548,3 +551,111 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)
})
})
+
+describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {
+ const appsApi = createMockAppsApi()
+ const coreApi = createMockCoreApi()
+ const customApi = createMockCustomApi()
+ let reconciler: McpServerReconciler
+ // The inventory-authority generation IS mcpWatchGeneration in production
+ // (k8sClient wiring): a routine McpServer watch recovery bumps it.
+ let authorityGeneration: number
+ let currentServer: McpServerCRD
+
+ // A rollout mid-flight (total 2 vs updated 1) that never converges on its own.
+ const midRollout = {
+ metadata: { generation: 2, labels: {} },
+ spec: { replicas: 1 },
+ status: {
+ observedGeneration: 2,
+ replicas: 2,
+ updatedReplicas: 1,
+ readyReplicas: 1,
+ availableReplicas: 1,
+ unavailableReplicas: 0,
+ },
+ }
+
+ beforeEach(() => {
+ vi.clearAllMocks()
+ vi.useFakeTimers()
+ authorityGeneration = 1
+ currentServer = makeServer()
+ coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
+ reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(coreApi),
+ customApi: asCustomApi(customApi),
+ })
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: authorityGeneration }))
+ reconciler.setResolveCurrentServer(name => (name === 'linear' ? currentServer : undefined))
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('rebinds the fence so a converged rollout is recorded after a watch-generation bump (T1)', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
+ // R1 arms the window under authority generation 1 (timer pending).
+ await reconciler.reconcile(makeServer())
+ // A routine McpServer watch recovery bumps the inventory-authority generation.
+ authorityGeneration = 2
+ // R2, current under generation 2, finds R1's pending window and rebinds it.
+ await reconciler.reconcile(makeServer())
+ // The rollout converges before the (single) pending tick fires.
+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
+ await vi.advanceTimersByTimeAsync(5000)
+
+ // Without the rebind, R1's tick evaluates its stale generation-1 fence,
+ // self-deletes, and DeploymentReady=True is never written.
+ expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
+ expect(
+ (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls.size
+ ).toBe(0)
+ })
+
+ it('control: without the generation bump the rollout converges the same way (T2)', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
+ await reconciler.reconcile(makeServer())
+ await reconciler.reconcile(makeServer()) // no bump
+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
+ })
+
+ it('rebind does NOT reset the attempt budget across the bump (E2 guard, T3)', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout) // never converges
+ await reconciler.reconcile(makeServer())
+ await vi.advanceTimersByTimeAsync(2 * 5000) // 2 attempts consumed
+ authorityGeneration = 2
+ await reconciler.reconcile(makeServer()) // rebind — must NOT reset attempts
+ await vi.advanceTimersByTimeAsync(21 * 5000) // 2 + 21 = 23 attempts, still short of 24
+
+ expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(false)
+ await vi.advanceTimersByTimeAsync(5000) // the 24th tick
+ expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(true)
+ })
+
+ it('does NOT adopt a same-name recreate — a new uid gets a fresh window (T4)', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
+ await reconciler.reconcile(makeServer())
+ const polls = (reconciler as unknown as { readinessPolls: Map<string, { attempts: number }> })
+ .readinessPolls
+ const firstWindow = polls.get('linear')
+ await vi.advanceTimersByTimeAsync(12 * 5000)
+
+ // Same name, NEW uid: identity retirement (beginStatusTracking), not a
+ // rebind, must own this — a rebind adopting the recreate would carry the
+ // old window's spent budget.
+ const recreated = { ...makeServer(), uid: 'uid-linear-9999' }
+ currentServer = recreated
+ await reconciler.reconcile(recreated)
+
+ const secondWindow = polls.get('linear')
+ expect(secondWindow).not.toBe(firstWindow)
+ expect(secondWindow?.attempts).toBe(0)
+ })
+})
diff --git a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
index a4b425ae7..fa30ec7a5 100644
--- a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
@@ -89,6 +89,7 @@ describe('McpServer generated pod hardening', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.ts
index 09c0b5b84..044452efd 100644
--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts
@@ -87,6 +87,7 @@ function buildReconciler(
customApi: MockCustomApi
): McpServerReconciler {
return new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -171,7 +172,14 @@ function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {
const calls = customApi.patchNamespacedCustomObjectStatus.mock.calls
const lastCall = calls[calls.length - 1]?.[0]
const patch = lastCall?.body
- const value = Array.isArray(patch) ? patch[0]?.value : undefined
+ const statusAdd = Array.isArray(patch)
+ ? patch.find(
+ candidate =>
+ candidate?.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ : undefined
+ const value = statusAdd?.value
return Array.isArray(value) ? value : (value?.conditions ?? [])
}
@@ -668,105 +676,6 @@ describe('McpServerReconciler remote egress proxy', () => {
})
})
- describe('canonicalizeRemoteEgressProxyImage', () => {
- it('patches stale remote McpServer spec.image to the platform egress proxy image', async () => {
- const staleRemote = cloneServer(REMOTE_SERVER, {
- image: LEGACY_REMOTE_EGRESS_IMAGE,
- })
-
- await (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)
-
- expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(
- expect.objectContaining({
- group: 'clerum.io',
- version: 'v1alpha1',
- namespace: 'mcp-server',
- plural: 'mcpservers',
- name: 'mcp-sentry-remote',
- body: {
- spec: {
- image: 'clerum/nginx-egress-proxy:0.1.0',
- },
- },
- }),
- expect.objectContaining({
- middleware: expect.any(Array),
- })
- )
- expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')
- expect(lastPatchedStatusConditions(customApi)).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- type: 'ImageCanonicalized',
- status: 'True',
- reason: 'RemoteEgressProxyImageMatches',
- message: 'Remote McpServer spec.image matches the platform egress proxy image',
- }),
- ])
- )
- })
-
- it('does not patch spec for a remote McpServer already using the platform image', async () => {
- const currentRemote = cloneServer(REMOTE_SERVER)
-
- await (reconciler as any).canonicalizeRemoteEgressProxyImage(currentRemote)
-
- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
- expect(lastPatchedStatusConditions(customApi)).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- type: 'ImageCanonicalized',
- status: 'True',
- reason: 'RemoteEgressProxyImageMatches',
- message: 'Remote McpServer spec.image matches the platform egress proxy image',
- }),
- ])
- )
- })
-
- it('does not patch local McpServers even when their image differs', async () => {
- const localServer = cloneServer(LOCAL_SERVER, {
- image: 'clerum/custom-local-mcp:1.2.3',
- })
-
- await (reconciler as any).canonicalizeRemoteEgressProxyImage(localServer)
-
- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- expect(localServer.spec.image).toBe('clerum/custom-local-mcp:1.2.3')
- })
-
- it('does not throw and records status when declarative image patching fails', async () => {
- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
- const staleRemote = cloneServer(REMOTE_SERVER, {
- image: LEGACY_REMOTE_EGRESS_IMAGE,
- })
- customApi.patchNamespacedCustomObject.mockRejectedValueOnce(new Error('rbac denied'))
-
- await expect(
- (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)
- ).resolves.toBeUndefined()
-
- expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)
- expect(warn).toHaveBeenCalledWith(
- expect.stringContaining('Failed to canonicalize remote McpServer'),
- expect.any(Error)
- )
- expect(lastPatchedStatusConditions(customApi)).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- type: 'ImageCanonicalized',
- status: 'False',
- reason: 'RemoteEgressProxyImagePatchFailed',
- message:
- 'Failed to patch remote McpServer spec.image to the platform egress proxy image',
- }),
- ])
- )
- warn.mockRestore()
- })
- })
-
// ─── Test 3: buildDeployment for local servers ─────────────────────────
describe('buildDeployment for local servers', () => {
@@ -835,16 +744,12 @@ describe('McpServerReconciler remote egress proxy', () => {
expect(cmIndex).toBeLessThan(deployIndex)
})
- it('should canonicalize stale remote image before syncing ConfigMap and Deployment', async () => {
+ it('should use the platform image without rewriting a divergent remote desired image', async () => {
const callOrder: string[] = []
const staleRemote = cloneServer(REMOTE_SERVER, {
image: LEGACY_REMOTE_EGRESS_IMAGE,
})
- customApi.patchNamespacedCustomObject.mockImplementation(async () => {
- callOrder.push('canonicalizeImage')
- return {}
- })
coreApi.createNamespacedConfigMap.mockImplementation(async () => {
callOrder.push('createConfigMap')
return {}
@@ -860,23 +765,28 @@ describe('McpServerReconciler remote egress proxy', () => {
await reconciler.reconcile(staleRemote)
- expect(callOrder).toEqual([
- 'canonicalizeImage',
- 'createConfigMap',
- 'createService',
- 'createDeployment',
- ])
- expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')
- expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(
- expect.objectContaining({
- body: {
- spec: {
- image: 'clerum/nginx-egress-proxy:0.1.0',
- },
- },
- }),
+ expect(callOrder).toEqual(['createConfigMap', 'createService', 'createDeployment'])
+ expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)
+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(
+ expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),
+ expect.anything()
+ )
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(
expect.objectContaining({
- middleware: expect.any(Array),
+ body: expect.objectContaining({
+ spec: expect.objectContaining({
+ template: expect.objectContaining({
+ spec: expect.objectContaining({
+ containers: [
+ expect.objectContaining({
+ name: 'egress-proxy',
+ image: 'clerum/nginx-egress-proxy:0.1.0',
+ }),
+ ],
+ }),
+ }),
+ }),
+ }),
})
)
})
@@ -1115,6 +1025,12 @@ describe('McpServerReconciler remote egress proxy', () => {
})
describe('reconcileDelete for remote servers', () => {
+ beforeEach(() => {
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 1 }))
+ reconciler.setResolveCurrentServer(() => undefined)
+ customApi.getNamespacedCustomObject.mockRejectedValue(make404Error())
+ })
+
it('should clean up ConfigMap via deleteResources', async () => {
mockHccOwnedRuntimeReads(appsApi, coreApi)
@@ -1136,11 +1052,14 @@ describe('McpServerReconciler remote egress proxy', () => {
})
it('should clear status tracking after delete', async () => {
+ let current: McpServerCRD | undefined = REMOTE_SERVER
+ reconciler.setResolveCurrentServer(() => current)
// First reconcile to populate status
await reconciler.reconcile(REMOTE_SERVER)
expect(reconciler.getStatus('mcp-sentry-remote').deployed).toBe(true)
// Then delete
+ current = undefined
await reconciler.reconcileDelete('mcp-sentry-remote', 'mcp-server')
const status = reconciler.getStatus('mcp-sentry-remote')
expect(status.deployed).toBe(false)
diff --git a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
index b3d979815..38b7c571f 100644
--- a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
@@ -91,6 +91,7 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -122,6 +123,19 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {
const container = capturedContainer(appsApi)
expect(container.imagePullPolicy).toBe('IfNotPresent')
})
+
+ it('does not mutate a watched desired spec while building the sanitized Deployment', async () => {
+ const server = makeServer({
+ imagePullPolicy: 'Always',
+ security: { runAsUser: 0, addCapabilities: ['CHOWN', 'SYS_ADMIN'] },
+ })
+ const originalSpec = structuredClone(server.spec)
+
+ await reconciler.reconcile(server)
+
+ expect(capturedContainer(appsApi).imagePullPolicy).toBe('IfNotPresent')
+ expect(server.spec).toEqual(originalSpec)
+ })
})
// ════════════════════════════════════════════════════════════════════
diff --git a/host-context-controller/src/bindingPolicyReconciler.test.ts b/host-context-controller/src/bindingPolicyReconciler.test.ts
index b2fd2bc8f..48d6fe191 100644
--- a/host-context-controller/src/bindingPolicyReconciler.test.ts
+++ b/host-context-controller/src/bindingPolicyReconciler.test.ts
@@ -2,6 +2,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'
+function deferred<T = void>() {
+ let resolve!: (value: T | PromiseLike<T>) => void
+ const promise = new Promise<T>(res => {
+ resolve = res
+ })
+ return { promise, resolve }
+}
+
function makeMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
@@ -40,7 +48,9 @@ describe('BindingPolicyReconciler', () => {
const singleBinding: BindingDef[] = [{ from: 'redis-mcp', to: 'redis', port: 6379 }]
it('creates egress + ingress policies for a binding', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
+ isCurrent: () => true,
+ })
// 2 calls: 1 egress + 1 ingress
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
@@ -75,7 +85,9 @@ describe('BindingPolicyReconciler', () => {
})
it('applies binding-allow labels to all policies', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
+ isCurrent: () => true,
+ })
for (const call of mockApi.createNamespacedNetworkPolicy.mock.calls) {
const labels = (call[0] as { body: { metadata: { labels: Record<string, string> } } }).body
@@ -89,12 +101,43 @@ describe('BindingPolicyReconciler', () => {
it('handles 409 conflict with create-or-replace pattern', async () => {
mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
+ isCurrent: () => true,
+ })
expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()
})
+ it('does not replace or start another apply after its authority lease is retired', async () => {
+ const readStarted = deferred()
+ const releaseRead = deferred<{ metadata: { resourceVersion: string } }>()
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockApi.readNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ readStarted.resolve()
+ return releaseRead.promise
+ })
+ let current = true
+
+ const reconcile = reconciler.reconcileBindings(
+ 'my-recipe',
+ singleBinding,
+ 'redis-mcp',
+ 'redis-mcp',
+ { isCurrent: () => current }
+ )
+ await readStarted.promise
+
+ current = false
+ releaseRead.resolve({ metadata: { resourceVersion: '1' } })
+ await reconcile
+
+ // The first create was authorized and returned 409 without mutating. Once
+ // the lease is retired, no replacement or second policy apply may start.
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('cleanupBindings deletes policies in both namespaces', async () => {
mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
@@ -108,8 +151,65 @@ describe('BindingPolicyReconciler', () => {
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalled()
})
+ it('does not delete listed policies when a same-name recipe is recreated during LIST', async () => {
+ const bothListsStarted = deferred()
+ const releaseLists = deferred<{
+ items: Array<{ metadata: { name: string } }>
+ }>()
+ let listCalls = 0
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {
+ listCalls += 1
+ if (listCalls === 2) bothListsStarted.resolve()
+ return releaseLists.promise
+ })
+ let recipeRecreated = false
+ const deleteAllowed = vi.fn(async () => !recipeRecreated)
+
+ const cleanup = reconciler.cleanupBindings('my-recipe', { deleteAllowed })
+ await bothListsStarted.promise
+
+ recipeRecreated = true
+ releaseLists.resolve({
+ items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
+ })
+ await cleanup
+
+ expect(deleteAllowed).toHaveBeenCalledTimes(2)
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('rejects cleanup when listing binding policies fails', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockRejectedValue(new Error('transient LIST failure'))
+
+ await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(
+ 'Failed to clean up binding policies for recipe "my-recipe"'
+ )
+ })
+
+ it('rejects cleanup when deleting a binding policy fails with a non-404 error', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
+ })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('transient DELETE failure'))
+
+ await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(
+ 'Failed to clean up binding policies for recipe "my-recipe"'
+ )
+ })
+
+ it('treats a 404 while deleting binding policies as successful cleanup', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
+ })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
+
+ await expect(reconciler.cleanupBindings('my-recipe')).resolves.toBeUndefined()
+ })
+
it('uses _from for ingress rules (K8s client ObjectSerializer maps _from → from)', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
+ isCurrent: () => true,
+ })
const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
const ingressCall = calls.find(
@@ -126,7 +226,9 @@ describe('BindingPolicyReconciler', () => {
})
it('uses correct pod selectors and namespace-aware placement', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
+ isCurrent: () => true,
+ })
const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
const egressCall = calls.find(
@@ -196,7 +298,8 @@ describe('BindingPolicyReconciler', () => {
'my-recipe',
singleBinding,
'redis-mcp',
- 'my-recipe-redis-mcp'
+ 'my-recipe-redis-mcp',
+ { isCurrent: () => true }
)
const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
diff --git a/host-context-controller/src/bindingPolicyReconciler.ts b/host-context-controller/src/bindingPolicyReconciler.ts
index b0486d63c..0281e69f6 100644
--- a/host-context-controller/src/bindingPolicyReconciler.ts
+++ b/host-context-controller/src/bindingPolicyReconciler.ts
@@ -37,6 +37,18 @@ export interface BindingDef {
protocol?: 'TCP' | 'UDP'
}
+export interface BindingPolicyReconcileOptions {
+ // Required (fail-closed): every caller must supply the authority predicate so
+ // a retired reconcile pass cannot keep mutating binding NetworkPolicies. There
+ // is no assume-current default — an omitted predicate is a compile error, not
+ // a silent fail-open (the old `?? (() => true)`) nor a silent no-op.
+ isCurrent: () => boolean
+}
+
+export interface BindingPolicyCleanupOptions {
+ deleteAllowed?: () => Promise<boolean>
+}
+
export class BindingPolicyReconciler {
private networkingApi: k8s.NetworkingV1Api
private mcpServerNamespace: string
@@ -60,70 +72,86 @@ export class BindingPolicyReconciler {
recipeName: string,
bindings: BindingDef[],
mcpWorkloadName: string,
- mcpServerName = mcpWorkloadName
+ mcpServerName = mcpWorkloadName,
+ options: BindingPolicyReconcileOptions
): Promise<void> {
+ const isCurrent = options.isCurrent
+ if (!isCurrent()) return
console.log(`[BindingNP] Reconciling ${bindings.length} binding(s) for recipe "${recipeName}"`)
- await Promise.all(
- bindings.map(async binding => {
- const protocol = binding.protocol ?? 'TCP'
+ // Sequence writes so authority can be rechecked between every Kubernetes
+ // effect. A retired pass must not start the next binding policy mutation.
+ for (const binding of bindings) {
+ if (!isCurrent()) return
+ const protocol = binding.protocol ?? 'TCP'
- // Determine namespaces based on which workload is the MCP server.
- // The MCP workload (with transport) lives in mcp-server namespace,
- // the other workload lives in sandbox-recipes namespace.
- const fromIsMcp = binding.from === mcpWorkloadName
- const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE
- const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace
- const fromPodSelector: Record<string, string> = fromIsMcp
- ? { [MCPSERVER_LABEL]: mcpServerName }
- : { app: binding.from }
- const toPodSelector: Record<string, string> = fromIsMcp
- ? { app: binding.to }
- : { [MCPSERVER_LABEL]: mcpServerName }
+ // Determine namespaces based on which workload is the MCP server.
+ // The MCP workload (with transport) lives in mcp-server namespace,
+ // the other workload lives in sandbox-recipes namespace.
+ const fromIsMcp = binding.from === mcpWorkloadName
+ const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE
+ const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace
+ const fromPodSelector: Record<string, string> = fromIsMcp
+ ? { [MCPSERVER_LABEL]: mcpServerName }
+ : { app: binding.from }
+ const toPodSelector: Record<string, string> = fromIsMcp
+ ? { app: binding.to }
+ : { [MCPSERVER_LABEL]: mcpServerName }
- const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')
- const egressPolicy = this.buildEgressPolicy(
- egressName,
- recipeName,
- fromPodSelector,
- toPodSelector,
- fromNs,
- toNs,
- binding.port,
- protocol
- )
- const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')
- const ingressPolicy = this.buildIngressPolicy(
- ingressName,
- recipeName,
- fromPodSelector,
- toPodSelector,
- fromNs,
- toNs,
- binding.port,
- protocol
- )
+ const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')
+ const egressPolicy = this.buildEgressPolicy(
+ egressName,
+ recipeName,
+ fromPodSelector,
+ toPodSelector,
+ fromNs,
+ toNs,
+ binding.port,
+ protocol
+ )
+ const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')
+ const ingressPolicy = this.buildIngressPolicy(
+ ingressName,
+ recipeName,
+ fromPodSelector,
+ toPodSelector,
+ fromNs,
+ toNs,
+ binding.port,
+ protocol
+ )
- // Egress and ingress target different namespaces — safe to parallelize
- await Promise.all([
- this.applyPolicy(egressName, egressPolicy, fromNs),
- this.applyPolicy(ingressName, ingressPolicy, toNs),
- ])
- })
- )
+ if (!isCurrent()) return
+ await this.applyPolicy(egressName, egressPolicy, fromNs, isCurrent)
+ if (!isCurrent()) return
+ await this.applyPolicy(ingressName, ingressPolicy, toNs, isCurrent)
+ }
}
/**
* Delete all binding-scoped NetworkPolicies for a recipe.
*/
- async cleanupBindings(recipeName: string): Promise<void> {
+ async cleanupBindings(
+ recipeName: string,
+ options: BindingPolicyCleanupOptions = {}
+ ): Promise<void> {
console.log(`[BindingNP] Cleaning up binding policies for recipe "${recipeName}"`)
- // Delete from both namespaces in parallel (independent operations)
- await Promise.all([
- this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE),
- this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace),
+ // LIST operations are read-only and namespace-independent. Each resulting
+ // delete is separately authorized after its LIST completes.
+ const results = await Promise.allSettled([
+ this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE, options.deleteAllowed),
+ this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace, options.deleteAllowed),
])
+ const failures = results.flatMap(result =>
+ result.status === 'rejected' ? [result.reason] : []
+ )
+ if (failures.length > 0) {
+ throw new AggregateError(
+ failures,
+ `Failed to clean up binding policies for recipe "${recipeName}"`
+ )
+ }
}
// ─── Policy Builders ────────────────────────────────────────────
@@ -242,43 +270,49 @@ export class BindingPolicyReconciler {
private async applyPolicy(
name: string,
policy: k8s.V1NetworkPolicy,
- namespace: string
+ namespace: string,
+ isCurrent: () => boolean
): Promise<void> {
- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]')
+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]', isCurrent)
}
- private async deletePoliciesForRecipe(recipeName: string, namespace: string): Promise<void> {
+ private async deletePoliciesForRecipe(
+ recipeName: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
+ let response: k8s.V1NetworkPolicyList
try {
- const response = await this.networkingApi.listNamespacedNetworkPolicy({
+ response = await this.networkingApi.listNamespacedNetworkPolicy({
namespace,
labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=binding-allow,${RECIPE_LABEL}=${recipeName}`,
})
-
- await Promise.all(
- (response.items || []).map(async policy => {
- const policyName = policy.metadata?.name
- if (!policyName) {
- console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)
- return
- }
- try {
- await this.networkingApi.deleteNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- })
- console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)
- } catch (error: unknown) {
- if (getErrorCode(error) !== 404) {
- console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)
- }
- }
- })
- )
} catch (error) {
console.error(
`[BindingNP] Failed to list policies for recipe "${recipeName}" in ${namespace}:`,
error
)
+ throw error
+ }
+
+ for (const policy of response.items || []) {
+ const policyName = policy.metadata?.name
+ if (!policyName) {
+ console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)
+ continue
+ }
+ try {
+ if (deleteAllowed && !(await deleteAllowed())) return
+ await this.networkingApi.deleteNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) continue
+ console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)
+ throw error
+ }
}
}
}
diff --git a/host-context-controller/src/config.externalEgressDnsTimeout.test.ts b/host-context-controller/src/config.externalEgressDnsTimeout.test.ts
new file mode 100644
index 000000000..7396924b9
--- /dev/null
+++ b/host-context-controller/src/config.externalEgressDnsTimeout.test.ts
@@ -0,0 +1,46 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+const originalEnv = process.env
+
+async function loadConfig(raw: string | undefined) {
+ vi.resetModules()
+ process.env = { ...originalEnv }
+ if (raw === undefined) {
+ delete process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS
+ } else {
+ process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS = raw
+ }
+ return import('./config')
+}
+
+afterEach(() => {
+ process.env = originalEnv
+ vi.resetModules()
+})
+
+describe('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS', () => {
+ it.each([undefined, '', ' '])('defaults to 5000 for %s', async raw => {
+ const { config } = await loadConfig(raw)
+
+ expect(config.externalEgressDnsResolveTimeoutMs).toBe(5_000)
+ })
+
+ it.each([
+ ['1', 1],
+ ['5000', 5_000],
+ ['2147483647', 2_147_483_647],
+ ])('accepts the positive bounded integer %s', async (raw, expected) => {
+ const { config } = await loadConfig(raw)
+
+ expect(config.externalEgressDnsResolveTimeoutMs).toBe(expected)
+ })
+
+ it.each(['0', '-1', '1.5', '5000junk', 'NaN', 'Infinity', '2147483648'])(
+ "fails configuration validation for '%s'",
+ async raw => {
+ await expect(loadConfig(raw)).rejects.toThrow(
+ 'HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer'
+ )
+ }
+ )
+})
diff --git a/host-context-controller/src/config.ts b/host-context-controller/src/config.ts
index 7b7632b75..162ccd06e 100644
--- a/host-context-controller/src/config.ts
+++ b/host-context-controller/src/config.ts
@@ -197,6 +197,10 @@ export interface Config {
// so accumulated IPs never expire between refreshes (issue #299).
externalEgressResyncIntervalSec: number
+ // Deadline for one DNS resolution attempt. A silent resolver cannot block
+ // safety convergence indefinitely.
+ externalEgressDnsResolveTimeoutMs: number
+
// Grace kept after a DNS TTL before an accumulated /32 may expire (seconds).
// The sliding window that fixes #299: entries live for TTL + overlap, so a
// provider that rotates a single A record still resolves through the union of
@@ -245,6 +249,17 @@ function getEnvInt(key: string, defaultValue: number): number {
return isNaN(parsed) ? defaultValue : parsed
}
+export function parseExternalEgressDnsTimeoutMs(raw: string | undefined): number {
+ if (raw === undefined || raw.trim() === '') return 5_000
+ const parsed = Number(raw)
+ if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 2_147_483_647) {
+ throw new Error(
+ `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer no greater than 2147483647, got '${raw}'`
+ )
+ }
+ return parsed
+}
+
/**
* Loud integer env parser for the external-egress knobs (audit R1-L1). Unlike
* getEnvInt (parseInt), this rejects non-canonical values instead of silently
@@ -714,6 +729,9 @@ export const config: Config = {
// they expire, closing the #299 rotation gap. 0 disables (reintroduces the
// stale-snapshot risk; only for operators who explicitly opt out).
externalEgressResyncIntervalSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_RESYNC_SEC', 60),
+ externalEgressDnsResolveTimeoutMs: parseExternalEgressDnsTimeoutMs(
+ getEnv('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS')
+ ),
externalEgressOverlapSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_OVERLAP_SEC', 300),
externalEgressRefreshFloorSec: getExternalEgressEnvInt(
'HCC_EXTERNAL_EGRESS_REFRESH_FLOOR_SEC',
diff --git a/host-context-controller/src/deploymentStrategy.test.ts b/host-context-controller/src/deploymentStrategy.test.ts
new file mode 100644
index 000000000..7a9258139
--- /dev/null
+++ b/host-context-controller/src/deploymentStrategy.test.ts
@@ -0,0 +1,66 @@
+import { describe, expect, it } from 'vitest'
+import { loadAll } from 'js-yaml'
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+
+/**
+ * HCC's safety model assumes a single writer: replicas:1 with NO leader
+ * election (see statelessLifecycleExecutor.ts — "Running >1 replica risks two
+ * controllers racing"). The default Deployment strategy is RollingUpdate with
+ * maxSurge>0, which starts the new pod before the old one exits — a guaranteed
+ * two-replica overlap on every rollout. During that window the outgoing replica
+ * can win a conflict-retry replace (applyNetworkPolicy → replaceWithConflictRetry)
+ * and land its wider spec back over a policy the new replica just hardened and
+ * certified. There is no NetworkPolicy watch or periodic resync to heal it, and
+ * the new replica's safety fence only sees failures in its own process — so
+ * /ready stays 200 over a stale allow. Recreate closes the overlap, which is the
+ * only way to honour the documented single-writer assumption during a rollout.
+ */
+describe('host-context-controller Deployment strategy', () => {
+ const docs = loadAll(
+ readFileSync(
+ path.join(__dirname, '../../deploy/base/control-plane/host-context-controller.yaml'),
+ 'utf8'
+ )
+ ) as Array<{ kind?: string; metadata?: { name?: string }; spec?: Record<string, unknown> }>
+
+ const deployment = docs.find(
+ d => d?.kind === 'Deployment' && d?.metadata?.name === 'host-context-controller'
+ )
+
+ it('defines the host-context-controller Deployment', () => {
+ expect(deployment).toBeDefined()
+ })
+
+ it('runs a single replica', () => {
+ expect(deployment?.spec?.replicas).toBe(1)
+ })
+
+ it('uses Recreate so a rollout never overlaps two writers', () => {
+ // RollingUpdate (the default) would start the new pod before terminating the
+ // old one; with no leader election that races two NetworkPolicy writers.
+ const strategy = deployment?.spec?.strategy as { type?: string } | undefined
+ expect(strategy?.type).toBe('Recreate')
+ })
+
+ it('carries no rollingUpdate block under Recreate', () => {
+ // A leftover spec.strategy.rollingUpdate is invalid under Recreate and, if a
+ // future edit flipped only `type` back, would quietly reintroduce surge.
+ const strategy = deployment?.spec?.strategy as { rollingUpdate?: unknown } | undefined
+ expect(strategy?.rollingUpdate).toBeUndefined()
+ })
+
+ it('bounds rollout-stall detection above the worst legitimate startup', () => {
+ // Detection only, not recovery: Recreate + replicas:1 means a botched
+ // rollout is healed operationally (kubectl rollout undo), never by K8s.
+ // This deadline just flips Progressing=False / fails `kubectl rollout
+ // status` deterministically for observers without their own --timeout. It
+ // must exceed the worst legitimate startup, which — until #205 fully
+ // decouples readiness from fleet convergence — scales with the McpServer/
+ // Context fleet: origin/dev observed 651s on clerum-dev (108 McpServers /
+ // 22 Contexts) on 2026-08-05, so 1200s (900s wait + 300s head-start) is the
+ // merged mitigation. Comes back down once readiness no longer waits on the
+ // fleet.
+ expect(deployment?.spec?.progressDeadlineSeconds).toBe(1200)
+ })
+})
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
new file mode 100644
index 000000000..ac8a5d926
--- /dev/null
+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
@@ -0,0 +1,968 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+ ExternalEgressConvergenceCoordinator,
+ type ExternalEgressRetryHandle,
+ type ExternalEgressWatchEventType,
+} from './externalEgressConvergenceCoordinator'
+import { externalEgressRetriesAtCap } from './metrics'
+import type { McpServerCRD } from './types'
+
+function deferred<T = void>() {
+ let resolve!: (value: T | PromiseLike<T>) => void
+ let reject!: (reason?: unknown) => void
+ const promise = new Promise<T>((resolvePromise, rejectPromise) => {
+ resolve = resolvePromise
+ reject = rejectPromise
+ })
+ return { promise, resolve, reject }
+}
+
+function server(
+ name: string,
+ generation = 1,
+ options: { bindings?: boolean; namespace?: string } = {}
+): McpServerCRD {
+ return {
+ name,
+ namespace: options.namespace ?? 'mcp-server',
+ uid: `${name}-uid`,
+ generation,
+ spec: {
+ contextRef: 'default',
+ image: `clerum/${name}:v${generation}`,
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings:
+ options.bindings === false ? undefined : [{ dns: `${name}.example`, port: 443 }],
+ },
+ }
+}
+
+type CoordinatorOverrides = Partial<
+ ConstructorParameters<typeof ExternalEgressConvergenceCoordinator>[0]
+>
+
+function coordinator(overrides: CoordinatorOverrides = {}) {
+ const servers = new Map<string, McpServerCRD>()
+ const mutate = vi.fn(async () => undefined)
+ const replay = vi.fn(async () => undefined)
+ const instance = new ExternalEgressConvergenceCoordinator({
+ listServers: () => [...servers.values()],
+ getCurrentServer: name => servers.get(name),
+ inventoryAuthoritative: () => true,
+ sameDesiredRevision: (left, right) =>
+ left.uid === right.uid &&
+ left.generation === right.generation &&
+ JSON.stringify(left.spec) === JSON.stringify(right.spec),
+ enqueue: async (_selected, work) => work(),
+ mutate,
+ replay,
+ externalEgressRefreshMinTtlMs: () => Infinity,
+ ...overrides,
+ })
+ return { instance, mutate, replay, servers }
+}
+
+async function flushMicrotasks(): Promise<void> {
+ await Promise.resolve()
+ await Promise.resolve()
+ await Promise.resolve()
+}
+
+describe('ExternalEgressConvergenceCoordinator', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it.each([
+ ['presence', 'DELETED' as const, undefined],
+ ['revision', 'MODIFIED' as const, server('redis-tools', 2)],
+ ])(
+ 'lets an existing timer consume the latest %s intent and resets its attempt ladder',
+ async (_change, nextType, nextServer) => {
+ vi.useFakeTimers()
+ const initial = server('redis-tools', 1)
+ const state = new Map<string, McpServerCRD>([[initial.name, initial]])
+ const replayed: Array<{ type: ExternalEgressWatchEventType; server: McpServerCRD }> = []
+ let instance!: ExternalEgressConvergenceCoordinator
+ instance = coordinator({
+ getCurrentServer: name => state.get(name),
+ replay: async (type, selected) => {
+ replayed.push({ type, server: selected })
+ instance.scheduleRetry(type, selected)
+ },
+ }).instance
+
+ instance.scheduleRetry('ADDED', initial)
+ if (nextServer) state.set(nextServer.name, nextServer)
+ else state.delete(initial.name)
+ instance.scheduleRetry(nextType, nextServer ?? initial)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(replayed).toEqual([
+ {
+ type: nextType,
+ server: expect.objectContaining({
+ generation: nextServer?.generation ?? initial.generation,
+ }),
+ },
+ ])
+
+ await vi.advanceTimersByTimeAsync(4999)
+ expect(replayed).toHaveLength(1)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(replayed).toHaveLength(2)
+ instance.stop()
+ }
+ )
+
+ it('continues retrying at the capped delay until the replay succeeds', async () => {
+ vi.useFakeTimers()
+ const selected = server('web-search')
+ let instance!: ExternalEgressConvergenceCoordinator
+ let attempts = 0
+ instance = coordinator({
+ getCurrentServer: () => selected,
+ replay: async (type, current, retry) => {
+ attempts += 1
+ if (attempts < 5) {
+ instance.scheduleRetry(type, current)
+ return
+ }
+ retry.complete()
+ },
+ }).instance
+
+ instance.scheduleRetry('ADDED', selected)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.advanceTimersByTimeAsync(15000)
+ await vi.advanceTimersByTimeAsync(30000)
+ expect(attempts).toBe(3)
+ await vi.advanceTimersByTimeAsync(29999)
+ expect(attempts).toBe(3)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(attempts).toBe(4)
+ await vi.advanceTimersByTimeAsync(29999)
+ expect(attempts).toBe(4)
+ await vi.advanceTimersByTimeAsync(1)
+
+ expect(attempts).toBe(5)
+ await vi.advanceTimersByTimeAsync(30000)
+ expect(attempts).toBe(5)
+ instance.stop()
+ })
+
+ it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {
+ vi.useFakeTimers()
+ const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')
+ const selected = server('web-search')
+ let instance!: ExternalEgressConvergenceCoordinator
+ let succeed = false
+ instance = coordinator({
+ getCurrentServer: () => selected,
+ replay: async (type, current, retry) => {
+ if (succeed) {
+ retry.complete()
+ return
+ }
+ instance.scheduleRetry(type, current)
+ },
+ }).instance
+
+ instance.scheduleRetry('ADDED', selected)
+ // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(setSpy).not.toHaveBeenCalled()
+
+ // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.
+ await vi.advanceTimersByTimeAsync(15000)
+ expect(setSpy).toHaveBeenLastCalledWith(1)
+
+ // Once the replay finally succeeds, the capped server is cleared back to 0.
+ succeed = true
+ await vi.advanceTimersByTimeAsync(30000)
+ expect(setSpy).toHaveBeenLastCalledWith(0)
+
+ instance.stop()
+ setSpy.mockRestore()
+ })
+
+ it('never clears a pending full replay after an egress-only success', async () => {
+ vi.useFakeTimers()
+ const selected = server('web-search')
+ const { instance, replay, servers } = coordinator()
+ servers.set(selected.name, selected)
+
+ instance.scheduleRetry('ADDED', selected)
+ const retry = await instance.reconcile('MODIFIED', selected)
+ expect(retry).toBeDefined()
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(replay).toHaveBeenCalledOnce()
+ instance.stop()
+ })
+
+ it('clears a retry only after its complete full-replay handle finishes', async () => {
+ vi.useFakeTimers()
+ const selected = server('web-search')
+ let instance!: ExternalEgressConvergenceCoordinator
+ const replays: ExternalEgressRetryHandle[] = []
+ instance = coordinator({
+ getCurrentServer: () => selected,
+ replay: async (type, current, retry) => {
+ replays.push(retry)
+ const completion = await instance.reconcile(type, current, { retry })
+ expect(completion).toBe(retry)
+ retry.complete()
+ },
+ }).instance
+
+ instance.scheduleRetry('ADDED', selected)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(replays).toHaveLength(1)
+ instance.stop()
+ })
+
+ it('prevents a stale retry handle from clearing a newer intent', async () => {
+ vi.useFakeTimers()
+ const first = server('web-search', 1)
+ const second = server('web-search', 2)
+ const state = new Map<string, McpServerCRD>([[first.name, first]])
+ const handles: ExternalEgressRetryHandle[] = []
+ const replayed: McpServerCRD[] = []
+ const firstReplayStarted = deferred()
+ const releaseFirstReplay = deferred()
+ const { instance } = coordinator({
+ getCurrentServer: name => state.get(name),
+ replay: async (_type, current, retry) => {
+ replayed.push(current)
+ handles.push(retry)
+ if (replayed.length === 1) {
+ firstReplayStarted.resolve()
+ await releaseFirstReplay.promise
+ }
+ },
+ })
+
+ instance.scheduleRetry('ADDED', first)
+ await vi.advanceTimersByTimeAsync(5000)
+ await firstReplayStarted.promise
+ expect(handles).toHaveLength(1)
+
+ state.set(second.name, second)
+ instance.scheduleRetry('MODIFIED', second)
+ handles[0].complete()
+ releaseFirstReplay.resolve()
+ await flushMicrotasks()
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(replayed).toEqual([
+ expect.objectContaining({ generation: 1 }),
+ expect.objectContaining({ generation: 2 }),
+ ])
+ instance.stop()
+ })
+
+ it('keeps retry, resync, and startup concurrency lanes independent', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const retryRelease = deferred()
+ const activeRetries = deferred()
+ let retryCount = 0
+ const resyncServer = server('resync-server')
+ const startupServer = server('startup-server')
+ const canonical = new Map<string, McpServerCRD>([
+ [resyncServer.name, resyncServer],
+ [startupServer.name, startupServer],
+ ])
+ const mutations: string[] = []
+ const { instance } = coordinator({
+ listServers: () => [resyncServer],
+ getCurrentServer: name => canonical.get(name),
+ mutate: async (_type, selected) => {
+ mutations.push(selected.name)
+ },
+ replay: async () => {
+ retryCount += 1
+ if (retryCount === 10) activeRetries.resolve()
+ await retryRelease.promise
+ },
+ })
+
+ for (let index = 0; index < 10; index += 1) {
+ const selected = server(`retry-${index}`, 1, { bindings: false })
+ instance.scheduleRetry('ADDED', selected)
+ }
+ await vi.advanceTimersByTimeAsync(5000)
+ await activeRetries.promise
+
+ const resync = instance.runResync()
+ const startup = instance.prepareStartupGates([startupServer])
+ await Promise.all([resync, startup.waitFor(startupServer)])
+
+ expect(mutations).toEqual(expect.arrayContaining(['resync-server', 'startup-server']))
+ retryRelease.resolve()
+ instance.stop()
+ })
+
+ it('queues at most one latest-wins retry job per key behind a saturated limiter', async () => {
+ vi.useFakeTimers()
+ const blockersRelease = deferred()
+ const tenBlockersStarted = deferred()
+ const finalReplayStarted = deferred()
+ const initial = server('churning-server', 1)
+ const second = server('churning-server', 2)
+ const final = server('churning-server', 3)
+ const state = new Map<string, McpServerCRD>([[initial.name, initial]])
+ const targetEnqueues: McpServerCRD[] = []
+ const targetReplays: McpServerCRD[] = []
+ let blockerCount = 0
+ const { instance } = coordinator({
+ getCurrentServer: name => state.get(name),
+ enqueue: async (selected, work) => {
+ if (selected.name === initial.name) targetEnqueues.push(selected)
+ await work()
+ },
+ replay: async (_type, selected, retry) => {
+ if (selected.name.startsWith('limiter-blocker-')) {
+ blockerCount += 1
+ if (blockerCount === 10) tenBlockersStarted.resolve()
+ await blockersRelease.promise
+ return
+ }
+ targetReplays.push(selected)
+ retry.complete()
+ finalReplayStarted.resolve()
+ },
+ })
+
+ for (let index = 0; index < 10; index += 1) {
+ instance.scheduleRetry('DELETED', server(`limiter-blocker-${index}`))
+ }
+ await vi.advanceTimersByTimeAsync(5000)
+ await tenBlockersStarted.promise
+
+ instance.scheduleRetry('ADDED', initial)
+ await vi.advanceTimersByTimeAsync(5000)
+ state.set(second.name, second)
+ instance.scheduleRetry('MODIFIED', second)
+ await vi.advanceTimersByTimeAsync(5000)
+ state.set(final.name, final)
+ instance.scheduleRetry('MODIFIED', final)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(targetEnqueues).toHaveLength(0)
+ blockersRelease.resolve()
+ await finalReplayStarted.promise
+ await flushMicrotasks()
+ await flushMicrotasks()
+
+ expect(targetEnqueues).toHaveLength(1)
+ expect(targetReplays).toEqual([expect.objectContaining({ generation: 3 })])
+ instance.stop()
+ })
+
+ it('re-arms a retry scheduled by an active replay only after that key leaves its slot', async () => {
+ vi.useFakeTimers()
+ const selected = server('slow-retry')
+ const firstReplayStarted = deferred()
+ const releaseFirstReplay = deferred()
+ let instance!: ExternalEgressConvergenceCoordinator
+ let replayCount = 0
+ instance = coordinator({
+ getCurrentServer: () => selected,
+ replay: async (type, current, retry) => {
+ replayCount += 1
+ if (replayCount === 1) {
+ instance.scheduleRetry(type, current)
+ firstReplayStarted.resolve()
+ await releaseFirstReplay.promise
+ return
+ }
+ retry.complete()
+ },
+ }).instance
+
+ instance.scheduleRetry('ADDED', selected)
+ await vi.advanceTimersByTimeAsync(5000)
+ await firstReplayStarted.promise
+
+ await vi.advanceTimersByTimeAsync(15000)
+ expect(replayCount).toBe(1)
+
+ releaseFirstReplay.resolve()
+ await flushMicrotasks()
+ await vi.advanceTimersByTimeAsync(14999)
+ expect(replayCount).toBe(1)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(replayCount).toBe(2)
+ instance.stop()
+ })
+
+ it('cancels a retry waiter on stop before limiter capacity becomes available', async () => {
+ vi.useFakeTimers()
+ const blockersRelease = deferred()
+ const tenBlockersStarted = deferred()
+ const queued = server('queued-before-stop')
+ let blockerCount = 0
+ const enqueuedTargets: string[] = []
+ const { instance } = coordinator({
+ enqueue: async (selected, work) => {
+ if (selected.name === queued.name) enqueuedTargets.push(selected.name)
+ await work()
+ },
+ replay: async (_type, selected) => {
+ if (!selected.name.startsWith('stop-blocker-')) return
+ blockerCount += 1
+ if (blockerCount === 10) tenBlockersStarted.resolve()
+ await blockersRelease.promise
+ },
+ })
+
+ for (let index = 0; index < 10; index += 1) {
+ instance.scheduleRetry('DELETED', server(`stop-blocker-${index}`))
+ }
+ await vi.advanceTimersByTimeAsync(5000)
+ await tenBlockersStarted.promise
+ instance.scheduleRetry('DELETED', queued)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ instance.stop()
+ blockersRelease.resolve()
+ await flushMicrotasks()
+ await flushMicrotasks()
+
+ expect(enqueuedTargets).toEqual([])
+ })
+
+ it('resolves the current canonical server after periodic jitter', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0.5)
+ const stale = server('web-search', 1)
+ const current = server('web-search', 2)
+ const state = new Map<string, McpServerCRD>([[stale.name, stale]])
+ const reconciled: McpServerCRD[] = []
+ const { instance } = coordinator({
+ listServers: () => [stale],
+ getCurrentServer: name => state.get(name),
+ mutate: async (_type, selected) => {
+ reconciled.push(selected)
+ },
+ })
+
+ const resync = instance.runResync()
+ await flushMicrotasks()
+ state.set(current.name, current)
+ await vi.advanceTimersByTimeAsync(2500)
+ await resync
+
+ expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])
+ instance.stop()
+ })
+
+ it('returns one global single-flight promise for overlapping resync requests', async () => {
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const selected = server('web-search')
+ const release = deferred()
+ const mutationStarted = deferred()
+ const { instance, servers } = coordinator({
+ mutate: async () => {
+ mutationStarted.resolve()
+ await release.promise
+ },
+ })
+ servers.set(selected.name, selected)
+
+ const first = instance.runResync()
+ const overlapping = instance.runResync()
+ expect(overlapping).toBe(first)
+ await mutationStarted.promise
+
+ release.resolve()
+ await first
+ instance.stop()
+ })
+
+ it('skips a resync key whose external-egress mutation is already in flight', async () => {
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const selected = server('web-search')
+ const release = deferred()
+ const mutationStarted = deferred()
+ const mutate = vi.fn(async () => {
+ mutationStarted.resolve()
+ await release.promise
+ })
+ const { instance, servers } = coordinator({ mutate })
+ servers.set(selected.name, selected)
+
+ const active = instance.reconcile('MODIFIED', selected)
+ await mutationStarted.promise
+ await instance.runResync()
+
+ expect(mutate).toHaveBeenCalledOnce()
+ release.resolve()
+ await active
+ instance.stop()
+ })
+
+ it('resolves canonical state after entering the keyed resync queue', async () => {
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const stale = server('web-search', 1)
+ const current = server('web-search', 2)
+ const state = new Map<string, McpServerCRD>([[stale.name, stale]])
+ const queueEntered = deferred()
+ const releaseQueue = deferred()
+ const reconciled: McpServerCRD[] = []
+ const { instance } = coordinator({
+ listServers: () => [stale],
+ getCurrentServer: name => state.get(name),
+ enqueue: async (_selected, work) => {
+ queueEntered.resolve()
+ await releaseQueue.promise
+ await work()
+ },
+ mutate: async (_type, selected) => {
+ reconciled.push(selected)
+ },
+ })
+
+ const resync = instance.runResync()
+ await queueEntered.promise
+ state.set(current.name, current)
+ releaseQueue.resolve()
+ await resync
+
+ expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])
+ instance.stop()
+ })
+
+ it('keeps a startup gate false when desired revision changes during egress', async () => {
+ const selected = server('web-search', 1)
+ const current = server('web-search', 2)
+ const state = new Map<string, McpServerCRD>([[selected.name, selected]])
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ let selectedLease: (() => boolean) | undefined
+ const { instance } = coordinator({
+ getCurrentServer: name => state.get(name),
+ mutate: async (_type, _server, options) => {
+ selectedLease = options.isCurrent
+ mutationStarted.resolve()
+ await releaseMutation.promise
+ },
+ })
+
+ const gates = instance.prepareStartupGates([selected])
+ await mutationStarted.promise
+ expect(selectedLease?.()).toBe(true)
+ state.set(current.name, current)
+ expect(selectedLease?.()).toBe(false)
+ releaseMutation.resolve()
+
+ await expect(gates.waitFor(selected)).resolves.toBe(false)
+ instance.stop()
+ })
+
+ it('keeps a failed startup DNS gate isolated and recovers it through the dedicated retry', async () => {
+ vi.useFakeTimers()
+ const selected = server('dns-held-server')
+ const dnsFailure = new Error('dns resolution failed')
+ const mutate = vi.fn().mockRejectedValueOnce(dnsFailure).mockResolvedValue(undefined)
+ const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ let instance!: ExternalEgressConvergenceCoordinator
+ const built = coordinator({
+ mutate,
+ replay: async (type, current, retry) => {
+ const completion = await instance.reconcile(type, current, { retry })
+ completion?.complete()
+ },
+ })
+ instance = built.instance
+ built.servers.set(selected.name, selected)
+
+ const gates = instance.prepareStartupGates([selected])
+ await expect(gates.waitFor(selected)).resolves.toBe(false)
+ expect(mutate).toHaveBeenCalledOnce()
+ expect(error).toHaveBeenCalledWith(
+ `[K8s] Initial external egress reconciliation failed for ${selected.namespace}/${selected.name}; ` +
+ 'runtime reconciliation will stay blocked until retry succeeds:',
+ dnsFailure
+ )
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(mutate).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(30000)
+ expect(mutate).toHaveBeenCalledTimes(2)
+ instance.stop()
+ })
+
+ it('retires a direct mutation lease when desired revision changes in place', async () => {
+ const selected = server('web-search', 1)
+ const replacement = server('web-search', 2)
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ let selectedLease: (() => boolean) | undefined
+ const { instance, servers } = coordinator({
+ mutate: async (_type, _server, options) => {
+ selectedLease = options.isCurrent
+ mutationStarted.resolve()
+ await releaseMutation.promise
+ },
+ })
+ servers.set(selected.name, selected)
+
+ const mutation = instance.reconcile('MODIFIED', selected)
+ await mutationStarted.promise
+ expect(selectedLease?.()).toBe(true)
+
+ servers.set(replacement.name, replacement)
+ expect(selectedLease?.()).toBe(false)
+ releaseMutation.resolve()
+ await mutation
+ instance.stop()
+ })
+
+ it('does not start a second mutation after stop releases an in-flight wait', async () => {
+ const firstMutation = deferred()
+ const selected = server('web-search')
+ const mutate = vi.fn(async () => firstMutation.promise)
+ const { instance, servers } = coordinator({ mutate })
+ servers.set(selected.name, selected)
+
+ const first = instance.reconcile('MODIFIED', selected)
+ await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())
+ const waiting = instance.reconcile('MODIFIED', selected)
+ instance.stop()
+ firstMutation.resolve()
+ await Promise.all([first, waiting])
+
+ expect(mutate).toHaveBeenCalledOnce()
+ })
+
+ it('rechecks per-key in-flight ownership when multiple waiters resume together', async () => {
+ const selected = server('web-search')
+ const releases = [deferred(), deferred(), deferred()]
+ const starts = [deferred(), deferred(), deferred()]
+ let active = 0
+ let maxActive = 0
+ let call = 0
+ const mutate = vi.fn(async () => {
+ const index = call++
+ active += 1
+ maxActive = Math.max(maxActive, active)
+ starts[index].resolve()
+ await releases[index].promise
+ active -= 1
+ })
+ const { instance, servers } = coordinator({ mutate })
+ servers.set(selected.name, selected)
+
+ const first = instance.reconcile('MODIFIED', selected)
+ await starts[0].promise
+ const second = instance.reconcile('MODIFIED', selected)
+ const third = instance.reconcile('MODIFIED', selected)
+
+ try {
+ releases[0].resolve()
+ await starts[1].promise
+ await flushMicrotasks()
+ expect(maxActive).toBe(1)
+ } finally {
+ for (const release of releases) release.resolve()
+ await Promise.all([first, second, third])
+ instance.stop()
+ }
+ })
+
+ it('rechecks a caller lease after waiting for an in-flight mutation', async () => {
+ const firstMutation = deferred()
+ const selected = server('web-search')
+ let current = true
+ const mutate = vi.fn(async () => firstMutation.promise)
+ const { instance, servers } = coordinator({ mutate })
+ servers.set(selected.name, selected)
+
+ const first = instance.reconcile('MODIFIED', selected)
+ await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())
+ const waiting = instance.reconcile('MODIFIED', selected, {
+ isCurrent: () => current,
+ })
+ current = false
+ firstMutation.resolve()
+ await Promise.all([first, waiting])
+
+ expect(mutate).toHaveBeenCalledOnce()
+ instance.stop()
+ })
+
+ it('keeps a retry pending while inventory authority is unavailable', async () => {
+ vi.useFakeTimers()
+ const selected = server('web-search')
+ let authoritative = false
+ const { instance, replay, servers } = coordinator({
+ inventoryAuthoritative: () => authoritative,
+ })
+ servers.set(selected.name, selected)
+
+ instance.scheduleRetry('ADDED', selected)
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(replay).not.toHaveBeenCalled()
+
+ authoritative = true
+ await vi.advanceTimersByTimeAsync(14_999)
+ expect(replay).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(replay).toHaveBeenCalledOnce()
+ instance.stop()
+ })
+
+ it('skips periodic resync while inventory authority is unavailable', async () => {
+ const selected = server('web-search')
+ let authoritative = false
+ const { instance, mutate, servers } = coordinator({
+ inventoryAuthoritative: () => authoritative,
+ })
+ servers.set(selected.name, selected)
+
+ await instance.runResync()
+ expect(mutate).not.toHaveBeenCalled()
+
+ authoritative = true
+ await instance.runResync()
+ expect(mutate).toHaveBeenCalledOnce()
+ instance.stop()
+ })
+
+ it('does not resume periodic work from jitter after stop', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0.5)
+ const selected = server('web-search')
+ const { instance, mutate, servers } = coordinator()
+ servers.set(selected.name, selected)
+
+ const resync = instance.runResync()
+ await flushMicrotasks()
+ instance.stop()
+ await vi.advanceTimersByTimeAsync(2500)
+ await resync
+
+ expect(mutate).not.toHaveBeenCalled()
+ })
+
+ it('does not start an eleventh startup mutation after stop releases the limiter', async () => {
+ const release = deferred()
+ const tenStarted = deferred()
+ let active = 0
+ const selected = Array.from({ length: 11 }, (_, index) => server(`startup-${index}`))
+ const state = new Map(selected.map(item => [item.name, item]))
+ const mutate = vi.fn(async () => {
+ active += 1
+ if (active === 10) tenStarted.resolve()
+ await release.promise
+ })
+ const { instance } = coordinator({
+ getCurrentServer: name => state.get(name),
+ mutate,
+ })
+
+ const gates = instance.prepareStartupGates(selected)
+ await tenStarted.promise
+ instance.stop()
+ release.resolve()
+ await Promise.all(selected.map(item => gates.waitFor(item)))
+
+ expect(mutate).toHaveBeenCalledTimes(10)
+ })
+
+ it('passes DELETE authority through to the watcher mutation without evaluating it', async () => {
+ const selected = server('web-search')
+ const deleteAllowed = vi.fn(async () => false)
+ const mutate = vi.fn(async () => undefined)
+ const { instance } = coordinator({ mutate })
+
+ await instance.reconcile('DELETED', selected, { deleteAllowed })
+
+ expect(mutate).toHaveBeenCalledWith('DELETED', selected, {
+ deleteAllowed,
+ isCurrent: expect.any(Function),
+ })
+ expect(deleteAllowed).not.toHaveBeenCalled()
+ instance.stop()
+ })
+
+ it('owns periodic resync scheduling and stops it immediately', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const selected = server('web-search')
+ const { instance, mutate, servers } = coordinator()
+ servers.set(selected.name, selected)
+
+ instance.startPeriodicResync(1, 1)
+ await vi.advanceTimersByTimeAsync(1000)
+ expect(mutate).toHaveBeenCalledOnce()
+
+ instance.stop()
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(mutate).toHaveBeenCalledOnce()
+ })
+
+ it('warns once when periodic resync is disabled (intervalSec <= 0) and arms no timer', async () => {
+ vi.useFakeTimers()
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const selected = server('web-search')
+ const { instance, mutate, servers } = coordinator()
+ servers.set(selected.name, selected)
+
+ instance.startPeriodicResync(0, 5)
+ // No timer armed: advancing time never triggers a resync pass.
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(mutate).not.toHaveBeenCalled()
+
+ expect(warnSpy).toHaveBeenCalledTimes(1)
+ expect(warnSpy).toHaveBeenCalledWith(
+ expect.stringContaining('External egress periodic DNS resync disabled')
+ )
+
+ // Called again while still disabled: no duplicate warning (once only).
+ instance.startPeriodicResync(-1, 5)
+ expect(warnSpy).toHaveBeenCalledTimes(1)
+
+ instance.stop()
+ })
+
+ it('does not warn about a disabled resync when it is enabled (intervalSec > 0)', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const selected = server('web-search')
+ const { instance, servers } = coordinator()
+ servers.set(selected.name, selected)
+
+ instance.startPeriodicResync(1, 1)
+ expect(warnSpy).not.toHaveBeenCalledWith(
+ expect.stringContaining('External egress periodic DNS resync disabled')
+ )
+
+ instance.stop()
+ })
+
+ it('arms one resync at a time, rescheduling only after each pass completes (H2)', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const selected = server('web-search')
+ const release: Array<() => void> = []
+ const mutate = vi.fn(
+ () =>
+ new Promise<void>(resolve => {
+ release.push(resolve)
+ })
+ )
+ const { instance, servers } = coordinator({ mutate })
+ servers.set(selected.name, selected)
+
+ instance.startPeriodicResync(10, 5)
+ // First pass fires at the configured interval (10s) and then blocks.
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(mutate).toHaveBeenCalledOnce()
+
+ // Complete the pass off the fixed grid (at t=17s). The successor is armed
+ // relative to completion (t=27s), NOT on a fixed 20s/30s interval grid.
+ await vi.advanceTimersByTimeAsync(7_000)
+ release[0]()
+ await flushMicrotasks()
+
+ // A fixed setInterval would fire a second pass at t=20s; the self-
+ // rescheduling timer must not, because it only re-armed at completion.
+ await vi.advanceTimersByTimeAsync(3_000)
+ expect(mutate).toHaveBeenCalledOnce()
+
+ // The successor fires 10s after completion (t=27s).
+ await vi.advanceTimersByTimeAsync(7_000)
+ expect(mutate).toHaveBeenCalledTimes(2)
+
+ instance.stop()
+ release.forEach(resolve => resolve())
+ })
+
+ it('advances the next periodic resync to <= observed TTL/2 (H2, issue #299)', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const selected = server('web-search')
+ // The mutation observes a low DNS TTL, ratcheting the coordinator's source
+ // down; scheduleNext reads it AFTER the pass and must advance to <= TTL/2.
+ let minObservedTtlMs = Infinity
+ const mutate = vi.fn(async () => {
+ minObservedTtlMs = 20_000
+ })
+ const { instance, servers } = coordinator({
+ mutate,
+ externalEgressRefreshMinTtlMs: () => minObservedTtlMs,
+ })
+ servers.set(selected.name, selected)
+
+ // Interval 60s, floor 5s. No TTL observed yet, so the first pass fires at 60s.
+ instance.startPeriodicResync(60, 5)
+ await vi.advanceTimersByTimeAsync(59_999)
+ expect(mutate).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(mutate).toHaveBeenCalledOnce()
+
+ // TTL/2 = 10s now governs the next delay, far below the 60s backstop.
+ await vi.advanceTimersByTimeAsync(9_999)
+ expect(mutate).toHaveBeenCalledOnce()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(mutate).toHaveBeenCalledTimes(2)
+
+ instance.stop()
+ })
+
+ it('logs and absorbs a rejected periodic resync pass, then re-arms (no unhandledRejection)', async () => {
+ vi.useFakeTimers()
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ // Capture any process-level unhandledRejection: the pre-fix code left the
+ // rejected `void runResync()` promise unhandled, which crashes modern Node.
+ const unhandled: unknown[] = []
+ const onUnhandled = (reason: unknown): void => {
+ unhandled.push(reason)
+ }
+ process.on('unhandledRejection', onUnhandled)
+
+ const selected = server('web-search')
+ // Reject OUTSIDE the per-server try/catch in runResyncCore (that catch only
+ // wraps the mutation). A rejecting `enqueue` propagates through the worker to
+ // Promise.all -> runResyncCore -> runResync, so the whole pass rejects.
+ const enqueue = vi.fn(async () => {
+ throw new Error('resync enqueue boom')
+ })
+ const { instance, servers } = coordinator({ enqueue })
+ servers.set(selected.name, selected)
+
+ try {
+ instance.startPeriodicResync(10, 5)
+ // First pass fires at 10s and rejects.
+ await vi.advanceTimersByTimeAsync(10_000)
+ await flushMicrotasks()
+
+ // The rejection is logged by the fix's .catch (RED without it: no such log).
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[K8s] External egress periodic resync pass failed:',
+ expect.any(Error)
+ )
+ expect(enqueue).toHaveBeenCalledTimes(1)
+
+ // The loop RE-ARMS despite the failure: the successor fires 10s later.
+ await vi.advanceTimersByTimeAsync(10_000)
+ await flushMicrotasks()
+ expect(enqueue).toHaveBeenCalledTimes(2)
+
+ // The fix handles the rejection, so nothing escaped as unhandled.
+ expect(unhandled).toEqual([])
+ } finally {
+ process.off('unhandledRejection', onUnhandled)
+ instance.stop()
+ }
+ })
+})
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.ts
new file mode 100644
index 000000000..6a8a19db2
--- /dev/null
+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.ts
@@ -0,0 +1,607 @@
+import { externalEgressRetriesAtCap } from './metrics'
+import type { McpServerCRD } from './types'
+
+const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const
+const EXTERNAL_EGRESS_MAX_CONCURRENCY = 10
+const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000
+
+export type ExternalEgressWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
+
+export type ExternalEgressRetryHandle = {
+ isCurrent: () => boolean
+ complete: () => void
+}
+
+type ExternalEgressRetryIntent = {
+ sequence: number
+ type: ExternalEgressWatchEventType
+ server: McpServerCRD
+}
+
+type ExternalEgressMutationOptions = {
+ deleteAllowed?: () => Promise<boolean>
+ isCurrent: () => boolean
+}
+
+type ExternalEgressReconcileOptions = {
+ deleteAllowed?: () => Promise<boolean>
+ retry?: ExternalEgressRetryHandle
+ isCurrent?: () => boolean
+}
+
+type ExternalEgressCoordinatorDependencies = {
+ listServers: () => McpServerCRD[]
+ getCurrentServer: (name: string) => McpServerCRD | undefined
+ inventoryAuthoritative: () => boolean
+ sameDesiredRevision: (left: McpServerCRD, right: McpServerCRD) => boolean
+ enqueue: (server: McpServerCRD, work: () => Promise<void>) => Promise<void>
+ mutate: (
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ options: ExternalEgressMutationOptions
+ ) => Promise<void>
+ replay: (
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ retry: ExternalEgressRetryHandle
+ ) => Promise<void>
+ // Smallest DNS TTL (ms) observed across external-egress resolutions, or
+ // Infinity if none. startPeriodicResync reads it to advance the next resync to
+ // <= TTL/2 (H2, issue #299).
+ externalEgressRefreshMinTtlMs: () => number
+}
+
+export type ExternalEgressStartupGates = {
+ waitFor: (server: Pick<McpServerCRD, 'name' | 'namespace'>) => Promise<boolean>
+}
+
+class AsyncConcurrencyLimiter {
+ private active = 0
+ private readonly waiters: Array<(admitted: boolean) => void> = []
+ private closed = false
+
+ constructor(private readonly limit: number) {}
+
+ async run<T>(work: () => Promise<T>): Promise<T | undefined> {
+ if (this.closed) return undefined
+ if (this.active >= this.limit) {
+ const admitted = await new Promise<boolean>(resolve => this.waiters.push(resolve))
+ if (!admitted) return undefined
+ } else {
+ this.active += 1
+ }
+ try {
+ return await work()
+ } finally {
+ this.release()
+ }
+ }
+
+ cancelPending(): void {
+ if (this.closed) return
+ this.closed = true
+ for (const resume of this.waiters.splice(0)) {
+ resume(false)
+ }
+ }
+
+ private release(): void {
+ this.active -= 1
+ if (this.closed) return
+ const resume = this.waiters.shift()
+ if (!resume) return
+ // Transfer the released permit before waking the waiter. Reserving it here
+ // prevents a new caller from entering between wake-up and its microtask.
+ this.active += 1
+ resume(true)
+ }
+}
+
+function delay(ms: number): Promise<void> {
+ return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+/**
+ * External-egress resync delay (ms), H2 (issue #299). The configured interval is
+ * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a
+ * rotating low-TTL host is sampled every rotation. The floor is the hard lower
+ * bound to avoid a hot-loop. Pure and unit-testable; consumed by
+ * {@link ExternalEgressConvergenceCoordinator.startPeriodicResync}.
+ */
+export function externalEgressResyncDelayMs(
+ intervalSec: number,
+ floorSec: number,
+ minObservedTtlMs = Infinity
+): number {
+ const floorMs = floorSec * 1000
+ const configuredMs = intervalSec * 1000
+ const ttlAwareMs = Number.isFinite(minObservedTtlMs)
+ ? Math.min(configuredMs, minObservedTtlMs / 2)
+ : configuredMs
+ return Math.max(ttlAwareMs, floorMs)
+}
+
+/**
+ * Owns temporal coordination for external-egress convergence. Kubernetes
+ * authority and mutations remain behind callbacks supplied by the watcher.
+ */
+export class ExternalEgressConvergenceCoordinator {
+ private readonly retryTimers = new Map<string, ReturnType<typeof setTimeout>>()
+ private readonly retryAttempts = new Map<string, number>()
+ private readonly retryIntents = new Map<string, ExternalEgressRetryIntent>()
+ private retrySequence = 0
+ private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
+ private readonly retrySlots = new Map<string, object>()
+ // Keys whose retry backoff is currently pinned at the capped (maximum) delay.
+ // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a
+ // binding stuck retrying forever is observable, not silently denied.
+ private readonly retriesAtCap = new Set<string>()
+ private readonly inFlight = new Map<string, Promise<void>>()
+ private resyncInFlight: Promise<void> | null = null
+ private resyncTimer: ReturnType<typeof setTimeout> | null = null
+ private resyncDisabledWarned = false
+ private stopped = false
+
+ constructor(private readonly dependencies: ExternalEgressCoordinatorDependencies) {}
+
+ startPeriodicResync(intervalSec: number, floorSec: number): void {
+ if (this.stopped || this.resyncTimer) return
+ if (intervalSec <= 0) {
+ // Disabled. Warn once (mirrors the Host/SFS/GFS resync lanes) so the
+ // operational consequence is visible: without periodic resync the
+ // accumulated external-egress DNS allows are never refreshed, so a
+ // binding whose upstream DNS rotates (issue #299) can drift until an
+ // unrelated McpServer event triggers reconciliation.
+ if (!this.resyncDisabledWarned) {
+ this.resyncDisabledWarned = true
+ console.warn(
+ '[K8s] External egress periodic DNS resync disabled; accumulated DNS allows will not ' +
+ 'be refreshed, so a binding whose upstream DNS rotates (issue #299) can drift until ' +
+ 'another McpServer event triggers reconciliation.'
+ )
+ }
+ return
+ }
+ // Self-rescheduling (not a fixed setInterval) so the delay can advance to
+ // <= observed TTL/2 (H2, issue #299), and rescheduling AFTER each pass
+ // completes guarantees no overlapping resyncs. runResync still dedupes any
+ // in-flight pass; the floor is the hard lower bound against a hot-loop.
+ const scheduleNext = (): void => {
+ if (this.stopped) return
+ const delayMs = externalEgressResyncDelayMs(
+ intervalSec,
+ floorSec,
+ this.dependencies.externalEgressRefreshMinTtlMs()
+ )
+ this.resyncTimer = setTimeout(() => {
+ // Re-arm FIRST (.finally) so a rejected pass still schedules its
+ // successor — the "re-arms after completion" property must survive a
+ // failure. Then .catch logs and absorbs the rejection so a rejecting
+ // pass (or a throwing scheduleNext) never becomes an unhandledRejection,
+ // which crashes the process on modern Node and would silently kill the
+ // periodic resync loop (reopening the #299 DNS-drift window).
+ void this.runResync()
+ .finally(scheduleNext)
+ .catch(error =>
+ console.error('[K8s] External egress periodic resync pass failed:', error)
+ )
+ }, delayMs)
+ this.resyncTimer.unref?.()
+ }
+ scheduleNext()
+ console.log(
+ `[K8s] External egress periodic DNS resync enabled (every ${intervalSec}s, advancing to <= TTL/2)`
+ )
+ }
+
+ runResync(): Promise<void> {
+ if (this.resyncInFlight) return this.resyncInFlight
+ const run = this.runResyncCore()
+ this.resyncInFlight = run
+ const settle = () => {
+ if (this.resyncInFlight === run) this.resyncInFlight = null
+ }
+ void run.then(settle, settle)
+ return run
+ }
+
+ prepareStartupGates(
+ servers: McpServerCRD[],
+ isCurrent: () => boolean = this.dependencies.inventoryAuthoritative
+ ): ExternalEgressStartupGates {
+ const gates = new Map<string, Promise<boolean>>()
+ const selectedServers = servers
+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
+ .sort((left, right) => this.key(left).localeCompare(this.key(right)))
+
+ if (selectedServers.length > 0) {
+ console.log(
+ `[K8s] Reconciling external egress before startup runtime reconciliation for ` +
+ `${selectedServers.length} McpServer(s)`
+ )
+ }
+
+ // Startup admission is intentionally independent from retry and periodic
+ // resync capacity, so a saturated recovery lane cannot head-of-line block
+ // initial runtime gates for unrelated servers.
+ const limiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
+ for (const selected of selectedServers) {
+ const key = this.key(selected)
+ const gate = limiter
+ .run(async () => {
+ if (this.stopped || !isCurrent()) return false
+ let ready = false
+ await this.dependencies.enqueue(selected, async () => {
+ if (this.stopped || !isCurrent()) return
+ const current = this.dependencies.getCurrentServer(selected.name)
+ if (
+ !current ||
+ current.namespace !== selected.namespace ||
+ !this.dependencies.sameDesiredRevision(selected, current)
+ ) {
+ return
+ }
+ try {
+ // Startup is an egress-only gate. Any pre-existing retry intent is
+ // completed only by its full replay, never by this mutation.
+ await this.reconcile('MODIFIED', current, { isCurrent })
+ const latest = this.dependencies.getCurrentServer(selected.name)
+ ready =
+ !this.stopped &&
+ isCurrent() &&
+ latest !== undefined &&
+ latest.namespace === selected.namespace &&
+ this.dependencies.sameDesiredRevision(selected, latest)
+ } catch (error) {
+ console.error(
+ `[K8s] Initial external egress reconciliation failed for ${key}; ` +
+ 'runtime reconciliation will stay blocked until retry succeeds:',
+ error
+ )
+ this.scheduleRetry('MODIFIED', current)
+ }
+ })
+ return ready
+ })
+ .then(result => result ?? false)
+ // Side-attach a logger so a rejection is never *unhandled* (which crashes
+ // the process on modern Node) if fullReconcile aborts before waitFor
+ // awaits this gate. This does NOT swallow the failure: gates.set stores
+ // the original `gate`, so the awaited path in fullReconcile still rejects
+ // and fails the pass loudly — this only covers the unawaited case.
+ gate.catch(error => {
+ console.error(
+ `[K8s] External egress startup gate for ${key} rejected before it was awaited:`,
+ error
+ )
+ })
+ gates.set(key, gate)
+ }
+
+ return {
+ waitFor: server => gates.get(this.key(server)) ?? Promise.resolve(true),
+ }
+ }
+
+ async reconcile(
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ options: ExternalEgressReconcileOptions = {}
+ ): Promise<ExternalEgressRetryHandle | undefined> {
+ if (!this.reconcileIsCurrent(type, server, options)) return undefined
+ const key = this.key(server)
+ let existing = this.inFlight.get(key)
+ while (existing) {
+ console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)
+ await existing
+ if (!this.reconcileIsCurrent(type, server, options)) return undefined
+ existing = this.inFlight.get(key)
+ }
+
+ if (!this.reconcileIsCurrent(type, server, options)) return undefined
+ const completion = options.retry ?? this.currentRetryHandle(key)
+ const run = this.dependencies.mutate(type, server, {
+ deleteAllowed: options.deleteAllowed,
+ isCurrent: () => this.reconcileIsCurrent(type, server, options),
+ })
+ this.inFlight.set(key, run)
+ try {
+ await run
+ return completion
+ } finally {
+ if (this.inFlight.get(key) === run) {
+ this.inFlight.delete(key)
+ }
+ }
+ }
+
+ scheduleRetry(type: string, server: McpServerCRD): void {
+ if (!this.isWatchEventType(type) || this.stopped) return
+
+ const cached = this.dependencies.getCurrentServer(server.name)
+ const retryType: ExternalEgressWatchEventType =
+ cached && cached.namespace === server.namespace
+ ? type === 'ADDED' && this.dependencies.sameDesiredRevision(server, cached)
+ ? 'ADDED'
+ : 'MODIFIED'
+ : 'DELETED'
+ const retryServer = cached && cached.namespace === server.namespace ? cached : server
+ const key = this.key(server)
+ const previousIntent = this.retryIntents.get(key)
+ if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {
+ this.retryAttempts.delete(key)
+ // A new target restarts the backoff ladder, so the key is no longer capped.
+ this.clearRetryAtCap(key)
+ }
+
+ const intent: ExternalEgressRetryIntent = {
+ sequence: ++this.retrySequence,
+ type: retryType,
+ server: retryServer,
+ }
+ this.retryIntents.set(key, intent)
+ // A timer or queued/active slot already owning the key consumes the newest
+ // intent when it next enters the retry and per-server lanes.
+ if (this.retryTimers.has(key) || this.retrySlots.has(key)) return
+ this.armRetryTimer(key)
+ }
+
+ private armRetryTimer(key: string): void {
+ if (
+ this.stopped ||
+ this.retryTimers.has(key) ||
+ this.retrySlots.has(key) ||
+ !this.retryIntents.has(key)
+ ) {
+ return
+ }
+ const attempt = Math.min(
+ (this.retryAttempts.get(key) ?? 0) + 1,
+ EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
+ )
+ const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
+ this.retryAttempts.set(key, attempt)
+ const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
+ if (capped) this.markRetryAtCap(key)
+ console.warn(
+ `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` +
+ `${capped ? ' (capped)' : ''} ` +
+ `for McpServer "${this.retryIntents.get(key)!.server.name}" in ${delayMs}ms`
+ )
+
+ const timer = setTimeout(() => {
+ if (this.retryTimers.get(key) !== timer) return
+ this.retryTimers.delete(key)
+ this.queueRetry(key)
+ }, delayMs)
+ this.retryTimers.set(key, timer)
+ }
+
+ private queueRetry(key: string): void {
+ if (this.stopped || !this.retryIntents.has(key)) return
+ if (this.retrySlots.has(key)) {
+ // A timer can overlap work admitted through another lane. The durable
+ // slot owns the latest intent and rearms it on exit if it remains live.
+ return
+ }
+
+ const slot = {}
+ this.retrySlots.set(key, slot)
+ void this.retryLimiter
+ .run(async () => {
+ if (
+ this.stopped ||
+ this.retrySlots.get(key) !== slot ||
+ !this.dependencies.inventoryAuthoritative()
+ ) {
+ return
+ }
+ // Do not capture intent before the limiter. Churn while capacity is
+ // saturated is represented by one slot and resolved latest-wins here.
+ const admittedIntent = this.retryIntents.get(key)
+ if (!admittedIntent) return
+ await this.dependencies.enqueue(admittedIntent.server, async () => {
+ if (
+ this.stopped ||
+ this.retrySlots.get(key) !== slot ||
+ !this.dependencies.inventoryAuthoritative()
+ ) {
+ return
+ }
+ // The keyed queue may also wait. Resolve the intent again so only the
+ // latest desired revision or absence is replayed.
+ const currentIntent = this.retryIntents.get(key)
+ if (!currentIntent) return
+ await this.replayIntent(currentIntent)
+ })
+ })
+ .catch(error => {
+ if (!this.stopped) {
+ console.error(`[K8s] External egress retry execution failed for ${key}:`, error)
+ }
+ })
+ .finally(() => {
+ if (this.retrySlots.get(key) !== slot) return
+ this.retrySlots.delete(key)
+ // A replay may schedule its successor while still active, or a newer
+ // intent may arrive after this slot selected work. Rearm only after the
+ // key leaves the slot so one key never accumulates limiter backlog.
+ if (!this.stopped && this.retryIntents.has(key) && !this.retryTimers.has(key)) {
+ this.armRetryTimer(key)
+ }
+ })
+ }
+
+ stop(): void {
+ if (this.stopped) return
+ this.stopped = true
+ this.retryLimiter.cancelPending()
+ if (this.resyncTimer) {
+ clearTimeout(this.resyncTimer)
+ this.resyncTimer = null
+ }
+ for (const timer of this.retryTimers.values()) {
+ clearTimeout(timer)
+ }
+ this.retryTimers.clear()
+ this.retryAttempts.clear()
+ this.retryIntents.clear()
+ this.retrySlots.clear()
+ if (this.retriesAtCap.size > 0) {
+ this.retriesAtCap.clear()
+ externalEgressRetriesAtCap.set(0)
+ }
+ this.inFlight.clear()
+ }
+
+ private async runResyncCore(): Promise<void> {
+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
+ const servers = this.dependencies
+ .listServers()
+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
+ .sort((left, right) => this.key(left).localeCompare(this.key(right)))
+ if (servers.length === 0) return
+
+ console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)
+ let index = 0
+ const workers = Array.from(
+ { length: Math.min(EXTERNAL_EGRESS_MAX_CONCURRENCY, servers.length) },
+ async () => {
+ while (!this.stopped && this.dependencies.inventoryAuthoritative()) {
+ const server = servers[index++]
+ if (!server) return
+ const key = this.key(server)
+ if (this.inFlight.has(key)) {
+ console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)
+ continue
+ }
+ const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)
+ if (jitterMs > 0) await delay(jitterMs)
+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
+
+ await this.dependencies.enqueue(server, async () => {
+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
+ // The fleet snapshot selects work only. Desired state is resolved
+ // after jitter and after entering the per-server FIFO lane.
+ const current = this.dependencies.getCurrentServer(server.name)
+ if (
+ !current ||
+ current.namespace !== server.namespace ||
+ (current.spec.egressBindings?.length ?? 0) === 0
+ ) {
+ return
+ }
+ try {
+ const retryIntent = this.retryIntents.get(key)
+ if (retryIntent) {
+ await this.replayIntent(retryIntent)
+ return
+ }
+ // Periodic refresh is egress-only and therefore cannot complete
+ // a retry intent that may appear while the mutation is running.
+ await this.reconcile('MODIFIED', current, {
+ isCurrent: this.dependencies.inventoryAuthoritative,
+ })
+ } catch (error) {
+ console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)
+ this.scheduleRetry('MODIFIED', current)
+ }
+ })
+ }
+ }
+ )
+ await Promise.all(workers)
+ }
+
+ private async replayIntent(intent: ExternalEgressRetryIntent): Promise<void> {
+ if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
+ const handle = this.retryHandle(intent)
+ if (!handle.isCurrent()) return
+
+ const current = this.dependencies.getCurrentServer(intent.server.name)
+ const type: ExternalEgressWatchEventType =
+ current && current.namespace === intent.server.namespace ? 'MODIFIED' : 'DELETED'
+ const server = type === 'DELETED' ? intent.server : current!
+ await this.dependencies.replay(type, server, handle)
+ }
+
+ private reconcileIsCurrent(
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ options: ExternalEgressReconcileOptions
+ ): boolean {
+ const callerIsCurrent =
+ !this.stopped &&
+ (!options.retry || options.retry.isCurrent()) &&
+ (!options.isCurrent || options.isCurrent())
+ if (!callerIsCurrent || type === 'DELETED') return callerIsCurrent
+
+ const current = this.dependencies.getCurrentServer(server.name)
+ return (
+ current !== undefined &&
+ current.namespace === server.namespace &&
+ this.dependencies.sameDesiredRevision(server, current)
+ )
+ }
+
+ private currentRetryHandle(key: string): ExternalEgressRetryHandle | undefined {
+ const intent = this.retryIntents.get(key)
+ return intent ? this.retryHandle(intent) : undefined
+ }
+
+ private retryHandle(intent: ExternalEgressRetryIntent): ExternalEgressRetryHandle {
+ return {
+ isCurrent: () =>
+ !this.stopped &&
+ this.retryIntents.get(this.key(intent.server))?.sequence === intent.sequence,
+ complete: () => {
+ if (this.retryIntents.get(this.key(intent.server))?.sequence !== intent.sequence) return
+ this.clearRetry(intent.server)
+ },
+ }
+ }
+
+ private clearRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {
+ const key = this.key(server)
+ const timer = this.retryTimers.get(key)
+ if (timer) {
+ clearTimeout(timer)
+ this.retryTimers.delete(key)
+ }
+ this.retryAttempts.delete(key)
+ this.retryIntents.delete(key)
+ this.clearRetryAtCap(key)
+ }
+
+ private markRetryAtCap(key: string): void {
+ if (this.retriesAtCap.has(key)) return
+ this.retriesAtCap.add(key)
+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)
+ }
+
+ private clearRetryAtCap(key: string): void {
+ if (!this.retriesAtCap.delete(key)) return
+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)
+ }
+
+ private sameRetryTarget(
+ previous: ExternalEgressRetryIntent,
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD
+ ): boolean {
+ const previousPresent = previous.type !== 'DELETED'
+ const nextPresent = type !== 'DELETED'
+ return (
+ previousPresent === nextPresent &&
+ this.dependencies.sameDesiredRevision(previous.server, server)
+ )
+ }
+
+ private key(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {
+ return `${server.namespace}/${server.name}`
+ }
+
+ private isWatchEventType(type: string): type is ExternalEgressWatchEventType {
+ return type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED'
+ }
+}
diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.ts
index c9cde6bcb..d2fd7965b 100644
--- a/host-context-controller/src/gfsReconciler.test.ts
+++ b/host-context-controller/src/gfsReconciler.test.ts
@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import { GfsK8sApi, GfsReconciler, GfsSeedClient } from './gfsReconciler'
import { GfsFactoryConfig } from './k8s/gfsFactory'
-import type { GlobalFileSystemCRD } from './types'
+import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'
const config: GfsFactoryConfig = {
gfsNamespace: 'gfs',
@@ -42,11 +42,12 @@ class FakeApi implements GfsK8sApi {
netpols: string[] = []
pvcs: string[] = []
deleted: string[] = []
- statuses: { phase?: string; serviceUrl?: string }[] = []
+ statuses: GlobalFileSystemStatus[] = []
writerNeedsUpdate = true
writerAvailable = true
writerAvailabilitySequence: boolean[] = []
failPvc = false
+ statusFailuresRemaining = 0
async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<void> {
if (this.failPvc) throw new Error('simulated PVC apply failure')
@@ -101,11 +102,11 @@ class FakeApi implements GfsK8sApi {
async deletePvc(name: string): Promise<void> {
this.deleted.push(`pvc/${name}`)
}
- async patchStatus(
- _name: string,
- _ns: string,
- status: { phase?: string; serviceUrl?: string }
- ): Promise<void> {
+ async patchStatus(_name: string, _ns: string, status: GlobalFileSystemStatus): Promise<void> {
+ if (this.statusFailuresRemaining > 0) {
+ this.statusFailuresRemaining--
+ throw new Error('simulated status patch failure')
+ }
this.statuses.push(status)
}
}
@@ -218,6 +219,59 @@ describe('GfsReconciler.reconcile', () => {
expect(seed.calls).toBe(1)
})
+ it('does not rewrite unchanged status but still publishes a later phase transition', async () => {
+ const api = new FakeApi()
+ api.writerNeedsUpdate = false
+ const reconciler = new GfsReconciler(api, config, undefined, 0, 0)
+
+ await reconciler.reconcile(gfs)
+ const observedReady = { ...gfs, status: api.statuses[0] }
+ await reconciler.reconcile(observedReady)
+
+ expect(api.statuses).toHaveLength(1)
+ expect(api.statuses[0]?.phase).toBe('Ready')
+
+ api.writerAvailable = false
+ await reconciler.reconcile(observedReady)
+
+ expect(api.statuses.map(status => status.phase)).toEqual(['Ready', 'Initializing'])
+ })
+
+ it('publishes status for a same-name recreation even when the DELETE watch was missed', async () => {
+ const api = new FakeApi()
+ const reconciler = new GfsReconciler(api, config)
+
+ await reconciler.reconcile(gfs)
+ await reconciler.reconcile({ ...gfs, status: api.statuses[0] })
+ await reconciler.reconcile({ ...gfs, status: undefined })
+
+ expect(api.statuses).toHaveLength(2)
+ })
+
+ it('repairs status that was removed externally after HCC observed its prior write', async () => {
+ const api = new FakeApi()
+ const reconciler = new GfsReconciler(api, config)
+
+ await reconciler.reconcile(gfs)
+ await reconciler.reconcile({ ...gfs, status: api.statuses[0] })
+ await reconciler.reconcile({ ...gfs, status: undefined })
+
+ expect(api.statuses).toHaveLength(2)
+ expect(api.statuses.at(-1)?.phase).toBe('Ready')
+ })
+
+ it('retries an unchanged status when the previous patch failed', async () => {
+ const api = new FakeApi()
+ api.statusFailuresRemaining = 1
+ const reconciler = new GfsReconciler(api, config)
+
+ await expect(reconciler.reconcile(gfs)).rejects.toThrow('simulated status patch failure')
+ await expect(reconciler.reconcile(gfs)).resolves.toBeUndefined()
+
+ expect(api.statuses).toHaveLength(1)
+ expect(api.statuses[0]?.phase).toBe('Ready')
+ })
+
it('reports Initializing and does NOT seed when the writer is not yet available', async () => {
const api = new FakeApi()
api.writerAvailable = false
@@ -265,4 +319,15 @@ describe('GfsReconciler.reconcileDelete', () => {
expect(api.deleted).toContain('pvc/gfs-drive')
expect(api.deleted.at(-1)).toBe('pvc/gfs-drive') // bytes removed LAST
})
+
+ it('publishes status again after delete clears the prior write suppression', async () => {
+ const api = new FakeApi()
+ const reconciler = new GfsReconciler(api, config)
+
+ await reconciler.reconcile(gfs)
+ await reconciler.reconcileDelete(gfs)
+ await reconciler.reconcile(gfs)
+
+ expect(api.statuses).toHaveLength(2)
+ })
})
diff --git a/host-context-controller/src/gfsReconciler.ts b/host-context-controller/src/gfsReconciler.ts
index de5e29ea2..0462a12e6 100644
--- a/host-context-controller/src/gfsReconciler.ts
+++ b/host-context-controller/src/gfsReconciler.ts
@@ -95,6 +95,27 @@ export class GfsReconciler {
} while (true)
}
+ private async writeStatus(
+ gfs: GlobalFileSystemCRD,
+ status: GlobalFileSystemStatus
+ ): Promise<void> {
+ // Compare against status observed on this exact LIST/watch object, not
+ // process memory keyed only by name. A local name cache could suppress the
+ // repair of a recreated object when its DELETE event was missed, or of
+ // status that another actor removed. Conversely, an identical observed
+ // status proves this is our own status-only MODIFIED event and closes the
+ // patch -> watch -> reconcile loop without another API write.
+ if (
+ gfs.status?.phase === status.phase &&
+ gfs.status?.pvcName === status.pvcName &&
+ gfs.status?.serviceName === status.serviceName &&
+ gfs.status?.serviceUrl === status.serviceUrl
+ ) {
+ return
+ }
+ await this.api.patchStatus(gfs.name, gfs.namespace, status)
+ }
+
async reconcile(gfs: GlobalFileSystemCRD): Promise<void> {
const ns = this.config.gfsNamespace
const writerDeployment = buildDeployment(gfs, this.config, 'writer')
@@ -135,7 +156,7 @@ export class GfsReconciler {
}
const phase = writerReady ? 'Ready' : 'Initializing'
- await this.api.patchStatus(gfs.name, gfs.namespace, {
+ await this.writeStatus(gfs, {
phase,
pvcName: pvcName(gfs),
serviceName: serviceName(gfs),
diff --git a/host-context-controller/src/hostFleetScheduler.test.ts b/host-context-controller/src/hostFleetScheduler.test.ts
new file mode 100644
index 000000000..0f6817cfa
--- /dev/null
+++ b/host-context-controller/src/hostFleetScheduler.test.ts
@@ -0,0 +1,284 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+ type HostFleetReconcileRequest,
+ type HostFleetRequestResult,
+ HostFleetScheduler,
+} from './hostFleetScheduler'
+
+function deferred<T = void>() {
+ let resolve!: (value: T | PromiseLike<T>) => void
+ let reject!: (reason?: unknown) => void
+ const promise = new Promise<T>((resolvePromise, rejectPromise) => {
+ resolve = resolvePromise
+ reject = rejectPromise
+ })
+ return { promise, resolve, reject }
+}
+
+describe('HostFleetScheduler', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ it('merges newer work into one trailing pass with full coverage and latest inputs', async () => {
+ const activePass = deferred()
+ const requests: HostFleetReconcileRequest[] = []
+ const results: HostFleetRequestResult[] = []
+ const scheduler = new HostFleetScheduler({
+ perform: async request => {
+ requests.push({ ...request })
+ if (requests.length === 1) await activePass.promise
+ },
+ recordRequest: result => results.push(result),
+ })
+
+ const firstGeneration = scheduler.beginLifecycleTransition()
+ scheduler.bumpInputRevision()
+ const active = scheduler.request('active lifecycle', firstGeneration, 'lifecycle')
+ await vi.waitFor(() => expect(requests).toHaveLength(1))
+
+ scheduler.bumpInputRevision()
+ const pendingFull = scheduler.request('pending full', firstGeneration, 'full')
+ const latestGeneration = scheduler.beginLifecycleTransition()
+ scheduler.bumpInputRevision()
+ const latestLifecycle = scheduler.request('latest lifecycle', latestGeneration, 'lifecycle')
+
+ activePass.resolve()
+ await Promise.all([active, pendingFull, latestLifecycle])
+
+ expect(requests).toEqual([
+ {
+ reason: 'active lifecycle',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: firstGeneration,
+ inputRevision: 1,
+ },
+ {
+ reason: 'latest lifecycle',
+ mode: 'full',
+ ccLifecycleGeneration: latestGeneration,
+ inputRevision: 3,
+ },
+ ])
+ expect(results).toEqual(['started', 'coalesced', 'coalesced', 'trailing'])
+ })
+
+ it('carries the known lifecycle generation when merging a generation-less full request', async () => {
+ const activePass = deferred()
+ const requests: HostFleetReconcileRequest[] = []
+ const scheduler = new HostFleetScheduler({
+ perform: async request => {
+ requests.push({ ...request })
+ if (requests.length === 1) await activePass.promise
+ },
+ recordRequest: vi.fn(),
+ })
+ const generation = scheduler.beginLifecycleTransition()
+ const active = scheduler.request('active')
+ await vi.waitFor(() => expect(requests).toHaveLength(1))
+ scheduler.bumpInputRevision()
+ const pendingLifecycle = scheduler.request('lifecycle', generation, 'lifecycle')
+ scheduler.bumpInputRevision()
+ const pendingFull = scheduler.request('full')
+
+ activePass.resolve()
+ await Promise.all([active, pendingLifecycle, pendingFull])
+
+ expect(requests[1]).toMatchObject({
+ reason: 'full',
+ mode: 'full',
+ ccLifecycleGeneration: generation,
+ inputRevision: 2,
+ })
+ })
+
+ it('reuses active work only when its mode, revision, and lifecycle generation cover the request', async () => {
+ const activePass = deferred()
+ const requests: HostFleetReconcileRequest[] = []
+ const scheduler = new HostFleetScheduler({
+ perform: async request => {
+ requests.push({ ...request })
+ if (requests.length === 1) await activePass.promise
+ },
+ recordRequest: vi.fn(),
+ })
+ const generation = scheduler.beginLifecycleTransition()
+ scheduler.bumpInputRevision()
+
+ const active = scheduler.request('full', generation, 'full')
+ await vi.waitFor(() => expect(requests).toHaveLength(1))
+ const covered = scheduler.request('covered lifecycle', generation, 'lifecycle')
+
+ activePass.resolve()
+ await Promise.all([active, covered])
+
+ expect(requests).toHaveLength(1)
+ })
+
+ it('rejects stale or already-applied lifecycle work without suppressing a current full pass', async () => {
+ const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(
+ async () => undefined
+ )
+ const scheduler = new HostFleetScheduler({
+ perform,
+ recordRequest: vi.fn(),
+ })
+ const firstGeneration = scheduler.beginLifecycleTransition()
+ scheduler.markLifecycleApplied(firstGeneration)
+
+ await scheduler.request('already applied', firstGeneration, 'lifecycle')
+ await scheduler.request('current full', firstGeneration, 'full')
+ scheduler.beginLifecycleTransition()
+ await scheduler.request('stale full', firstGeneration, 'full')
+
+ expect(perform).toHaveBeenCalledOnce()
+ expect(perform).toHaveBeenCalledWith(
+ expect.objectContaining({ reason: 'current full', mode: 'full' })
+ )
+ })
+
+ it('does not let a rejected pass poison later work', async () => {
+ const failure = new Error('transient failure')
+ const perform = vi
+ .fn<(request: HostFleetReconcileRequest) => Promise<void>>()
+ .mockRejectedValueOnce(failure)
+ .mockResolvedValueOnce(undefined)
+ const scheduler = new HostFleetScheduler({
+ perform,
+ recordRequest: vi.fn(),
+ })
+
+ await expect(scheduler.request('fails')).rejects.toThrow(failure)
+ await expect(scheduler.request('recovers')).resolves.toBeUndefined()
+
+ expect(perform).toHaveBeenCalledTimes(2)
+ })
+
+ it('releases active and queued waiters on stop without starting queued work', async () => {
+ const activePass = deferred()
+ const requests: HostFleetReconcileRequest[] = []
+ const scheduler = new HostFleetScheduler({
+ perform: async request => {
+ requests.push({ ...request })
+ await activePass.promise
+ },
+ recordRequest: vi.fn(),
+ })
+
+ const active = scheduler.request('active')
+ await vi.waitFor(() => expect(requests).toHaveLength(1))
+ scheduler.bumpInputRevision()
+ const queued = scheduler.request('queued')
+
+ scheduler.stop()
+ await Promise.all([active, queued, scheduler.request('after stop')])
+ activePass.resolve()
+ await Promise.resolve()
+
+ expect(requests).toHaveLength(1)
+ })
+
+ it('tracks lifecycle state and cancels obsolete retries', async () => {
+ vi.useFakeTimers()
+ const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(
+ async () => undefined
+ )
+ const scheduler = new HostFleetScheduler({
+ perform,
+ recordRequest: vi.fn(),
+ })
+ const firstGeneration = scheduler.beginLifecycleTransition()
+
+ expect(scheduler.currentLifecycleGeneration).toBe(firstGeneration)
+ expect(scheduler.appliedLifecycleGeneration).toBe(-1)
+
+ scheduler.scheduleLifecycleRetry({
+ reason: 'failed lifecycle',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: firstGeneration,
+ inputRevision: 0,
+ })
+ await vi.advanceTimersByTimeAsync(4999)
+ expect(perform).not.toHaveBeenCalled()
+
+ const secondGeneration = scheduler.beginLifecycleTransition()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(perform).not.toHaveBeenCalled()
+
+ scheduler.scheduleLifecycleRetry({
+ reason: 'current lifecycle',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: secondGeneration,
+ inputRevision: 0,
+ })
+ scheduler.markLifecycleApplied(secondGeneration)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(scheduler.appliedLifecycleGeneration).toBe(secondGeneration)
+ expect(perform).not.toHaveBeenCalled()
+ })
+
+ it('does not let a stale applied marker clear the current generation retry', async () => {
+ vi.useFakeTimers()
+ const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(
+ async () => undefined
+ )
+ const scheduler = new HostFleetScheduler({
+ perform,
+ recordRequest: vi.fn(),
+ })
+ const staleGeneration = scheduler.beginLifecycleTransition()
+ const currentGeneration = scheduler.beginLifecycleTransition()
+ scheduler.scheduleLifecycleRetry({
+ reason: 'current lifecycle',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: currentGeneration,
+ inputRevision: 0,
+ })
+
+ scheduler.markLifecycleApplied(staleGeneration)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(perform).toHaveBeenCalledWith(
+ expect.objectContaining({ ccLifecycleGeneration: currentGeneration })
+ )
+ })
+
+ it('retries an unapplied current lifecycle generation with bounded backoff', async () => {
+ vi.useFakeTimers()
+ const requests: HostFleetReconcileRequest[] = []
+ const scheduler = new HostFleetScheduler({
+ perform: async request => {
+ requests.push({ ...request })
+ },
+ recordRequest: vi.fn(),
+ })
+ const generation = scheduler.beginLifecycleTransition()
+ const failedRequest: HostFleetReconcileRequest = {
+ reason: 'failed lifecycle',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: generation,
+ inputRevision: 0,
+ }
+
+ scheduler.scheduleLifecycleRetry(failedRequest)
+ await vi.advanceTimersByTimeAsync(5000)
+ scheduler.scheduleLifecycleRetry(failedRequest)
+ await vi.advanceTimersByTimeAsync(14999)
+ expect(requests).toHaveLength(1)
+ await vi.advanceTimersByTimeAsync(1)
+
+ expect(requests).toEqual([
+ expect.objectContaining({
+ reason: 'CommunicationChannel lifecycle convergence retry',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: generation,
+ }),
+ expect.objectContaining({
+ reason: 'CommunicationChannel lifecycle convergence retry',
+ mode: 'lifecycle',
+ ccLifecycleGeneration: generation,
+ }),
+ ])
+ })
+})
diff --git a/host-context-controller/src/hostFleetScheduler.ts b/host-context-controller/src/hostFleetScheduler.ts
new file mode 100644
index 000000000..d872d6cad
--- /dev/null
+++ b/host-context-controller/src/hostFleetScheduler.ts
@@ -0,0 +1,246 @@
+const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000] as const
+
+export type HostFleetReconcileMode = 'full' | 'lifecycle'
+
+export type HostFleetReconcileRequest = {
+ reason: string
+ mode: HostFleetReconcileMode
+ ccLifecycleGeneration?: number
+ inputRevision: number
+}
+
+type PendingHostFleetReconcile = HostFleetReconcileRequest & {
+ promise: Promise<void>
+ resolve: () => void
+}
+
+type ActiveHostFleetReconcile = HostFleetReconcileRequest & {
+ promise: Promise<void>
+}
+
+export type HostFleetRequestResult = 'started' | 'coalesced' | 'trailing'
+
+type HostFleetSchedulerDependencies = {
+ perform: (request: HostFleetReconcileRequest) => Promise<void>
+ recordRequest: (result: HostFleetRequestResult) => void
+}
+
+function hostFleetRequestCovers(
+ active: HostFleetReconcileRequest,
+ requested: HostFleetReconcileRequest
+): boolean {
+ const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'
+ if (!modeCovered) return false
+ if (active.inputRevision !== requested.inputRevision) return false
+ if (requested.ccLifecycleGeneration === undefined) return true
+ return active.ccLifecycleGeneration === requested.ccLifecycleGeneration
+}
+
+function mergeHostFleetRequests(
+ pending: HostFleetReconcileRequest,
+ requested: HostFleetReconcileRequest
+): HostFleetReconcileRequest {
+ const pendingGeneration = pending.ccLifecycleGeneration
+ const requestedGeneration = requested.ccLifecycleGeneration
+ return {
+ reason: requested.reason,
+ mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',
+ inputRevision: Math.max(pending.inputRevision, requested.inputRevision),
+ ccLifecycleGeneration:
+ pendingGeneration === undefined
+ ? requestedGeneration
+ : requestedGeneration === undefined
+ ? pendingGeneration
+ : Math.max(pendingGeneration, requestedGeneration),
+ }
+}
+
+/**
+ * Owns temporal coordination for Host fleet convergence. Kubernetes reads and
+ * mutations stay behind `perform`, so this scheduler only serializes requests,
+ * coalesces a single trailing pass, and manages lifecycle retry/shutdown state.
+ */
+export class HostFleetScheduler {
+ private ccLifecycleGeneration = 0
+ private ccAppliedLifecycleGeneration = -1
+ private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null
+ private ccFleetRetryAttempt = 0
+ private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null
+ private hostFleetReconcilePending: PendingHostFleetReconcile | null = null
+ private hostFleetInputRevision = 0
+ private resolveHostFleetShutdown: () => void = () => {}
+ private readonly hostFleetShutdown = new Promise<void>(resolve => {
+ this.resolveHostFleetShutdown = resolve
+ })
+ private stopped = false
+
+ constructor(private readonly dependencies: HostFleetSchedulerDependencies) {}
+
+ get currentLifecycleGeneration(): number {
+ return this.ccLifecycleGeneration
+ }
+
+ get appliedLifecycleGeneration(): number {
+ return this.ccAppliedLifecycleGeneration
+ }
+
+ beginLifecycleTransition(): number {
+ this.ccLifecycleGeneration += 1
+ this.ccFleetRetryAttempt = 0
+ this.clearLifecycleRetry()
+ return this.ccLifecycleGeneration
+ }
+
+ bumpInputRevision(): number {
+ this.hostFleetInputRevision += 1
+ return this.hostFleetInputRevision
+ }
+
+ markLifecycleApplied(generation: number): void {
+ if (generation !== this.ccLifecycleGeneration) return
+ this.ccAppliedLifecycleGeneration = generation
+ this.ccFleetRetryAttempt = 0
+ this.clearLifecycleRetry()
+ }
+
+ scheduleLifecycleRetry(request: HostFleetReconcileRequest): void {
+ const generation = request.ccLifecycleGeneration
+ if (generation === undefined) return
+ if (
+ this.stopped ||
+ generation !== this.ccLifecycleGeneration ||
+ generation === this.ccAppliedLifecycleGeneration ||
+ this.ccFleetRetryTimer
+ ) {
+ return
+ }
+
+ const delayIndex = Math.min(
+ this.ccFleetRetryAttempt,
+ COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1
+ )
+ const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]
+ this.ccFleetRetryAttempt += 1
+ this.ccFleetRetryTimer = setTimeout(() => {
+ this.ccFleetRetryTimer = null
+ if (
+ this.stopped ||
+ generation !== this.ccLifecycleGeneration ||
+ generation === this.ccAppliedLifecycleGeneration
+ ) {
+ return
+ }
+ void this.request(
+ 'CommunicationChannel lifecycle convergence retry',
+ generation,
+ request.mode
+ )
+ }, retryDelay)
+ }
+
+ /**
+ * Cache transitions, periodic resyncs, and retries share one active pass and
+ * at most one merged trailing pass. Each caller waits only for work covering
+ * its request, keeping startup bounded under a sustained retry stream.
+ */
+ request(
+ reason: string,
+ ccLifecycleGeneration?: number,
+ mode?: HostFleetReconcileMode
+ ): Promise<void> {
+ if (this.stopped) return Promise.resolve()
+ const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')
+ if (
+ ccLifecycleGeneration !== undefined &&
+ (ccLifecycleGeneration !== this.ccLifecycleGeneration ||
+ (requestMode === 'lifecycle' &&
+ ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))
+ ) {
+ return Promise.resolve()
+ }
+
+ const request: HostFleetReconcileRequest = {
+ reason,
+ mode: requestMode,
+ ccLifecycleGeneration,
+ inputRevision: this.hostFleetInputRevision,
+ }
+ const active = this.hostFleetReconcileInFlight
+ if (!active) {
+ this.dependencies.recordRequest('started')
+ return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))
+ }
+ if (hostFleetRequestCovers(active, request)) {
+ this.dependencies.recordRequest('coalesced')
+ return this.waitForHostFleetOrShutdown(active.promise)
+ }
+
+ if (!this.hostFleetReconcilePending) {
+ let resolve!: () => void
+ const promise = new Promise<void>(resolvePromise => {
+ resolve = resolvePromise
+ })
+ this.hostFleetReconcilePending = {
+ ...request,
+ promise,
+ resolve,
+ }
+ } else {
+ const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)
+ this.hostFleetReconcilePending.reason = merged.reason
+ this.hostFleetReconcilePending.mode = merged.mode
+ this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration
+ this.hostFleetReconcilePending.inputRevision = merged.inputRevision
+ }
+ this.dependencies.recordRequest('coalesced')
+ return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)
+ }
+
+ stop(): void {
+ if (this.stopped) return
+ this.stopped = true
+ this.resolveHostFleetShutdown()
+ this.hostFleetReconcilePending?.resolve()
+ this.hostFleetReconcilePending = null
+ this.clearLifecycleRetry()
+ }
+
+ private clearLifecycleRetry(): void {
+ if (!this.ccFleetRetryTimer) return
+ clearTimeout(this.ccFleetRetryTimer)
+ this.ccFleetRetryTimer = null
+ }
+
+ private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {
+ return Promise.race([coverage, this.hostFleetShutdown])
+ }
+
+ private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {
+ const reconcileRequest: HostFleetReconcileRequest = {
+ reason: request.reason,
+ mode: request.mode,
+ ccLifecycleGeneration: request.ccLifecycleGeneration,
+ inputRevision: request.inputRevision,
+ }
+ const promise = this.dependencies.perform(reconcileRequest)
+ const active: ActiveHostFleetReconcile = { ...reconcileRequest, promise }
+ this.hostFleetReconcileInFlight = active
+
+ const settle = () => {
+ if (this.hostFleetReconcileInFlight !== active) return
+ this.hostFleetReconcileInFlight = null
+ const pending = this.hostFleetReconcilePending
+ this.hostFleetReconcilePending = null
+ if (!pending) return
+ if (this.stopped) {
+ pending.resolve()
+ return
+ }
+ this.dependencies.recordRequest('trailing')
+ const trailing = this.startHostFleetReconcile(pending)
+ void trailing.then(pending.resolve, pending.resolve)
+ }
+ void promise.then(settle, settle)
+ return promise
+ }
+}
diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.ts
index 495a4bc38..53625ff43 100644
--- a/host-context-controller/src/hostReconciler.test.ts
+++ b/host-context-controller/src/hostReconciler.test.ts
@@ -943,6 +943,42 @@ describe('HostReconciler cleanup authority gate', () => {
expect(del).not.toHaveBeenCalled()
})
+ it('defers orphan deletion when authority changes before serializer admission', async () => {
+ const getObj = vi.fn().mockRejectedValue({ code: 404 })
+ const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
+ const authority = { known: true, generation: 3 }
+ reconciler.setHostWatchAuthority(() => ({ ...authority }))
+ reconciler.setResolveCurrentHost(() => undefined)
+ const internals = reconciler as unknown as {
+ lifecycle: {
+ serializeByHost(name: string, task: () => Promise<void>): Promise<void>
+ }
+ listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
+ deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
+ }
+ vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([
+ orphanDeployment('authority-race-host'),
+ ])
+ const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue(undefined)
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = internals.lifecycle.serializeByHost('authority-race-host', async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ })
+ await blockerStarted.promise
+
+ const cleanup = reconciler.fullReconcile([])
+ await vi.waitFor(() => expect(getObj).toHaveBeenCalledOnce())
+ authority.known = false
+ authority.generation = 4
+ releaseBlocker.resolve(undefined)
+ await occupied
+ await cleanup
+
+ expect(del).not.toHaveBeenCalled()
+ })
+
it('defers an orphan whose fresh authoritative read fails', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 503 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
@@ -996,3 +1032,687 @@ describe('HostReconciler cleanup authority gate', () => {
expect(del).not.toHaveBeenCalled()
})
})
+
+describe('HostReconciler Host inventory mutation authority', () => {
+ function wireAuthoritativeHost(
+ reconciler: HostReconciler,
+ host: HostCRD,
+ authorityRef: { current: { known: boolean; generation: number } }
+ ): void {
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => authorityRef.current)
+ }
+
+ function stubPreDeploymentReconcileEffects(reconciler: HostReconciler): void {
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockResolvedValue({ ok: true })
+ for (const method of [
+ 'ensureHostServiceAccount',
+ 'ensureHostRole',
+ 'ensureHostRoleBinding',
+ 'ensurePvc',
+ 'ensureService',
+ 'ensureMcpHostIngressNetworkPolicy',
+ 'ensureMcpHostGfsEgressNetworkPolicy',
+ 'ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy',
+ 'ensureRpcProxyMcpHostIngressNetworkPolicy',
+ 'ensureRpcProxyHostEgressNetworkPolicy',
+ 'ensureDesktopNetworkPolicy',
+ 'ensureChannelReaderEgressNetworkPolicy',
+ 'ensureWorkflowApprovalReaderHostEgressNetworkPolicy',
+ ]) {
+ vi.spyOn(reconciler as any, method).mockResolvedValue(undefined)
+ }
+ }
+
+ function runtimeTokenScopeWithoutChannels(host: HostCRD): string {
+ return (
+ HostReconciler as unknown as {
+ runtimeTokenScopeHash(selected: HostCRD, hasChannelIngress: boolean): string
+ }
+ ).runtimeTokenScopeHash(host, false)
+ }
+
+ it('rejects a queued reconcile when Host authority changes before admission', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = makeHost({ name: 'authority-race' })
+ const authority = { current: { known: true, generation: 7 } }
+ wireAuthoritativeHost(reconciler, host, authority)
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }) as Promise<void>
+ await blockerStarted.promise
+
+ const pending = reconciler.reconcile(host)
+ authority.current = { known: false, generation: 8 }
+ releaseBlocker.resolve(undefined)
+ await occupied
+
+ await expect(pending).rejects.toThrow(/Host inventory authority/)
+ expect(reconcileCore).not.toHaveBeenCalled()
+ })
+
+ it('rejects a queued fleet worker after Context authority retires and recovers', async () => {
+ const reconciler = makeBasicReconciler()
+ const hostA = makeHost({ name: 'context-authority-a' })
+ const hostB = makeHost({ name: 'context-authority-b' })
+ const hosts = new Map([
+ [hostA.name, hostA],
+ [hostB.name, hostB],
+ ])
+ reconciler.setResolveCurrentHost(name => hosts.get(name))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 7 }))
+ const mutationAuthority = {
+ current: {
+ known: true,
+ hostGeneration: 7,
+ contextGeneration: 11,
+ },
+ }
+ ;(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+ vi.spyOn(reconciler as any, 'collectHostCleanupFailures').mockResolvedValue([])
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = (reconciler as any).lifecycle.serializeByHost(hostB.name, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }) as Promise<void>
+ await blockerStarted.promise
+
+ const fleet = reconciler.fullReconcile([hostA, hostB])
+ await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))
+
+ // Context retired and recovered to a fresh LIST -> WATCH generation while
+ // Host authority remained continuously known: a host-only lease would miss
+ // this ABA transition and admit Host B with stale derived mounts.
+ mutationAuthority.current = {
+ known: true,
+ hostGeneration: 7,
+ contextGeneration: 13,
+ }
+ releaseBlocker.resolve(undefined)
+ await occupied
+
+ await expect(fleet).rejects.toBeInstanceOf(HostFleetReconcileError)
+ expect(reconcileCore).toHaveBeenCalledTimes(1)
+ expect(reconcileCore).not.toHaveBeenCalledWith(hostB, expect.any(Function))
+ })
+
+ it('retires a Host reconcile when its Context is superseded within the same watch', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'context-revision-race', contextRef: 'context-a' }),
+ uid: 'context-revision-race-uid',
+ generation: 5,
+ }
+ const contextV1 = { name: 'context-a', spec: { sharedFileSystems: [] } }
+ const contextV2 = {
+ name: 'context-a',
+ spec: {
+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
+ },
+ }
+ let selectedContext: unknown = contextV1
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostMutationAuthority(() => ({
+ known: true,
+ hostGeneration: 7,
+ contextGeneration: 11,
+ }))
+ reconciler.setResolveHostMutationDependencies(() => [selectedContext])
+
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred()
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
+ secretReadStarted.resolve(undefined)
+ await releaseSecretRead.promise
+ return { ok: true }
+ })
+ const ensureServiceAccount = vi
+ .spyOn(reconciler as any, 'ensureHostServiceAccount')
+ .mockResolvedValue(undefined)
+
+ // R1-M2: a benign mid-pass supersession must NOT be reported as a hard
+ // reconcile failure. The single reconcile() catch used to fire
+ // enqueueControllerError('reconcile_exception') → controller_error telemetry
+ // (status:'failed') AND an administrative 'failed' outcome to control-api for
+ // these retirements. Spy (observe-only) to prove the benign branch emits
+ // neither, while still re-throwing so callers treat it as a retire.
+ const enqueueControllerError = vi.spyOn(reconciler as any, 'enqueueControllerError')
+ const enqueueAdministrativeOutcome = vi.spyOn(reconciler as any, 'enqueueAdministrativeOutcome')
+
+ const pending = reconciler.reconcile(host)
+ await secretReadStarted.promise
+ selectedContext = contextV2
+ releaseSecretRead.resolve(undefined)
+
+ await expect(pending).rejects.toThrow(/Host mutation dependency changed/)
+ expect(ensureServiceAccount).not.toHaveBeenCalled()
+ expect(enqueueControllerError).not.toHaveBeenCalled()
+ expect(enqueueAdministrativeOutcome).not.toHaveBeenCalledWith(
+ expect.anything(),
+ 'failed',
+ expect.anything()
+ )
+ })
+
+ it('keeps a Host reconcile admitted when a watch replaces dependencies with deep-equal values', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'deep-equal-dependency', contextRef: 'context-a' }),
+ uid: 'deep-equal-dependency-uid',
+ generation: 5,
+ }
+ let selectedContext: unknown = {
+ name: 'context-a',
+ spec: {
+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
+ },
+ }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostMutationAuthority(() => ({
+ known: true,
+ hostGeneration: 7,
+ contextGeneration: 11,
+ }))
+ reconciler.setResolveHostMutationDependencies(() => [selectedContext])
+
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred()
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
+ secretReadStarted.resolve(undefined)
+ await releaseSecretRead.promise
+ return { ok: true }
+ })
+ const admitted = new Error('deep-equal dependency admitted')
+ const ensureServiceAccount = vi
+ .spyOn(reconciler as any, 'ensureHostServiceAccount')
+ .mockRejectedValue(admitted)
+
+ const pending = reconciler.reconcile(host)
+ await secretReadStarted.promise
+ selectedContext = structuredClone(selectedContext)
+ releaseSecretRead.resolve(undefined)
+
+ await expect(pending).rejects.toBe(admitted)
+ expect(ensureServiceAccount).toHaveBeenCalled()
+ })
+
+ it.each([
+ [
+ 'referenced SharedFileSystem revision',
+ { name: 'documents', spec: { storageClassName: 'fast-v1' } },
+ { name: 'documents', spec: { storageClassName: 'fast-v2' } },
+ ],
+ ['referenced SharedFileSystem mountability', true, false],
+ [
+ 'Host-scoped CommunicationChannel revision',
+ { name: 'telegram', spec: { hostRef: 'dependency-race' } },
+ {
+ name: 'telegram',
+ spec: {
+ hostRef: 'dependency-race',
+ credentialsSecretRef: { name: 'telegram-v2' },
+ },
+ },
+ ],
+ ])('retires a Host reconcile when its %s changes', async (_label, original, replacement) => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'dependency-race', contextRef: 'context-a' }),
+ uid: 'dependency-race-uid',
+ generation: 5,
+ }
+ const context = { name: 'context-a', spec: { sharedFileSystems: [] } }
+ let selectedDependency: unknown = original
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostMutationAuthority(() => ({
+ known: true,
+ hostGeneration: 7,
+ contextGeneration: 11,
+ }))
+ reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])
+
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred()
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
+ secretReadStarted.resolve(undefined)
+ await releaseSecretRead.promise
+ return { ok: true }
+ })
+ const ensureServiceAccount = vi
+ .spyOn(reconciler as any, 'ensureHostServiceAccount')
+ .mockResolvedValue(undefined)
+
+ const pending = reconciler.reconcile(host)
+ await secretReadStarted.promise
+ selectedDependency = replacement
+ releaseSecretRead.resolve(undefined)
+
+ await expect(pending).rejects.toThrow(/Host mutation dependency changed/)
+ expect(ensureServiceAccount).not.toHaveBeenCalled()
+ })
+
+ it('does not delete runtime after a Secret rejection when Host authority changes in flight', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = { ...makeHost({ name: 'secret-authority-race' }), uid: 'host-uid', generation: 3 }
+ const authority = { current: { known: true, generation: 7 } }
+ wireAuthoritativeHost(reconciler, host, authority)
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred()
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
+ secretReadStarted.resolve(undefined)
+ await releaseSecretRead.promise
+ return {
+ ok: false,
+ reason: 'SecretNotFound',
+ message: 'secret missing in stale spec',
+ }
+ })
+ const deleteRuntime = vi
+ .spyOn(reconciler as any, 'deleteHostRuntimeResources')
+ .mockResolvedValue(undefined)
+
+ const pending = reconciler.reconcile(host)
+ await secretReadStarted.promise
+ authority.current = { known: false, generation: 8 }
+ releaseSecretRead.resolve(undefined)
+
+ await expect(pending).rejects.toThrow(/Host inventory authority/)
+ expect(deleteRuntime).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ [
+ 'UID',
+ (host: HostCRD) => ({ ...host, uid: 'replacement-host-uid', generation: 1 }),
+ /Host identity/,
+ ],
+ [
+ 'spec generation',
+ (host: HostCRD) => ({
+ ...host,
+ generation: (host.generation ?? 0) + 1,
+ spec: { ...host.spec, secretRef: 'replacement-secret' },
+ }),
+ /Host spec generation/,
+ ],
+ ] as const)(
+ 'does not delete runtime after a Secret rejection when the Host %s changes in flight',
+ async (_, replaceCurrent, expectedError) => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'secret-identity-race' }),
+ uid: 'host-uid',
+ generation: 3,
+ }
+ let current: HostCRD = host
+ const authority = { current: { known: true, generation: 7 } }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => authority.current)
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred()
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
+ secretReadStarted.resolve(undefined)
+ await releaseSecretRead.promise
+ return {
+ ok: false,
+ reason: 'SecretNotFound',
+ message: 'secret missing in stale spec',
+ }
+ })
+ const deleteRuntime = vi
+ .spyOn(reconciler as any, 'deleteHostRuntimeResources')
+ .mockResolvedValue(undefined)
+
+ const pending = reconciler.reconcile(host)
+ await secretReadStarted.promise
+ current = replaceCurrent(host)
+ releaseSecretRead.resolve(undefined)
+
+ await expect(pending).rejects.toThrow(expectedError)
+ expect(deleteRuntime).not.toHaveBeenCalled()
+ }
+ )
+
+ it('revalidates the Host spec after token provisioning and before Deployment mutation', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'deployment-generation-race' }),
+ uid: 'host-uid',
+ generation: 3,
+ }
+ let current = host
+ const authority = { current: { known: true, generation: 11 } }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => authority.current)
+ stubPreDeploymentReconcileEffects(reconciler)
+ const tokenProvisionStarted = deferred()
+ const releaseTokenProvision = deferred()
+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockImplementation(async () => {
+ tokenProvisionStarted.resolve(undefined)
+ await releaseTokenProvision.promise
+ return { revision: 'stale-revision', scopeHash: 'stale-scope' }
+ })
+ const ensureDeployment = vi
+ .spyOn(reconciler as any, 'ensureDeployment')
+ .mockResolvedValue(undefined)
+
+ const pending = reconciler.reconcile(host)
+ await tokenProvisionStarted.promise
+ current = {
+ ...host,
+ generation: 4,
+ spec: { ...host.spec, contextRef: 'replacement-context' },
+ }
+ releaseTokenProvision.resolve(undefined)
+
+ await expect(pending).rejects.toThrow(/Host spec generation/)
+ expect(ensureDeployment).not.toHaveBeenCalled()
+ })
+
+ it('revalidates the Host spec at the final Deployment create-state boundary', async () => {
+ const appsApi = createMockAppsApi()
+ const reconciler = new HostReconciler({} as k8s.KubeConfig, {
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(createMockCoreApi()),
+ networkingApi: asNetworkingApi(createMockNetworkingApi()),
+ rbacApi: asRbacApi(createMockRbacApi()),
+ })
+ const host = {
+ ...makeHost({ name: 'deployment-create-race' }),
+ uid: 'host-uid',
+ generation: 3,
+ }
+ let current = host
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))
+ stubPreDeploymentReconcileEffects(reconciler)
+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
+ revision: 'runtime-revision',
+ scopeHash: runtimeTokenScopeWithoutChannels(host),
+ })
+ let channelIngressReads = 0
+ vi.spyOn(reconciler as any, 'hasChannelIngress').mockImplementation(() => {
+ channelIngressReads += 1
+ if (channelIngressReads === 2) {
+ current = {
+ ...host,
+ generation: 4,
+ spec: { ...host.spec, contextRef: 'replacement-context' },
+ }
+ }
+ return false
+ })
+
+ await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
+
+ expect(channelIngressReads).toBe(2)
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ })
+
+ it('revalidates the Host spec after a Deployment create conflict and before replace', async () => {
+ const appsApi = createMockAppsApi()
+ const reconciler = new HostReconciler({} as k8s.KubeConfig, {
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(createMockCoreApi()),
+ networkingApi: asNetworkingApi(createMockNetworkingApi()),
+ rbacApi: asRbacApi(createMockRbacApi()),
+ })
+ const host = {
+ ...makeHost({ name: 'deployment-replace-race' }),
+ uid: 'host-uid',
+ generation: 3,
+ }
+ let current = host
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))
+ stubPreDeploymentReconcileEffects(reconciler)
+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
+ revision: 'runtime-revision',
+ scopeHash: runtimeTokenScopeWithoutChannels(host),
+ })
+ vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
+ appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {
+ current = {
+ ...host,
+ generation: 4,
+ spec: { ...host.spec, contextRef: 'replacement-context' },
+ }
+ return { metadata: { resourceVersion: 'rv-existing' } }
+ })
+
+ await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
+
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ })
+
+ it('revalidates the Host spec before retrying a conflicted Deployment replace', async () => {
+ const appsApi = createMockAppsApi()
+ const reconciler = new HostReconciler({} as k8s.KubeConfig, {
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(createMockCoreApi()),
+ networkingApi: asNetworkingApi(createMockNetworkingApi()),
+ rbacApi: asRbacApi(createMockRbacApi()),
+ })
+ const host = {
+ ...makeHost({ name: 'deployment-retry-race' }),
+ uid: 'host-uid',
+ generation: 3,
+ }
+ let current = host
+ let deploymentReads = 0
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))
+ stubPreDeploymentReconcileEffects(reconciler)
+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
+ revision: 'runtime-revision',
+ scopeHash: runtimeTokenScopeWithoutChannels(host),
+ })
+ vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
+ appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ appsApi.readNamespacedDeployment.mockImplementation(async () => {
+ deploymentReads += 1
+ if (deploymentReads === 2) {
+ current = {
+ ...host,
+ generation: 4,
+ spec: { ...host.spec, contextRef: 'replacement-context' },
+ }
+ }
+ return { metadata: { resourceVersion: `rv-existing-${deploymentReads}` } }
+ })
+ appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ const random = vi.spyOn(Math, 'random').mockReturnValue(0)
+
+ try {
+ await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
+ } finally {
+ random.mockRestore()
+ }
+
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ })
+
+ it('resolves the latest authoritative Host object at reconcile admission', async () => {
+ const reconciler = makeBasicReconciler()
+ const stale = { ...makeHost({ name: 'latest-host' }), generation: 1 }
+ const current = {
+ ...stale,
+ generation: 2,
+ spec: { ...stale.spec, contextRef: 'current-context' },
+ }
+ const authority = { current: { known: true, generation: 7 } }
+ wireAuthoritativeHost(reconciler, current, authority)
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+
+ await reconciler.reconcile(stale)
+
+ expect(reconcileCore).toHaveBeenCalledOnce()
+ expect(reconcileCore).toHaveBeenCalledWith(
+ expect.objectContaining({
+ generation: 2,
+ spec: expect.objectContaining({ contextRef: 'current-context' }),
+ }),
+ expect.any(Function)
+ )
+ })
+
+ it('does not patch channel-reader after Host authority changes during revision resolution', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'channel-authority-race' }),
+ uid: 'channel-authority-race-uid',
+ generation: 3,
+ }
+ const authority = { current: { known: true, generation: 9 } }
+ wireAuthoritativeHost(reconciler, host, authority)
+ const revisionStarted = deferred()
+ const releaseRevision = deferred<string>()
+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(
+ async () => {
+ revisionStarted.resolve(undefined)
+ return releaseRevision.promise
+ }
+ )
+ const patchDeployment = vi
+ .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
+ .mockResolvedValue(undefined)
+
+ const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)
+ await revisionStarted.promise
+ authority.current = { known: false, generation: 10 }
+ releaseRevision.resolve('resolved-revision')
+ await pending
+
+ expect(patchDeployment).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ ['UID', (host: HostCRD) => ({ ...host, uid: 'replacement-channel-reader-uid' })],
+ [
+ 'spec generation',
+ (host: HostCRD) => ({
+ ...host,
+ generation: (host.generation ?? 0) + 1,
+ spec: { ...host.spec, contextRef: 'replacement-context' },
+ }),
+ ],
+ ] as const)(
+ 'does not patch channel-reader when the Host %s changes during revision resolution',
+ async (_, replaceCurrent) => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'channel-host-race' }),
+ uid: 'channel-host-race-uid',
+ generation: 3,
+ }
+ let current: HostCRD = host
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 9 }))
+ const revisionStarted = deferred()
+ const releaseRevision = deferred<string>()
+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(
+ async () => {
+ revisionStarted.resolve(undefined)
+ return releaseRevision.promise
+ }
+ )
+ const patchDeployment = vi
+ .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
+ .mockResolvedValue(undefined)
+
+ const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)
+ await revisionStarted.promise
+ current = replaceCurrent(host)
+ releaseRevision.resolve('resolved-revision')
+ await pending
+
+ expect(patchDeployment).not.toHaveBeenCalled()
+ }
+ )
+
+ it('serializes a fresh channel-reader revision behind an older in-flight Host reconcile', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = {
+ ...makeHost({ name: 'channel-revision-order' }),
+ uid: 'channel-revision-order-uid',
+ generation: 4,
+ }
+ const authority = { current: { known: true, generation: 9 } }
+ wireAuthoritativeHost(reconciler, host, authority)
+ const staleReconcileStarted = deferred()
+ const releaseStaleReconcile = deferred()
+ const order: string[] = []
+ let liveRevision = ''
+
+ vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async () => {
+ order.push('reconcile-a:start')
+ staleReconcileStarted.resolve(undefined)
+ await releaseStaleReconcile.promise
+ liveRevision = 'revision-a'
+ order.push('reconcile-a:write')
+ })
+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue(
+ 'revision-b'
+ )
+ const patchDeployment = vi
+ .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
+ .mockImplementation(async (request: any) => {
+ const { body } = request
+ liveRevision = body.spec.template.metadata.annotations['clerum.io/credentials-revision']
+ order.push('revision-b:patch')
+ })
+
+ const staleReconcile = reconciler.reconcile(host)
+ await staleReconcileStarted.promise
+ const freshRevision = reconciler.patchChannelReaderRevisionAnnotation(host.name)
+
+ // Give B the opportunity to patch before A is released. Without a shared
+ // Host lane, B writes first and A deterministically overwrites it.
+ await Promise.resolve()
+ await Promise.resolve()
+ releaseStaleReconcile.resolve(undefined)
+ await Promise.all([staleReconcile, freshRevision])
+
+ expect(liveRevision).toBe('revision-b')
+ expect(patchDeployment).toHaveBeenCalledOnce()
+ expect(order).toEqual(['reconcile-a:start', 'reconcile-a:write', 'revision-b:patch'])
+ })
+
+ it('rejects a queued delete when Host authority changes before admission', async () => {
+ const reconciler = makeBasicReconciler()
+ const host = makeHost({ name: 'delete-authority-race' })
+ const authority = { current: { known: true, generation: 11 } }
+ reconciler.setResolveCurrentHost(() => undefined)
+ reconciler.setHostWatchAuthority(() => authority.current)
+ const deleteRuntime = vi
+ .spyOn(reconciler as any, 'deleteHostRuntimeResources')
+ .mockResolvedValue(undefined)
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }) as Promise<void>
+ await blockerStarted.promise
+
+ const pending = reconciler.reconcileDelete(host.name, host.namespace)
+ authority.current = { known: false, generation: 12 }
+ releaseBlocker.resolve(undefined)
+ await occupied
+
+ await expect(pending).rejects.toThrow(/Host inventory authority/)
+ expect(deleteRuntime).not.toHaveBeenCalled()
+ })
+})
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 5730e5553..9e2f236d5 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -1,6 +1,7 @@
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash, randomUUID } from 'crypto'
+import { isDeepStrictEqual } from 'node:util'
import * as path from 'path'
import type { AdministrativeOutcomeReporter } from './administrativeOutcomeReporter'
import { config } from './config'
@@ -43,7 +44,7 @@ import {
resolveStatelessImagePullPolicy,
} from './statelessDeployment'
import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'
-import { StatelessLifecycleExecutor } from './statelessLifecycleExecutor'
+import { ReflectHostOutcomeFn, StatelessLifecycleExecutor } from './statelessLifecycleExecutor'
import {
CommunicationChannelCRD,
HostCRD,
@@ -78,6 +79,74 @@ export type HostReconcileSource = 'urgent' | 'retry' | 'fleet'
/** Immutable Host watch authority snapshot captured at a full-pass boundary. */
export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }
+/**
+ * Immutable lease for every inventory that contributes to a Host mutation.
+ * Host resources come from the Host watch; mounted SharedFileSystems come from
+ * the Context watch. Keeping both generations prevents a false-stable
+ * Host-only lease when Context retires and recovers during queued work.
+ */
+export type HostMutationAuthoritySnapshot = {
+ known: boolean
+ hostGeneration: number
+ contextGeneration: number
+}
+
+class HostInventoryAuthorityUnavailableError extends Error {
+ constructor(
+ action: string,
+ captured: HostMutationAuthoritySnapshot,
+ current: HostMutationAuthoritySnapshot
+ ) {
+ super(
+ `Host inventory authority changed before ${action} admission ` +
+ `(captured known=${captured.known} hostGeneration=${captured.hostGeneration} ` +
+ `contextGeneration=${captured.contextGeneration}, current known=${current.known} ` +
+ `hostGeneration=${current.hostGeneration} contextGeneration=${current.contextGeneration})`
+ )
+ this.name = 'HostInventoryAuthorityUnavailableError'
+ }
+}
+
+class HostMutationIdentityChangedError extends Error {
+ constructor(action: string, hostName: string) {
+ super(`Host identity changed before ${action} admission for "${hostName}"`)
+ this.name = 'HostMutationIdentityChangedError'
+ }
+}
+
+class HostMutationSpecRevisionChangedError extends Error {
+ constructor(action: string, hostName: string) {
+ super(`Host spec generation changed before ${action} admission for "${hostName}"`)
+ this.name = 'HostMutationSpecRevisionChangedError'
+ }
+}
+
+class HostMutationDependencyChangedError extends Error {
+ constructor(action: string, hostName: string) {
+ super(`Host mutation dependency changed before ${action} admission for "${hostName}"`)
+ this.name = 'HostMutationDependencyChangedError'
+ }
+}
+
+/**
+ * The four benign mid-pass supersession signals. revalidateHostMutationBoundary()
+ * throws these when the watch observes a newer spec/uid/authority-generation while
+ * a reconcile is in flight; a fresh reconcile runs afterwards, so the retired pass
+ * is a WITHDRAWAL, not a hard failure. Matched by `error.name` (string), NOT
+ * `instanceof`: the classes are unexported and statelessLifecycleExecutor.ts
+ * synthesizes name-equivalent plain Errors for the same races.
+ */
+const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([
+ 'HostInventoryAuthorityUnavailableError',
+ 'HostMutationIdentityChangedError',
+ 'HostMutationSpecRevisionChangedError',
+ 'HostMutationDependencyChangedError',
+])
+
+function isBenignSupersessionError(error: unknown): boolean {
+ return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)
+}
+
/**
* The exact §13.2 destructive-cleanup predicate. Deletion of an orphan Host's
* owned bundle is permitted ONLY when watch authority is known, the watch
@@ -378,6 +447,12 @@ export class HostReconciler {
* snapshot for reconcile, and cleanup stays fail-closed (authority unknown).
*/
private resolveCurrentHost: ((name: string) => HostCRD | undefined) | null = null
+ /**
+ * H2: late-bound cache reflector. Wired by McpServerWatcher next to
+ * setResolveCurrentHost; null (standalone) means lifecycle outcomes reflect
+ * only onto the caller's own object (legacy contract).
+ */
+ private reflectHostOutcome: ReflectHostOutcomeFn | null = null
/**
* Snapshot the current Host watch authority + generation. Wired by
* McpServerWatcher. Default is fail-closed (unknown) so orphan cleanup never
@@ -387,6 +462,27 @@ export class HostReconciler {
known: false,
generation: 0,
})
+ /**
+ * Positive Host mutations depend on Host plus Context authority. The default
+ * preserves the standalone HostReconciler contract by deriving a Host-only
+ * lease; McpServerWatcher wires the complete composite lease.
+ */
+ private hostMutationAuthority: () => HostMutationAuthoritySnapshot = () => {
+ const host = this.hostWatchAuthority()
+ return {
+ known: host.known,
+ hostGeneration: host.generation,
+ contextGeneration: 0,
+ }
+ }
+ /**
+ * Exact revisions of the cross-resource inputs selected by a Host. Watch
+ * generations fence LIST -> WATCH authority, but normal MODIFIED events stay
+ * in the same generation. The watcher supplies the selected Context,
+ * referenced SharedFileSystems plus mountability, and Host-scoped
+ * CommunicationChannels so unrelated fleet changes do not retire this lease.
+ */
+ private resolveHostMutationDependencies: ((host: HostCRD) => readonly unknown[]) | null = null
constructor(kc: k8s.KubeConfig, deps?: HostReconcilerDeps) {
this.appsApi =
@@ -420,7 +516,10 @@ export class HostReconciler {
now: this.now,
countCommunicationChannels: hostName => this.countCommunicationChannels(hostName),
isCommunicationChannelCacheSynced: () => this.ccCacheSyncedFn(),
- reconcileCore: host => this.reconcileCore(host),
+ reconcileCore: (host, revalidate) => this.reconcileCore(host, revalidate),
+ prepareHostMutationAdmission: (action, host) =>
+ this.prepareHostMutationAdmission(action, host),
+ reflectHostOutcome: (name, uid, apply) => this.reflectHostOutcome?.(name, uid, apply),
onLifecycleStatusCommitted: (host, lifecycle) => {
const occurredAt = this.now().toISOString()
this.infrastructureTelemetryReporter?.enqueueHealthTransition({
@@ -486,6 +585,15 @@ export class HostReconciler {
this.resolveCurrentHost = fn
}
+ /**
+ * H2: late-bound setter for the guarded cache reflector. Wired by
+ * McpServerWatcher from its `hosts` cache (uid-guarded apply). Pattern
+ * mirrors setResolveCurrentHost.
+ */
+ setReflectHostOutcome(fn: ReflectHostOutcomeFn): void {
+ this.reflectHostOutcome = fn
+ }
+
/**
* Late-bound setter for the Host watch authority/generation snapshot. Wired by
* McpServerWatcher so orphan cleanup can require known authority and a stable
@@ -495,6 +603,103 @@ export class HostReconciler {
this.hostWatchAuthority = fn
}
+ setHostMutationAuthority(fn: () => HostMutationAuthoritySnapshot): void {
+ this.hostMutationAuthority = fn
+ }
+
+ setResolveHostMutationDependencies(fn: (host: HostCRD) => readonly unknown[]): void {
+ this.resolveHostMutationDependencies = fn
+ }
+
+ /**
+ * A standalone HostReconciler receives an explicit Host object and has no
+ * watch-backed cache to fence. Once a live-cache resolver is wired (the
+ * production McpServerWatcher path), every mutation captures and revalidates
+ * Host LIST -> WATCH authority at its actual serializer admission boundary.
+ */
+ private captureHostMutationAuthority(): HostMutationAuthoritySnapshot | null {
+ return this.resolveCurrentHost === null ? null : this.hostMutationAuthority()
+ }
+
+ private requireHostMutationAuthority(
+ action: string,
+ captured: HostMutationAuthoritySnapshot | null
+ ): void {
+ if (captured === null) return
+ const current = this.hostMutationAuthority()
+ if (
+ !captured.known ||
+ !current.known ||
+ captured.hostGeneration !== current.hostGeneration ||
+ captured.contextGeneration !== current.contextGeneration
+ ) {
+ throw new HostInventoryAuthorityUnavailableError(action, captured, current)
+ }
+ }
+
+ /**
+ * Heartbeat writers own their serializer inside StatelessLifecycleExecutor.
+ * Capture authority here at dispatch, then let the returned closure re-check
+ * it and resolve the current same-UID Host when that serializer admits the
+ * work. This preserves one queue per Host without a nested-lock deadlock.
+ */
+ private prepareHostMutationAdmission(action: string, requested: HostCRD): () => HostCRD {
+ const capturedAuthority = this.captureHostMutationAuthority()
+ const capturedDependencies = this.resolveHostMutationDependencies?.(requested)
+ return this.makeHostMutationAdmission(
+ action,
+ requested,
+ capturedAuthority,
+ capturedDependencies
+ )
+ }
+
+ private makeHostMutationAdmission(
+ action: string,
+ requested: HostCRD,
+ capturedAuthority: HostMutationAuthoritySnapshot | null,
+ capturedDependencies: readonly unknown[] | undefined
+ ): () => HostCRD {
+ return () => {
+ this.requireHostMutationAuthority(action, capturedAuthority)
+ if (this.resolveCurrentHost === null) return requested
+ const current = this.resolveCurrentHost(requested.name)
+ if (
+ current === undefined ||
+ requested.uid === undefined ||
+ current.uid === undefined ||
+ requested.uid !== current.uid
+ ) {
+ throw new HostMutationIdentityChangedError(action, requested.name)
+ }
+ if (
+ requested.generation === undefined ||
+ current.generation === undefined ||
+ requested.generation !== current.generation
+ ) {
+ throw new HostMutationSpecRevisionChangedError(action, requested.name)
+ }
+ if (this.resolveHostMutationDependencies !== null) {
+ const currentDependencies = this.resolveHostMutationDependencies(current)
+ const sameDependencies =
+ capturedDependencies !== undefined &&
+ capturedDependencies.length === currentDependencies.length &&
+ capturedDependencies.every((value, index) =>
+ isDeepStrictEqual(value, currentDependencies[index])
+ )
+ if (!sameDependencies) {
+ throw new HostMutationDependencyChangedError(action, requested.name)
+ }
+ }
+ // McpServerWatcher owns the canonical LIST/WATCH cache object. Mutation
+ // bodies historically reflect status into their admitted object for
+ // same-pass decisions, so detach the working snapshot whenever a live
+ // cache resolver is wired. Standalone callers without a resolver retain
+ // the legacy caller-visible reflection contract.
+ return structuredClone(current)
+ }
+ }
+
/**
* Late-bound setter for the host-ref lookup used by
* `patchChannelReaderRevisionAnnotation` to discover Secret(s) referenced
@@ -2092,29 +2297,52 @@ export class HostReconciler {
* Host CRD is present).
*/
async patchChannelReaderRevisionAnnotation(hostName: string): Promise<void> {
+ const capturedAuthority = this.captureHostMutationAuthority()
const depName = `channel-reader-${hostName}`
try {
- const revision = await this.computeChannelReaderRevisionForHost(hostName)
- const patchBody = {
- spec: {
- template: {
- metadata: {
- annotations: { 'clerum.io/credentials-revision': revision },
+ await this.lifecycle.serializeByHost(hostName, async () => {
+ const action = `channel-reader "${hostName}" revision patch`
+ this.requireHostMutationAuthority(action, capturedAuthority)
+
+ const resolveCurrentHost = this.resolveCurrentHost
+ const admittedHost = resolveCurrentHost?.(hostName)
+ if (resolveCurrentHost !== null && admittedHost === undefined) return
+ const revalidate = admittedHost
+ ? this.makeHostMutationAdmission(
+ action,
+ admittedHost,
+ capturedAuthority,
+ this.resolveHostMutationDependencies?.(admittedHost)
+ )
+ : () => this.requireHostMutationAuthority(action, capturedAuthority)
+
+ revalidate()
+ const revision = await this.computeChannelReaderRevisionForHost(hostName)
+ // The revision calculation can await multiple Secret reads. Revalidate
+ // Host LIST/WATCH authority plus UID and generation immediately before
+ // entering the Kubernetes mutation, while still holding this Host lane.
+ revalidate()
+ const patchBody = {
+ spec: {
+ template: {
+ metadata: {
+ annotations: { 'clerum.io/credentials-revision': revision },
+ },
},
},
- },
- }
- await this.appsApi.patchNamespacedDeployment(
- { name: depName, namespace: config.channelsNamespace, body: patchBody },
- {
- middleware: [
- k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
- ],
}
- )
- console.log(
- `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`
- )
+ await this.appsApi.patchNamespacedDeployment(
+ { name: depName, namespace: config.channelsNamespace, body: patchBody },
+ {
+ middleware: [
+ k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
+ ],
+ }
+ )
+ console.log(
+ `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`
+ )
+ })
} catch (err) {
if (getErrorCode(err) === 404) return
console.error(`[HostReconciler] Failed to reconcile ${depName} revision:`, err)
@@ -3549,16 +3777,37 @@ export class HostReconciler {
async reconcile(host: HostCRD, source: HostReconcileSource = 'urgent'): Promise<void> {
const dispatchedAt = Date.now()
+ const capturedAuthority = this.captureHostMutationAuthority()
+ const capturedDependencies = this.resolveHostMutationDependencies?.(host)
return this.lifecycle.serializeByHost(host.name, async () => {
+ this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)
+ const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host
+ if (!admittedHost) return
+ const revalidate = this.makeHostMutationAdmission(
+ `Host "${host.name}" reconcile`,
+ admittedHost,
+ capturedAuthority,
+ capturedDependencies
+ )
const admittedAt = Date.now()
hostReconcileInFlight.inc({ lane: source })
try {
- await this.reconcileCore(host)
- this.enqueueReconcileOutcome(host)
+ await this.reconcileCore(admittedHost, revalidate)
+ this.enqueueReconcileOutcome(admittedHost)
this.observeReconcileLatency(source, 'success', dispatchedAt, admittedAt)
} catch (error) {
- this.enqueueControllerError(host, 'reconcile_exception', error)
- this.enqueueReconcileOutcome(host)
+ if (isBenignSupersessionError(error)) {
+ // A newer spec/uid/authority-generation superseded this pass mid-flight;
+ // a fresh reconcile runs after. Model it as a withdrawal (superseded),
+ // NOT a hard failure — do NOT emit controller_error/reconcile_exception
+ // or an administrative 'failed' outcome. Callers already treat the
+ // rethrow as a retire (reconcileDelete → 'superseded', watch callers
+ // only console.error), so the rethrow is preserved.
+ this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)
+ throw error
+ }
+ this.enqueueControllerError(admittedHost, 'reconcile_exception', error)
+ this.enqueueReconcileOutcome(admittedHost)
this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)
throw error
} finally {
@@ -3569,7 +3818,7 @@ export class HostReconciler {
private observeReconcileLatency(
source: HostReconcileSource,
- outcome: 'success' | 'error',
+ outcome: 'success' | 'error' | 'superseded',
dispatchedAt: number,
admittedAt: number
): void {
@@ -3583,7 +3832,19 @@ export class HostReconciler {
)
}
- private async reconcileCore(host: HostCRD): Promise<void> {
+ private async reconcileCore(host: HostCRD, revalidate?: () => HostCRD): Promise<void> {
+ // A per-Host serializer orders work, but it does not invalidate an older
+ // reconcile when the watch cache observes a newer spec while this body is
+ // awaiting Kubernetes or issuer I/O. Re-check the captured authority,
+ // identity, and spec generation at every material mutation boundary.
+ // Individual API writes remain atomic at the Kubernetes resource level;
+ // these checks prevent subsequent stale writes from spanning an observed
+ // generation change.
+ const revalidateHostMutationBoundary = (): void => {
+ revalidate?.()
+ }
+
+ revalidateHostMutationBoundary()
// Wake fast-path (Stage 4.3) BEFORE the heavy reconcile body: reconciles
// are serialized PER HOST (serializeByHost), so a pending wake for THIS
// Host must not wait behind token issuance, NetworkPolicies or the
@@ -3592,6 +3853,7 @@ export class HostReconciler {
// funnels through this same method, so a watch event dropped on disconnect
// is recovered here.
const forceFreshForWake = (await this.lifecycle.handleWakeFastPath(host)) === true
+ revalidateHostMutationBoundary()
// Track whether this host has desktop enabled
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
@@ -3602,11 +3864,18 @@ export class HostReconciler {
}
const secretResult = await this.validateHostSecret(host)
+ revalidateHostMutationBoundary()
if (!secretResult.ok) {
if (
secretResult.reason === 'SecretNotFound' ||
secretResult.reason === 'SecretAccessDenied'
) {
+ // This branch deletes every ephemeral runtime resource. The Secret
+ // decision may have blocked on the API while Host watch authority or
+ // the Host UID/spec generation changed. Revalidate at the destructive
+ // commit boundary so a stale secretRef can never tear down the current
+ // runtime; the recovered/current Host reconcile will decide again.
+ revalidateHostMutationBoundary()
await this.deleteHostRuntimeResources(host.name, host.namespace, {
deleteWorkspacePvc: false,
})
@@ -3627,8 +3896,11 @@ export class HostReconciler {
// Per-Host SA + Role + RoleBinding must exist BEFORE the Deployment,
// otherwise the kubelet can't mount the SA token and the pod will
// crash-loop until RBAC is created.
+ revalidateHostMutationBoundary()
await this.ensureHostServiceAccount(host)
+ revalidateHostMutationBoundary()
await this.ensureHostRole(host)
+ revalidateHostMutationBoundary()
await this.ensureHostRoleBinding(host)
let mounts: ResolvedSfsMount[] = []
@@ -3641,10 +3913,12 @@ export class HostReconciler {
// will inject them.
console.error(`[HostReconciler] Failed to resolve context mounts for "${host.name}":`, err)
}
+ revalidateHostMutationBoundary()
// Stateless lifecycle (Stage 2): assess enable/reject and persist the
// durable state to the Host status subresource BEFORE building the
// Deployment, so replicas derive from the same assessment.
let lifecycle = await this.lifecycle.assessLifecycle(host, mounts)
+ revalidateHostMutationBoundary()
// Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
// suspension IMMEDIATELY before replicas:0 derives from this assessment
// — see StatelessLifecycleExecutor.resolveWakeBeforeScaleDown. A null
@@ -3652,14 +3926,17 @@ export class HostReconciler {
// pass (the periodic resync retries) rather than scaling to 0 on stale
// data.
const guardedLifecycle = await this.lifecycle.resolveWakeBeforeScaleDown(host, lifecycle)
+ revalidateHostMutationBoundary()
if (guardedLifecycle === null) {
return
}
lifecycle = guardedLifecycle
+ revalidateHostMutationBoundary()
const lifecycleStatusCommitted = await this.lifecycle.writeLifecycleStatusToCluster(
host,
lifecycle
)
+ revalidateHostMutationBoundary()
if (lifecycleStatusCommitted) {
this.enqueueHostTelemetry(host, 'lifecycle_transition', lifecycle.effective.state, {
status: 'committed',
@@ -3670,7 +3947,9 @@ export class HostReconciler {
})
}
+ revalidateHostMutationBoundary()
await this.ensurePvc(host)
+ revalidateHostMutationBoundary()
await this.ensureService(host)
// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and
@@ -3686,23 +3965,31 @@ export class HostReconciler {
// surface it via HostRuntimeStatus instead of swallowing it.
const npFailures: string[] = []
try {
+ revalidateHostMutationBoundary()
await this.ensureMcpHostIngressNetworkPolicy(host)
+ revalidateHostMutationBoundary()
await this.ensureMcpHostGfsEgressNetworkPolicy(host)
+ revalidateHostMutationBoundary()
await this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure mcp-host NP for "${host.name}":`, err)
npFailures.push(`mcp-host NP: ${(err as Error).message}`)
}
try {
+ revalidateHostMutationBoundary()
await this.ensureRpcProxyMcpHostIngressNetworkPolicy(host)
+ revalidateHostMutationBoundary()
await this.ensureRpcProxyHostEgressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure rpc-proxy host NP for "${host.name}":`, err)
npFailures.push(`rpc-proxy NP: ${(err as Error).message}`)
}
+ revalidateHostMutationBoundary()
await this.ensureDesktopNetworkPolicy(host)
try {
+ revalidateHostMutationBoundary()
await this.ensureChannelReaderEgressNetworkPolicy(host)
+ revalidateHostMutationBoundary()
await this.ensureWorkflowApprovalReaderHostEgressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure channels egress NP for "${host.name}":`, err)
@@ -3717,8 +4004,10 @@ export class HostReconciler {
)
if (bootstrapLifecycle !== lifecycle) {
lifecycle = bootstrapLifecycle
+ revalidateHostMutationBoundary()
await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
}
+ revalidateHostMutationBoundary()
// Bootstrap captures the scope contract used for issuance. The Deployment
// guard below compares that contract with the live channel cache after this
@@ -3727,23 +4016,28 @@ export class HostReconciler {
forceFreshForWake,
targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
+ revalidateHostMutationBoundary()
// replaceWithConflictRetry may wait and re-read after a 409, so each body is
// rebuilt from a stable lifecycle plus token-scope pair before mutation.
const resolveDeploymentLifecycle = async (): Promise<EffectiveHostLifecycle> => {
+ revalidateHostMutationBoundary()
const deploymentLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(
host.name,
lifecycle
)
if (deploymentLifecycle !== lifecycle) {
lifecycle = deploymentLifecycle
+ revalidateHostMutationBoundary()
await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
}
+ revalidateHostMutationBoundary()
return lifecycle.effective
}
const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {
for (let attempt = 1; attempt <= 3; attempt++) {
+ revalidateHostMutationBoundary()
const currentScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
this.hasChannelIngress(host)
@@ -3755,6 +4049,7 @@ export class HostReconciler {
targetSuspended:
lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
+ revalidateHostMutationBoundary()
const postProvisionScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
this.hasChannelIngress(host)
@@ -3774,6 +4069,7 @@ export class HostReconciler {
const resolveDeploymentState = async (): Promise<DeploymentMutationState> => {
for (let attempt = 1; attempt <= 3; attempt++) {
+ revalidateHostMutationBoundary()
await resolveDeploymentLifecycle()
await ensureCurrentRuntimeTokenScope()
const effective = await resolveDeploymentLifecycle()
@@ -3782,6 +4078,7 @@ export class HostReconciler {
this.hasChannelIngress(host)
)
if (runtimeTokenProvision.scopeHash === currentScopeHash) {
+ revalidateHostMutationBoundary()
return {
lifecycle: effective,
runtimeTokenRevision: runtimeTokenProvision.revision,
@@ -3798,6 +4095,7 @@ export class HostReconciler {
)
}
+ revalidateHostMutationBoundary()
await this.ensureDeployment(
host,
mounts,
@@ -3805,6 +4103,7 @@ export class HostReconciler {
lifecycle.effective,
resolveDeploymentState
)
+ revalidateHostMutationBoundary()
const suspended = lifecycle.effective.stateless && lifecycle.effective.state === 'suspended'
if (suspended) {
@@ -3813,6 +4112,7 @@ export class HostReconciler {
this.lifecycle.markHostNotSuspended(host.name)
}
const ready = suspended ? false : await this.checkDeploymentReady(host.name, host.namespace)
+ revalidateHostMutationBoundary()
// If any per-Host NP failed to apply, mark the host as degraded:
// deployed remains true (the pod is in place), but ready=false +
// a message that names the missing security boundary. Operators see
@@ -3848,6 +4148,7 @@ export class HostReconciler {
() => this.reconcileChannelReaderDeployment(host),
]) {
try {
+ revalidateHostMutationBoundary()
await reconcileResource()
} catch (err) {
channelReaderFailures.push(err)
@@ -3868,6 +4169,7 @@ export class HostReconciler {
// Best-effort — checkChannelReaderStatus never throws.
const ccCount = this.countCommunicationChannels(host.name)
const channelReaderStatus = await this.checkChannelReaderStatus(host.name, ccCount)
+ revalidateHostMutationBoundary()
if (channelReaderFailures.length > 0) {
// Overlay the reconcile error onto the status message so both signals
// are visible (the deploy error may explain why the Deployment is missing).
@@ -3902,18 +4204,21 @@ export class HostReconciler {
* that path, so cache presence at admission is the correct fence. This
* mirrors the F2/#827 in-serializer re-check in collectHostCleanupFailures.
*
- * Callers that omit `opts` keep the previous unconditional behavior exactly.
+ * Callers that omit `opts` drop only the extra `skipIf` term; the
+ * resolveCurrentHost supersede fence still gates the delete.
*/
async reconcileDelete(
name: string,
namespace: string,
opts?: { skipIf?: () => boolean }
): Promise<void> {
+ const capturedAuthority = this.captureHostMutationAuthority()
// A DELETED event must not race an in-flight create/update for the same
// Host: route it through the same per-Host serializer as reconcile(). The
// global convergence tail no longer provides this ordering.
await this.lifecycle.serializeByHost(name, async () => {
- if (opts?.skipIf?.()) {
+ this.requireHostMutationAuthority(`Host "${name}" delete`, capturedAuthority)
+ if (this.resolveCurrentHost?.(name) !== undefined || opts?.skipIf?.()) {
// Same vocabulary the watch path already reports for this condition
// (reconcileHostWatchEvent), so the skip is observable, not silent.
hostDeleteCleanupTotal.inc({ outcome: 'superseded' })
@@ -4060,6 +4365,15 @@ export class HostReconciler {
}
try {
await this.lifecycle.serializeByHost(hostName, async () => {
+ // The fresh 404 and the cache omission are only destructive
+ // authority while the same LIST -> WATCH generation remains live.
+ // Revalidate inside the serializer: queued Host work can otherwise
+ // outlive that generation and delete a same-name recreation whose
+ // ADDED event has not reached the retired cache.
+ if (!authorityValid()) {
+ hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
+ return
+ }
// F2 / #827 TOCTOU: a same-name Host recreated between the fresh read
// above and admission into this serializer updates the current cache
// (its ADDED event sets the cache before its reconcile runs, and that
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 2aa0ccea4..e45095c41 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -1,13 +1,19 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import * as http from 'http'
import { HostFleetReconcileError } from './hostReconciler'
import { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'
+import { externalEgressResyncDelayMs } from './externalEgressConvergenceCoordinator'
import {
McpServerWatcher,
- externalEgressResyncDelayMs,
+ getContext,
listAllCommunicationChannels,
+ listAllGlobalFileSystems,
listAllHosts,
+ listAllSharedFileSystems,
} from './k8sClient'
-import type { HostCRD } from './types'
+import { registry } from './metrics'
+import { ContextMapperServer } from './server'
+import type { HostCRD, McpServerCRD } from './types'
function deferred<T = void>(): {
promise: Promise<T>
@@ -23,19 +29,116 @@ function deferred<T = void>(): {
return { promise, resolve, reject }
}
+async function flushMicrotasks(turns = 8): Promise<void> {
+ for (let turn = 0; turn < turns; turn += 1) {
+ await Promise.resolve()
+ }
+}
+
+async function readInitialConvergenceMetric(
+ name:
+ | 'clerum_hcc_initial_convergence_retries_total'
+ | 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ lane: 'McpServer' | 'NetworkPolicy'
+): Promise<number> {
+ const metric = registry.getSingleMetric(name)
+ if (!metric) throw new Error(`${name} is not registered`)
+ const snapshot = await metric.get()
+ return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0
+}
+
+async function requestReadyOverHttp(server: ContextMapperServer): Promise<{
+ statusCode: number | undefined
+ body: string
+}> {
+ const listener = (server as unknown as { server: http.Server | null }).server
+ const address = listener?.address()
+ if (!address || typeof address === 'string') {
+ throw new Error('HCC readiness listener did not bind a TCP port')
+ }
+
+ return new Promise((resolve, reject) => {
+ const request = http.get(
+ { host: '127.0.0.1', port: address.port, path: '/ready' },
+ response => {
+ let body = ''
+ response.setEncoding('utf8')
+ response.on('data', chunk => {
+ body += chunk
+ })
+ response.on('end', () => resolve({ statusCode: response.statusCode, body }))
+ }
+ )
+ request.once('error', reject)
+ })
+}
+
+function stubAuthoritativeInventoryWatch(
+ watcher: McpServerWatcher,
+ kind: 'McpServer' | 'Context',
+ effect?: () => void | Promise<void>
+) {
+ const method = kind === 'McpServer' ? 'startMcpServerWatch' : 'startContextWatch'
+ const generationField = kind === 'McpServer' ? 'mcpWatchGeneration' : 'contextWatchGeneration'
+ const requestField = kind === 'McpServer' ? 'mcpWatchRequest' : 'ctxWatchRequest'
+ return vi.spyOn(watcher as any, method).mockImplementation(async () => {
+ const generation = ((watcher as any)[generationField] as number) + 1
+ ;(watcher as any)[generationField] = generation
+ ;(watcher as any)[requestField] = { abort: vi.fn() }
+ await effect?.()
+ return generation
+ })
+}
+
+function markHostInventoryAuthoritative(watcher: McpServerWatcher): void {
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+}
+
+function markMcpServerInventoryAuthoritative(watcher: McpServerWatcher): void {
+ ;(watcher as any).mcpServerCacheSynced = true
+}
+
+function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): void {
+ ;(watcher as any).networkPolicyRevocationContextGeneration = (
+ watcher as any
+ ).contextWatchGeneration
+ ;(watcher as any).networkPolicyRevocationServerGeneration = (watcher as any).mcpWatchGeneration
+ ;(watcher as any).networkPolicyRevocationContextRevision = (watcher as any).contextDesiredRevision
+ ;(watcher as any).networkPolicyRevocationServerRevision = (
+ watcher as any
+ ).mcpServerDesiredRevision
+}
+
+function newContextAuthoritativeWatcher(): McpServerWatcher {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ return watcher
+}
+
const mocks = vi.hoisted(() => {
const listNamespacedCustomObject = vi.fn()
+ const getNamespacedCustomObject = vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))
const ensureDefaultPolicies = vi.fn().mockResolvedValue(undefined)
- const netPolFullReconcile = vi.fn().mockResolvedValue(undefined)
+ const hasCertifiedSafetyInventory = vi.fn().mockReturnValue(true)
+ const netPolFullReconcile = vi.fn().mockImplementation(async (...args: unknown[]) => {
+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
+ options?.onAuthoritativeRevocationComplete?.()
+ })
const serverFullReconcile = vi.fn().mockResolvedValue(undefined)
const hostFullReconcile = vi.fn().mockResolvedValue(undefined)
const hostReconcileHosts = vi.fn().mockResolvedValue(undefined)
const sfsFullReconcile = vi.fn().mockResolvedValue(undefined)
const watch = vi.fn().mockResolvedValue({ abort: vi.fn() })
const hostListCallOptions = vi.fn()
+ const sharedFileSystemListCallOptions = vi.fn()
const createAdministrativeOutcomeReporter = vi.fn().mockReturnValue(undefined)
return {
+ hasCertifiedSafetyInventory,
listNamespacedCustomObject,
+ getNamespacedCustomObject,
ensureDefaultPolicies,
netPolFullReconcile,
serverFullReconcile,
@@ -44,12 +147,15 @@ const mocks = vi.hoisted(() => {
sfsFullReconcile,
watch,
hostListCallOptions,
+ sharedFileSystemListCallOptions,
createAdministrativeOutcomeReporter,
}
})
-// Mutable so a test can enable the external-egress resync timer (R1-M4); every
-// other test keeps externalEgressResyncIntervalSec: 0 (reset in afterEach).
+// Kept at externalEgressResyncIntervalSec: 0 in every test (reset in afterEach):
+// the external-egress periodic resync timer now lives in the convergence
+// coordinator and is unit-tested in externalEgressConvergenceCoordinator.test.ts,
+// so the k8sClient integration tests leave it disabled to avoid a stray timer.
const mockConfig = vi.hoisted(() => ({
devMode: false,
namespace: 'mcp-server',
@@ -90,9 +196,12 @@ vi.mock('@kubernetes/client-node', () => {
return {
listNamespacedCustomObject: async (request: { plural?: string }, options?: unknown) => {
if (request.plural === 'hosts') mocks.hostListCallOptions(options)
+ if (request.plural === 'sharedfilesystems')
+ mocks.sharedFileSystemListCallOptions(options)
const response = await mocks.listNamespacedCustomObject(request)
if (
- request.plural === 'hosts' &&
+ request.plural &&
+ ['hosts', 'mcpservers', 'contexts'].includes(request.plural) &&
response &&
typeof response === 'object' &&
!('metadata' in response)
@@ -102,11 +211,17 @@ vi.mock('@kubernetes/client-node', () => {
// production missing-resourceVersion failure path.
return {
...response,
- metadata: { resourceVersion: 'test-host-collection-rv' },
+ metadata: {
+ resourceVersion:
+ request.plural === 'hosts'
+ ? 'test-host-collection-rv'
+ : `test-${request.plural}-collection-rv`,
+ },
}
}
return response
},
+ getNamespacedCustomObject: mocks.getNamespacedCustomObject,
}
}
return {}
@@ -131,6 +246,8 @@ vi.mock('./reconciler', () => ({
reconcile = vi.fn()
reconcileDelete = vi.fn()
getStatus = vi.fn()
+ setInventoryAuthority = vi.fn()
+ setResolveCurrentServer = vi.fn()
},
}))
@@ -153,6 +270,28 @@ vi.mock('./hostReconciler', () => ({
// and the watch-authority snapshot into HostReconciler at construction.
setResolveCurrentHost = vi.fn()
setHostWatchAuthority = vi.fn()
+ setHostMutationAuthority = vi.fn()
+ // H2: capture the injected uid-guarded cache reflector so Test C can invoke it.
+ _reflectHostOutcomeFn:
+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
+ | null = null
+ setReflectHostOutcome = vi.fn().mockImplementation(function (
+ this: {
+ _reflectHostOutcomeFn:
+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
+ | null
+ },
+ fn: (name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void
+ ) {
+ this._reflectHostOutcomeFn = fn
+ })
+ _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null = null
+ setResolveHostMutationDependencies = vi.fn().mockImplementation(function (
+ this: { _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null },
+ fn: (host: HostCRD) => readonly unknown[]
+ ) {
+ this._hostMutationDependenciesFn = fn
+ })
// Stores the injected callback so tests can invoke it directly.
_countCommunicationChannelsFn: ((host: string) => number) | null = null
setCountCommunicationChannels = vi.fn().mockImplementation(function (
@@ -172,16 +311,26 @@ vi.mock('./hostReconciler', () => ({
// B2: mock the CC cache-sync setter so McpServerWatcher constructor succeeds.
setIsCommunicationChannelCacheSynced = vi.fn()
patchChannelReaderRevisionAnnotation = vi.fn().mockResolvedValue(undefined)
+ reconcileChannelReaderRevision = vi.fn().mockResolvedValue(undefined)
},
}))
vi.mock('./networkPolicyReconciler', () => ({
+ sameContextDesiredRevision: (
+ expected: { name: string; namespace: string; spec: unknown },
+ current: { name: string; namespace: string; spec: unknown }
+ ) =>
+ expected.name === current.name &&
+ expected.namespace === current.namespace &&
+ JSON.stringify(expected.spec) === JSON.stringify(current.spec),
NetworkPolicyReconciler: class {
ensureDefaultPolicies = mocks.ensureDefaultPolicies
fullReconcile = mocks.netPolFullReconcile
reconcileExternalEgress = vi.fn()
reconcileContext = vi.fn()
+ reconcileDeleteContext = vi.fn()
cleanupExternalEgress = vi.fn()
+ hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory
},
}))
@@ -199,6 +348,7 @@ vi.mock('./sharedFileSystemReconciler', () => ({
reconcileDelete = vi.fn()
getStatus = vi.fn()
setListContextRefs = vi.fn()
+ isMountable = vi.fn().mockReturnValue(false)
},
}))
@@ -213,7 +363,7 @@ describe('McpServerWatcher startup', () => {
mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'mcpservers') return { items: [] }
- if (plural === 'contexts') throw new Error('context discovery failed')
+ if (plural === 'contexts') return { items: [] }
if (plural === 'hosts') return { items: [] }
if (plural === 'sharedfilesystems') return { items: [] }
if (plural === 'communicationchannels')
@@ -224,60 +374,24 @@ describe('McpServerWatcher startup', () => {
afterEach(() => {
vi.useRealTimers()
- mockConfig.externalEgressResyncIntervalSec = 0 // R1-M4: keep the timer off by default
+ vi.restoreAllMocks()
+ // Certified is the resting state. vi.clearAllMocks() in beforeEach clears
+ // calls but not implementations, so a test that degrades this getter and
+ // fails before its own restore would silently re-gate readiness for every
+ // later test in the file. Restore it structurally rather than by
+ // per-test discipline.
+ mocks.hasCertifiedSafetyInventory.mockReset().mockReturnValue(true)
+ mockConfig.externalEgressResyncIntervalSec = 0 // keep periodic resync off by default
mocks.ensureDefaultPolicies.mockReset().mockResolvedValue(undefined)
- mocks.netPolFullReconcile.mockReset().mockResolvedValue(undefined)
+ mocks.netPolFullReconcile.mockReset().mockImplementation(async (...args: unknown[]) => {
+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
+ options?.onAuthoritativeRevocationComplete?.()
+ })
mocks.serverFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.sfsFullReconcile.mockReset().mockResolvedValue(undefined)
})
- // R1-M4 (zach88): the self-rescheduling resync timer (only the pure delay
- // helper was covered). Assert the behavioral invariants the comments promise:
- // arms once, reschedules only AFTER a pass resolves, and stop() ends the chain.
- it('R1-M4: external-egress resync timer arms once, reschedules after each pass, and stops', async () => {
- vi.useFakeTimers()
- mockConfig.externalEgressResyncIntervalSec = 60
- const watcher = new McpServerWatcher()
- for (const m of [
- 'startMcpServerWatch',
- 'startContextWatch',
- 'startSharedFileSystemWatch',
- 'startGlobalFileSystemWatch',
- 'startCommunicationChannelWatch',
- ]) {
- vi.spyOn(watcher as any, m).mockImplementation(async () => undefined)
- }
- // Control each resync pass's completion so we can prove the reschedule waits.
- let pending = deferred()
- const runSpy = vi
- .spyOn(watcher as any, 'runExternalEgressResync')
- .mockImplementation(() => pending.promise)
-
- await watcher.start()
- expect(runSpy).not.toHaveBeenCalled() // not until the first delay elapses
-
- await vi.advanceTimersByTimeAsync(60_000)
- expect(runSpy).toHaveBeenCalledTimes(1) // armed once, fired once
-
- // The NEXT pass must not be scheduled until this one resolves.
- await vi.advanceTimersByTimeAsync(60_000)
- expect(runSpy).toHaveBeenCalledTimes(1) // still 1 — no overlap while pending
-
- pending.resolve()
- await Promise.resolve()
- pending = deferred()
- await vi.advanceTimersByTimeAsync(60_000)
- expect(runSpy).toHaveBeenCalledTimes(2) // rescheduled after the pass resolved
-
- // stop() ends the chain — no further fires.
- watcher.stop()
- pending.resolve()
- await Promise.resolve()
- await vi.advanceTimersByTimeAsync(60_000 * 5)
- expect(runSpy).toHaveBeenCalledTimes(2)
- })
-
it('passes the governed tracing switch to the administrative reporter factory', () => {
new McpServerWatcher()
@@ -286,35 +400,83 @@ describe('McpServerWatcher startup', () => {
})
})
- it('ensures default policies even when initial Context discovery fails', async () => {
+ it('ensures baseline policies before opening any inventory watch', async () => {
const watcher = new McpServerWatcher()
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
- // #281 (Task 9): startCommunicationChannelWatch is wired into start() — stub to prevent the real watch from firing.
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
- async () => undefined
- )
await watcher.start()
expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(1)
- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])
+ expect(mocks.ensureDefaultPolicies.mock.invocationCallOrder[0]).toBeLessThan(
+ mocks.watch.mock.invocationCallOrder[0]
+ )
+ await vi.waitFor(() =>
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {
+ runEffect: expect.any(Function),
+ })
+ )
expect(mocks.hostFullReconcile).toHaveBeenCalledWith([])
expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([])
+ await watcher.stop()
})
- it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {
+ it('loads the SharedFileSystem inventory before scheduling the first Host fleet pass', async () => {
+ const sharedFileSystemsListed = deferred<{ items: unknown[] }>()
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return sharedFileSystemsListed.promise
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'sfs-ordering-cc-rv' }, items: [] }
+ }
return { items: [] }
})
+ const watcher = new McpServerWatcher()
+ // Sampling "not yet called" after a fixed number of microtask turns cannot
+ // distinguish "correctly waiting" from "scheduled but not drained yet".
+ // Record what the first pass actually observed instead: the invariant is
+ // that it never runs against an unpopulated SharedFileSystem cache, because
+ // that is what writes mount-less templates and rerolls the fleet.
+ let sfsCacheSizeAtFirstPass: number | undefined
+ mocks.hostFullReconcile.mockImplementation(async () => {
+ if (sfsCacheSizeAtFirstPass === undefined) {
+ sfsCacheSizeAtFirstPass = (watcher as any).sharedFileSystems.size
+ }
+ })
+ const start = watcher.start()
- await expect(listAllHosts()).resolves.toEqual([])
+ try {
+ // Await the whole startup first. That is the positive signal that the
+ // cold-start Host recovery already reached the point where it decides
+ // whether to schedule the fleet pass, so the absence assertion below
+ // cannot pass merely because nothing has been drained yet.
+ await start
+ await flushMicrotasks()
+ expect(mocks.hostFullReconcile).not.toHaveBeenCalled()
- expect(mocks.hostListCallOptions).toHaveBeenCalledWith(
+ sharedFileSystemsListed.resolve({
+ items: [{ metadata: { name: 'initial-sfs', namespace: 'mcp-host' }, spec: {} }],
+ })
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
+ expect(sfsCacheSizeAtFirstPass).toBe(1)
+ expect((watcher as any).sharedFileSystems.get('initial-sfs')).toEqual(
+ expect.objectContaining({ name: 'initial-sfs' })
+ )
+ } finally {
+ sharedFileSystemsListed.resolve({ items: [] })
+ await start.catch(() => undefined)
+ await watcher.stop()
+ }
+ })
+
+ it('lists the SharedFileSystem inventory through the deadline-bearing client', async () => {
+ // The cold-start Host fleet pass waits on this inventory. Bounding the wait
+ // is not enough: an unbounded request stays hung, so the socket leaks and
+ // `startSharedFileSystemWatch()` — which runs after this await — never
+ // starts. Only the deadline-bearing client aborts the request itself, and
+ // it is observable because it appends its deadline middleware to the call.
+ mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })
+
+ await listAllSharedFileSystems()
+
+ expect(mocks.sharedFileSystemListCallOptions).toHaveBeenCalledWith(
expect.objectContaining({
middleware: [expect.objectContaining({ pre: expect.any(Function) })],
middlewareMergeStrategy: 'append',
@@ -322,125 +484,4142 @@ describe('McpServerWatcher startup', () => {
)
})
- it('preserves Host UID from the Kubernetes inventory boundary', async () => {
+ it('schedules the first Host fleet pass when the SharedFileSystem watch never starts', async () => {
+ // `k8s.Watch` carries no transport deadline — client-node overwrites its own
+ // AbortSignal.timeout with a bare controller signal — so awaiting the watch
+ // start inside the inventory promise re-strands the cold-start fleet pass
+ // that bounding the LIST was meant to protect.
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'sfs-watch-stall-cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ mocks.watch.mockImplementation(async (path: string) => {
+ if (path.endsWith('/sharedfilesystems')) return new Promise(() => undefined)
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const start = watcher.start()
+
+ try {
+ await start
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
+ } finally {
+ await watcher.stop()
+ await start.catch(() => undefined)
+ }
+ })
+
+ it('schedules the first Host fleet pass when the SharedFileSystem inventory fails', async () => {
+ // The inventory LIST runs on the deadline-bearing client, so an apiserver
+ // that never answers is aborted and surfaces here as a rejection. The Host
+ // fleet must still converge: stranding it would leave every Host
+ // unreconciled behind an already-certified readiness.
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'sharedfilesystems') throw new Error('sharedfilesystems LIST aborted')
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'sfs-abort-cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ const start = watcher.start()
+
+ try {
+ await start
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
+ } finally {
+ await watcher.stop()
+ await start.catch(() => undefined)
+ }
+ })
+
+ it('retains Context uid and generation from the authoritative initial snapshot', async () => {
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
return {
items: [
{
metadata: {
- name: 'uid-host',
- namespace: 'mcp-host',
- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
- generation: 4,
+ name: 'identity-context',
+ namespace: 'mcp-server',
+ uid: 'context-uid-1',
+ generation: 7,
},
- spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },
+ spec: { contextId: 'identity-context', mcpServers: [] },
},
],
}
}
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'context-identity-cc-rv' }, items: [] }
+ }
return { items: [] }
})
+ const watcher = new McpServerWatcher()
- await expect(listAllHosts()).resolves.toEqual([
- expect.objectContaining({
- name: 'uid-host',
- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
- generation: 4,
- }),
- ])
+ try {
+ await watcher.start()
+ expect((watcher as any).contexts.get('identity-context')).toEqual(
+ expect.objectContaining({ uid: 'context-uid-1', generation: 7 })
+ )
+ } finally {
+ await watcher.stop()
+ }
})
- it('reconciles startup external egress before runtime full reconciliation', async () => {
- const eventLog: string[] = []
- const server = {
- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
- spec: {
- contextRef: 'wf-research',
- image: 'clerum/web-search:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
- },
+ it('fails safe bootstrap and keeps readiness unavailable when baseline NetworkPolicies fail', async () => {
+ const policyError = new Error('baseline NetworkPolicies unavailable')
+ mocks.ensureDefaultPolicies.mockRejectedValueOnce(policyError)
+ const watcher = new McpServerWatcher()
+ const server = new ContextMapperServer(watcher, 0)
+ await server.start()
+ const bootstrap = watcher.start().then(() => server.setReady(true))
+
+ try {
+ await expect(bootstrap).rejects.toThrow(policyError)
+ const ready = await requestReadyOverHttp(server)
+ expect(ready.statusCode).toBe(503)
+ expect(JSON.parse(ready.body)).toEqual({ status: 'starting', ready: false })
+ expect(mocks.watch).not.toHaveBeenCalled()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ } finally {
+ await server.stop()
+ await watcher.stop()
+ }
+ })
+
+ it.each(['LIST', 'WATCH'] as const)(
+ 'stays live and recovers readiness after a transient Host %s bootstrap failure',
+ async failureBoundary => {
+ vi.useFakeTimers()
+ let hostListAttempts = 0
+ let hostWatchAttempts = 0
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListAttempts += 1
+ if (failureBoundary === 'LIST' && hostListAttempts === 1) {
+ throw new Error('host discovery temporarily unavailable')
+ }
+ return { metadata: { resourceVersion: 'host-recovery-rv' }, items: [] }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ }
+ )
+ mocks.watch.mockImplementation(async path => {
+ if (path.endsWith('/hosts')) {
+ hostWatchAttempts += 1
+ if (failureBoundary === 'WATCH' && hostWatchAttempts === 1) {
+ throw new Error('host watch temporarily unavailable')
+ }
+ }
+ return { abort: vi.fn() }
+ })
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+ server.setReady(true)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
+
+ expect(hostListAttempts).toBe(2)
+ expect(hostWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)
+ expect((watcher as any).ccAppliedLifecycleGeneration).toBe(1)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+ } finally {
+ await server.stop()
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ }
+ )
+
+ it.each(['LIST', 'WATCH'] as const)(
+ 'stays live and recovers readiness after a transient McpServer %s bootstrap failure',
+ async failureBoundary => {
+ vi.useFakeTimers()
+ let mcpServerListAttempts = 0
+ let mcpServerWatchAttempts = 0
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') {
+ mcpServerListAttempts += 1
+ if (failureBoundary === 'LIST' && mcpServerListAttempts === 1) {
+ throw new Error('mcpserver discovery temporarily unavailable')
+ }
+ return { metadata: { resourceVersion: 'mcpserver-recovery-rv' }, items: [] }
+ }
+ if (plural === 'hosts') {
+ return { metadata: { resourceVersion: 'host-rv' }, items: [] }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ }
+ )
+ mocks.watch.mockImplementation(async path => {
+ if (path.endsWith('/mcpservers')) {
+ mcpServerWatchAttempts += 1
+ if (failureBoundary === 'WATCH' && mcpServerWatchAttempts === 1) {
+ throw new Error('mcpserver watch temporarily unavailable')
+ }
+ }
+ return { abort: vi.fn() }
+ })
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ // The McpServer lane runs for real (LIST + WATCH) so its bootstrap
+ // failure and recovery exercise production code. Context stays
+ // authoritative so readiness can be promoted once the lane recovers.
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+ server.setReady(true)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ expect((watcher as any).mcpServerCacheRecoveryTimer).not.toBeNull()
+
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
+
+ expect(mcpServerListAttempts).toBe(2)
+ expect(mcpServerWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+ } finally {
+ await server.stop()
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ }
+ )
+
+ it.each(['LIST', 'WATCH'] as const)(
+ 'stays live and recovers readiness after a transient Context %s bootstrap failure',
+ async failureBoundary => {
+ vi.useFakeTimers()
+ let contextListAttempts = 0
+ let contextWatchAttempts = 0
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ contextListAttempts += 1
+ if (failureBoundary === 'LIST' && contextListAttempts === 1) {
+ throw new Error('context discovery temporarily unavailable')
+ }
+ return { metadata: { resourceVersion: 'context-recovery-rv' }, items: [] }
+ }
+ if (plural === 'hosts') {
+ return { metadata: { resourceVersion: 'host-rv' }, items: [] }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ }
+ )
+ mocks.watch.mockImplementation(async path => {
+ if (path.endsWith('/contexts')) {
+ contextWatchAttempts += 1
+ if (failureBoundary === 'WATCH' && contextWatchAttempts === 1) {
+ throw new Error('context watch temporarily unavailable')
+ }
+ }
+ return { abort: vi.fn() }
+ })
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ // The Context lane runs for real (LIST + WATCH) so its bootstrap
+ // failure and recovery exercise production code. McpServer stays
+ // authoritative so readiness can be promoted once the lane recovers.
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+ server.setReady(true)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ expect((watcher as any).contextCacheRecoveryTimer).not.toBeNull()
+
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
+
+ expect(contextListAttempts).toBe(2)
+ expect(contextWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+ } finally {
+ await server.stop()
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
}
+ )
+ it('cancels a pending Host bootstrap recovery when the watcher stops', async () => {
+ vi.useFakeTimers()
+ let hostListAttempts = 0
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [server] }
- if (plural === 'contexts') return { items: [] }
- if (plural === 'hosts') return { items: [] }
- if (plural === 'sharedfilesystems') return { items: [] }
- if (plural === 'communicationchannels')
- return { metadata: { resourceVersion: '1' }, items: [] }
+ if (plural === 'hosts') {
+ hostListAttempts += 1
+ throw new Error('host inventory remains unavailable')
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
return { items: [] }
})
- mocks.ensureDefaultPolicies.mockImplementation(async () => {
- eventLog.push('defaults')
- })
- mocks.serverFullReconcile.mockImplementation(async () => {
- eventLog.push('runtime')
- })
-
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockImplementation(async () => {
- eventLog.push('egress')
- })
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
- async () => undefined
- )
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- await watcher.start()
+ try {
+ await watcher.start()
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
- expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([
- expect.objectContaining({ name: 'web-search' }),
- ])
+ await watcher.stop()
+ await vi.advanceTimersByTimeAsync(5000)
- watcher.stop()
+ expect(hostListAttempts).toBe(1)
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
+ } finally {
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
})
- it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {
+ it('defers every Host-dependent effect while a bootstrap LIST lacks WATCH authority', async () => {
vi.useFakeTimers()
- const server = {
- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
+ const staleHost = {
+ metadata: {
+ name: 'stale-bootstrap-host',
+ namespace: 'mcp-host',
+ generation: 1,
+ uid: 'stale-bootstrap-host-uid',
+ },
spec: {
- contextRef: 'wf-research',
- image: 'clerum/web-search:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
+ host: 'stale-bootstrap-host',
+ contextRef: 'bootstrap-context',
+ secretRef: 'bootstrap-secret',
+ model: { provider: 'openai', name: 'gpt-4o-mini' },
},
}
-
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [server] }
- if (plural === 'contexts') return { items: [] }
- if (plural === 'hosts') return { items: [] }
- if (plural === 'sharedfilesystems') return { items: [] }
- if (plural === 'communicationchannels')
- return { metadata: { resourceVersion: '1' }, items: [] }
+ if (plural === 'hosts') {
+ return {
+ metadata: { resourceVersion: 'stale-bootstrap-rv' },
+ items: [staleHost],
+ }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
return { items: [] }
})
-
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('dns resolution failed'))
+ mocks.watch.mockImplementation(async path => {
+ if (path.endsWith('/hosts')) {
+ throw new Error('host watch remains unavailable')
+ }
+ return { abort: vi.fn() }
+ })
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
- async () => undefined
- )
-
- await watcher.start()
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const hostReconciler = (watcher as any).hostReconciler
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ try {
+ await watcher.start()
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ expect(watcher.getHost('stale-bootstrap-host')).toBeUndefined()
+ const ccWatchCallback = (watcher as any).getCommunicationChannelWatchCallback()
+ await ccWatchCallback('ADDED', {
+ metadata: { name: 'bootstrap-channel', namespace: 'channels' },
+ spec: { hostRef: 'stale-bootstrap-host' },
+ })
+ ;(watcher as any).contexts.set('bootstrap-context', {
+ name: 'bootstrap-context',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'bootstrap-context',
+ sharedFileSystems: [{ name: 'bootstrap-sfs', mountPath: '/shared' }],
+ },
+ })
+ await (watcher as any).reconcileHostsReferencingSfs('bootstrap-sfs')
+ await (watcher as any).reconcileHostsReferencingContext('bootstrap-context')
+ ;(watcher as any).ccCacheSynced = false
+ await (watcher as any).reconcileChannelReaderRevision('bootstrap-channel-secret', 'channels')
+
+ expect(hostReconciler.reconcile).not.toHaveBeenCalled()
+ expect(hostReconciler.patchChannelReaderRevisionAnnotation).not.toHaveBeenCalled()
+ expect(hostReconciler.reconcileChannelReaderRevision).not.toHaveBeenCalled()
+ expect(warnSpy).toHaveBeenCalledWith(
+ '[K8s] Deferring CommunicationChannel Secret "channels/bootstrap-channel-secret" Host convergence; CommunicationChannel cache is not authoritative'
+ )
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+ } finally {
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ })
+
+ it('defers Host fleet convergence until both Host and Context inventories are authoritative', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = false
+ ;(watcher as any).hosts.set('context-dependent-host', {
+ name: 'context-dependent-host',
+ namespace: 'mcp-host',
+ spec: {
+ host: 'context-dependent-host',
+ contextRef: 'current-context',
+ secretRef: 'host-secret',
+ },
+ })
+
+ await (watcher as any).performHostFleetReconcileOnce({
+ reason: 'Context authority test',
+ mode: 'full',
+ })
+
+ expect(mocks.hostFullReconcile).not.toHaveBeenCalled()
+ ;(watcher as any).contextCacheSynced = true
+ await (watcher as any).performHostFleetReconcileOnce({
+ reason: 'Context authority recovered',
+ mode: 'full',
+ })
+
+ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
+ expect(mocks.hostFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'context-dependent-host' }),
+ ])
+ await watcher.stop()
+ })
+
+ it('applies a pending CommunicationChannel lifecycle generation during Context recovery', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = false
+ const pendingGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
+
+ await (watcher as any).recoverContextInventoryAndWatch()
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
+
+ expect((watcher as any).ccAppliedLifecycleGeneration).toBe(pendingGeneration)
+ await watcher.stop()
+ })
+
+ it('starts McpServer and Context watches from their inventory resourceVersions', async () => {
+ const watcher = new McpServerWatcher()
+
+ await (watcher as any).startMcpServerWatch('mcp-inventory-rv')
+ await (watcher as any).startContextWatch('context-inventory-rv')
+
+ expect(mocks.watch).toHaveBeenNthCalledWith(
+ 1,
+ expect.stringContaining('/mcpservers'),
+ { resourceVersion: 'mcp-inventory-rv' },
+ expect.any(Function),
+ expect.any(Function)
+ )
+ expect(mocks.watch).toHaveBeenNthCalledWith(
+ 2,
+ expect.stringContaining('/contexts'),
+ { resourceVersion: 'context-inventory-rv' },
+ expect.any(Function),
+ expect.any(Function)
+ )
+ watcher.stop()
+ })
+
+ it.each([
+ ['Context', 'startContextWatch', 'contextWatchGeneration'],
+ ['Host', 'startHostWatch', 'hostWatchGeneration'],
+ ] as const)(
+ 'assigns a fresh generation to every replacement %s watch',
+ async (_kind, method, generationField) => {
+ const watcher = new McpServerWatcher()
+ const before = (watcher as any)[generationField] as number
+
+ const first = await (watcher as any)[method]('first-rv')
+ const second = await (watcher as any)[method]('second-rv')
+
+ expect(first).toBe(before + 1)
+ expect(second).toBe(before + 2)
+ expect((watcher as any)[generationField]).toBe(before + 2)
+ await watcher.stop()
+ }
+ )
+
+ it.each([
+ [
+ 'McpServer',
+ 'restartMcpServerWatch',
+ 'startMcpServerWatch',
+ 'mcpWatchGeneration',
+ 'mcpServerCacheSynced',
+ { resourceVersion: 'mcp-rv', servers: [] },
+ ],
+ [
+ 'Context',
+ 'restartContextWatch',
+ 'startContextWatch',
+ 'contextWatchGeneration',
+ 'contextCacheSynced',
+ { resourceVersion: 'context-rv', contexts: [] },
+ ],
+ ] as const)(
+ 'rejects a %s LIST snapshot that is not paired with a live watch request',
+ async (_kind, restartMethod, startMethod, generationField, cacheField, snapshot) => {
+ const watcher = new McpServerWatcher()
+ vi.spyOn(watcher as any, startMethod).mockImplementation(async () => {
+ ;(watcher as any)[generationField] += 1
+ return (watcher as any)[generationField]
+ })
+
+ await expect((watcher as any)[restartMethod](snapshot)).rejects.toThrow(
+ 'snapshot could not be paired with an active watch'
+ )
+ expect((watcher as any)[cacheField]).toBe(false)
+ await watcher.stop()
+ }
+ )
+
+ it.each([
+ ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],
+ ['Context', 'contexts', 'contextCacheRecoveryTimer'],
+ ] as const)(
+ 'stays live and arms in-process recovery when the %s inventory LIST is unavailable',
+ async (kind, failingPlural, recoveryTimerField) => {
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === failingPlural) throw new Error(`${plural} inventory unavailable`)
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,
+ expect.objectContaining({ message: `${failingPlural} inventory unavailable` })
+ )
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ } finally {
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ }
+ )
+
+ it.each([
+ ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],
+ ['Context', 'contexts', 'contextCacheRecoveryTimer'],
+ ] as const)(
+ 'stays live and arms in-process recovery when the %s collection resourceVersion is missing',
+ async (kind, missingPlural, recoveryTimerField) => {
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural }: { plural: string }) => {
+ if (plural === missingPlural) return { metadata: {}, items: [] }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,
+ expect.objectContaining({ message: `${kind} snapshot missing resourceVersion` })
+ )
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ } finally {
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ }
+ )
+
+ it('replays McpServer and Context changes from the inventory resourceVersion before convergence', async () => {
+ const staleServer = {
+ metadata: {
+ name: 'gap-server',
+ namespace: 'mcp-server',
+ uid: 'gap-server-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/gap-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const staleContext = {
+ metadata: { name: 'gap-context', namespace: 'mcp-server' },
+ spec: { contextId: 'gap-context', mcpServers: [] },
+ }
+ const currentContext = {
+ metadata: { name: 'gap-context', namespace: 'mcp-server' },
+ spec: { contextId: 'gap-context', mcpServers: ['current-server'] },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') {
+ return { metadata: { resourceVersion: 'opaque/mcp:101' }, items: [staleServer] }
+ }
+ if (plural === 'contexts') {
+ return { metadata: { resourceVersion: 'opaque/context:202' }, items: [staleContext] }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ mocks.watch.mockImplementation(async (path, options, callback) => {
+ if (path.endsWith('/mcpservers')) {
+ expect(options).toEqual({ resourceVersion: 'opaque/mcp:101' })
+ await callback('DELETED', staleServer)
+ }
+ if (path.endsWith('/contexts')) {
+ expect(options).toEqual({ resourceVersion: 'opaque/context:202' })
+ await callback('MODIFIED', currentContext)
+ }
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+
+ await watcher.start()
+
+ await vi.waitFor(() => {
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {
+ runEffect: expect.any(Function),
+ })
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(
+ [expect.objectContaining({ spec: currentContext.spec })],
+ [],
+ expect.objectContaining({
+ serverInventoryComplete: true,
+ ensureDefaults: false,
+ contextInventoryAuthoritative: expect.any(Function),
+ serverInventoryAuthoritative: expect.any(Function),
+ contextDesiredRevision: expect.any(Function),
+ serverDesiredRevision: expect.any(Function),
+ })
+ )
+ })
+ expect(watcher.getAllServers()).toEqual([])
+ await watcher.stop()
+ })
+
+ it('recovers a closed McpServer watch through a fresh authoritative snapshot', async () => {
+ vi.useFakeTimers()
+ const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ const watchQueries: unknown[] = []
+ mocks.watch.mockImplementation(async (path, options, callback, done) => {
+ if (path.endsWith('/mcpservers')) {
+ callbacks.push(callback)
+ doneCallbacks.push(done)
+ watchQueries.push(options)
+ }
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') {
+ return {
+ metadata: { resourceVersion: 'mcp-recovery-rv' },
+ items: [
+ {
+ metadata: {
+ name: 'recovered-server',
+ namespace: 'mcp-server',
+ uid: 'recovered-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/recovered:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ await (watcher as any).restartMcpServerWatch({
+ resourceVersion: 'mcp-start-rv',
+ servers: [],
+ })
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+
+ doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.waitFor(() => expect(watchQueries).toHaveLength(2))
+
+ expect(watchQueries).toEqual([
+ { resourceVersion: 'mcp-start-rv' },
+ { resourceVersion: 'mcp-recovery-rv' },
+ ])
+ expect(watcher.getAllServers()).toEqual([expect.objectContaining({ name: 'recovered-server' })])
+ expect(callbacks).toHaveLength(2)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ await watcher.stop()
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ })
+
+ it('withdraws readiness when the safety pass leaves its inventory uncertified', async () => {
+ // Losing a delete fence is the one doom cause that bumps no watch
+ // generation and moves no desired revision, so every equality below stays
+ // satisfied while an allow this pass classified as stale is still live.
+ // The reconciler already degrades its fence at the point of loss; readiness
+ // has to read it, or /ready keeps answering 200 over that allow until a
+ // retry lands (>=5s, and indefinitely against a persistent second writer —
+ // e.g. the two-replica overlap every rolling update opens, since the
+ // Deployment carries no leader election).
+ const watcher = newContextAuthoritativeWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+
+ try {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ } finally {
+ // vi.clearAllMocks() clears calls, not implementations, so an escaped
+ // mockReturnValue would silently re-gate every later test in this file.
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ }
+ })
+
+ it('restores readiness once a later safety pass certifies its inventory again', async () => {
+ // The withdrawal above must not be a one-way latch: a fence raised by a
+ // transient loss has to clear on the next completed pass, or the retry
+ // machinery would leave the controller permanently unready.
+ const watcher = newContextAuthoritativeWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ try {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ } finally {
+ // An assertion failure above must not escape a false implementation into
+ // the rest of the file — vi.clearAllMocks() would not undo it.
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ }
+ })
+
+ it('ignores a late McpServer callback from a retired watch generation', async () => {
+ const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []
+ mocks.watch.mockImplementation(async (path, _options, callback) => {
+ if (path.endsWith('/mcpservers')) callbacks.push(callback)
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ await (watcher as any).restartMcpServerWatch({
+ resourceVersion: 'mcp-old-rv',
+ servers: [],
+ })
+ await (watcher as any).restartMcpServerWatch({
+ resourceVersion: 'mcp-new-rv',
+ servers: [
+ {
+ name: 'current-server',
+ namespace: 'mcp-server',
+ uid: 'current-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/current:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ },
+ ],
+ })
+
+ await callbacks[0]('ADDED', {
+ metadata: {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ uid: 'stale-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+
+ expect(watcher.getAllServers().map(server => server.name)).toEqual(['current-server'])
+ await watcher.stop()
+ })
+
+ it('recovers a closed Context watch through a fresh authoritative snapshot', async () => {
+ vi.useFakeTimers()
+ const doneCallbacks: Array<(error: Error | null) => void> = []
+ const watchQueries: unknown[] = []
+ mocks.watch.mockImplementation(async (path, options, _callback, done) => {
+ if (path.endsWith('/contexts')) {
+ doneCallbacks.push(done)
+ watchQueries.push(options)
+ }
+ return { abort: vi.fn() }
+ })
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ return {
+ metadata: { resourceVersion: 'context-recovery-rv' },
+ items: [
+ {
+ metadata: { name: 'recovered-context', namespace: 'mcp-server' },
+ spec: { contextId: 'recovered-context', mcpServers: [] },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ await (watcher as any).restartContextWatch({
+ resourceVersion: 'context-start-rv',
+ contexts: [],
+ })
+
+ doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.waitFor(() => expect(watchQueries).toHaveLength(2))
+
+ expect(watchQueries).toEqual([
+ { resourceVersion: 'context-start-rv' },
+ { resourceVersion: 'context-recovery-rv' },
+ ])
+ expect((watcher as any).contexts.get('recovered-context')).toEqual(
+ expect.objectContaining({ name: 'recovered-context' })
+ )
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
+ await watcher.stop()
+ })
+
+ it('becomes ready after authoritative revocation while every additive convergence lane remains pending', async () => {
+ const initialServerFleet = deferred()
+ const initialNetworkPolicies = deferred()
+ const initialHostFleet = deferred()
+ const initialSfsFleet = deferred()
+ const initialGfsFleet = deferred()
+ let completeAuthoritativeRevocation: (() => void) | undefined
+ mocks.serverFullReconcile.mockImplementationOnce(() => initialServerFleet.promise)
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown,
+ _servers: unknown,
+ options?: { onAuthoritativeRevocationComplete?: () => void }
+ ) => {
+ completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete
+ await initialNetworkPolicies.promise
+ }
+ )
+ mocks.hostFullReconcile.mockImplementationOnce(() => initialHostFleet.promise)
+ mocks.sfsFullReconcile.mockImplementationOnce(() => initialSfsFleet.promise)
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'safe-bootstrap-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ const startMcpServerWatch = stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ const startContextWatch = stubAuthoritativeInventoryWatch(watcher, 'Context')
+ const startSharedFileSystemWatch = vi
+ .spyOn(watcher as any, 'startSharedFileSystemWatch')
+ .mockResolvedValue(undefined)
+ const startGlobalFileSystemWatch = vi
+ .spyOn(watcher as any, 'startGlobalFileSystemWatch')
+ .mockResolvedValue(undefined)
+ const gfsFullReconcile = vi
+ .spyOn((watcher as any).gfsReconciler, 'fullReconcile')
+ .mockImplementationOnce(() => initialGfsFleet.promise)
+
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ const bootstrap = watcher.start()
+
+ try {
+ await vi.waitFor(() => {
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
+ expect(mocks.sfsFullReconcile).toHaveBeenCalledOnce()
+ expect(gfsFullReconcile).toHaveBeenCalledOnce()
+ })
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+
+ await expect(bootstrap).resolves.toBeUndefined()
+ server.setReady(true)
+ const beforeRevocation = await requestReadyOverHttp(server)
+ expect(beforeRevocation.statusCode).toBe(503)
+ expect(completeAuthoritativeRevocation).toBeTypeOf('function')
+
+ completeAuthoritativeRevocation?.()
+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
+ const ready = await requestReadyOverHttp(server)
+ expect(ready.statusCode).toBe(200)
+ expect(JSON.parse(ready.body)).toEqual({ status: 'ready', ready: true })
+ expect(startMcpServerWatch).toHaveBeenCalledOnce()
+ expect(startContextWatch).toHaveBeenCalledOnce()
+ expect(startSharedFileSystemWatch).toHaveBeenCalledOnce()
+ expect(startGlobalFileSystemWatch).toHaveBeenCalledOnce()
+ expect(watcher.isCommunicationChannelCacheSynced()).toBe(true)
+
+ // Runtime convergence starts at the safety-certification boundary, not
+ // after the NetworkPolicy additive fleet completes. This prevents the
+ // startup egress lane from racing the policy safety pass while retaining
+ // readiness decoupling from both additive fleets.
+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+ } finally {
+ initialServerFleet.resolve(undefined)
+ initialNetworkPolicies.resolve(undefined)
+ initialHostFleet.resolve(undefined)
+ initialSfsFleet.resolve(undefined)
+ initialGfsFleet.resolve(undefined)
+ await bootstrap.catch(() => undefined)
+ await server.stop()
+ watcher.stop()
+ }
+ })
+
+ it('keeps readiness after a current Context safety delta while an older additive pass is pending', async () => {
+ const olderAdditivePass = deferred()
+ const currentDeltaSafety = deferred<boolean>()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ let networkPolicyPasses = 0
+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
+ if (path.endsWith('/contexts')) contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
+ networkPolicyPasses += 1
+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
+ options?.onAuthoritativeRevocationComplete?.()
+ if (networkPolicyPasses === 1) await olderAdditivePass.promise
+ })
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('context-delta-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('delta-context', {
+ name: 'delta-context',
+ namespace: 'mcp-server',
+ uid: 'delta-context-uid',
+ spec: { contextId: 'delta-context', mcpServers: [] },
+ })
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
+ ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
+ () => currentDeltaSafety.promise
+ )
+
+ try {
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+
+ const currentDelta = contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'delta-context',
+ namespace: 'mcp-server',
+ uid: 'delta-context-uid',
+ },
+ spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },
+ })
+
+ await vi.waitFor(() =>
+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()
+ )
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ // The scoped revocation completed, which is what authorises the delta
+ // certificate; see the aborted-pass twin above for the negative side.
+ currentDeltaSafety.resolve(true)
+ await currentDelta
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+ } finally {
+ olderAdditivePass.resolve(undefined)
+ await initialPass
+ await server.stop()
+ await watcher.stop()
+ }
+ })
+
+ it('does not certify a Context delta whose stale-allow revocation aborted mid-pass', async () => {
+ const olderAdditivePass = deferred()
+ const abortedDeltaSafety = deferred<boolean>()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ let networkPolicyPasses = 0
+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
+ if (path.endsWith('/contexts')) contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
+ networkPolicyPasses += 1
+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
+ options?.onAuthoritativeRevocationComplete?.()
+ if (networkPolicyPasses === 1) await olderAdditivePass.promise
+ })
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('aborted-delta-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('delta-context', {
+ name: 'delta-context',
+ namespace: 'mcp-server',
+ uid: 'delta-context-uid',
+ spec: { contextId: 'delta-context', mcpServers: [] },
+ })
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
+ // `reconcileContext` reports that its authority fence broke, so the scoped
+ // revocation never finished deleting this Context's stale allows.
+ ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
+ () => abortedDeltaSafety.promise
+ )
+
+ try {
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+
+ const abortedDelta = contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'delta-context',
+ namespace: 'mcp-server',
+ uid: 'delta-context-uid',
+ },
+ spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },
+ })
+
+ await vi.waitFor(() =>
+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()
+ )
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ abortedDeltaSafety.resolve(false)
+ await abortedDelta
+
+ // An incomplete revocation must leave readiness withheld until the
+ // authoritative full pass re-certifies. Certifying here would report
+ // Ready while a stale allow is still live.
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ } finally {
+ olderAdditivePass.resolve(undefined)
+ await initialPass
+ await server.stop()
+ await watcher.stop()
+ }
+ })
+
+ it('does not certify a Context delta while the namespace-wide inventory is uncertified', async () => {
+ const olderAdditivePass = deferred()
+ const uncertifiedDelta = deferred<boolean>()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ let networkPolicyPasses = 0
+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
+ if (path.endsWith('/contexts')) contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
+ networkPolicyPasses += 1
+ const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
+ options?.onAuthoritativeRevocationComplete?.()
+ if (networkPolicyPasses === 1) await olderAdditivePass.promise
+ })
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('uncertified-inventory-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('delta-context', {
+ name: 'delta-context',
+ namespace: 'mcp-server',
+ uid: 'delta-context-uid',
+ spec: { contextId: 'delta-context', mcpServers: [] },
+ })
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+ const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
+ ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
+ () => uncertifiedDelta.promise
+ )
+ try {
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
+ // Baseline: the inventory is still certified here, so readiness is
+ // genuinely green. Degrading the fence before this point would assert a
+ // 200 that only the missing readiness gate could produce.
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
+
+ // The delta's own revocation completes, but the last authoritative pass left
+ // the namespace-wide inventory uncertified. A label-scoped delta cannot
+ // vouch for that namespace, so readiness must stay withheld.
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+
+ const delta = contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'delta-context',
+ namespace: 'mcp-server',
+ uid: 'delta-context-uid',
+ },
+ spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },
+ })
+
+ await vi.waitFor(() =>
+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()
+ )
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ uncertifiedDelta.resolve(true)
+ await delta
+
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ } finally {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ olderAdditivePass.resolve(undefined)
+ await initialPass
+ await server.stop()
+ await watcher.stop()
+ }
+ })
+
+ it('recovers when a Context delta loses the delete fence with no certificate to withhold', async () => {
+ // A MODIFIED that does not move the desired revision — a status or label
+ // write by another controller — still runs a scoped revocation, but builds
+ // no delta certificate. Nothing reads the returned boolean there, so a lost
+ // delete fence must stay loud and reach the convergence retry instead of
+ // leaving a stale allow live with no NetworkPolicy resync to recover it.
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string; generation?: number }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
+ if (path.endsWith('/contexts')) contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('lost-fence-no-certificate-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('steady-context', {
+ name: 'steady-context',
+ namespace: 'mcp-server',
+ uid: 'steady-context-uid',
+ generation: 4,
+ spec: { contextId: 'steady-context', mcpServers: [] },
+ })
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ const passesBefore = mocks.netPolFullReconcile.mock.calls.length
+ // With honorsLostFence false the real reconciler throws the 409 rather than
+ // reporting it, which is the whole point: the throw is what reaches the
+ // convergence retry below.
+ ;(watcher as any).netPolReconciler.reconcileContext.mockRejectedValueOnce(
+ Object.assign(new Error('the UID in the precondition does not match'), { code: 409 })
+ )
+
+ try {
+ await contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'steady-context',
+ namespace: 'mcp-server',
+ uid: 'steady-context-uid',
+ generation: 4,
+ },
+ spec: { contextId: 'steady-context', mcpServers: [] },
+ })
+
+ // This lane cannot act on the boolean, so it must not have opted into the
+ // reported outcome.
+ expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'steady-context' }),
+ expect.objectContaining({ honorsLostFence: false })
+ )
+ await vi.waitFor(() =>
+ expect(mocks.netPolFullReconcile.mock.calls.length).toBeGreaterThan(passesBefore)
+ )
+ } finally {
+ await watcher.stop()
+ }
+ })
+
+ it('does not issue a Context delta certificate before any authoritative safety certificate', async () => {
+ const pendingFullSafetyPass = deferred()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('uncertified-context-delta-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('uncertified-context', {
+ name: 'uncertified-context',
+ namespace: 'mcp-server',
+ uid: 'uncertified-context-uid',
+ spec: { contextId: 'uncertified-context', mcpServers: [] },
+ })
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+
+ try {
+ await contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'uncertified-context',
+ namespace: 'mcp-server',
+ uid: 'uncertified-context-uid',
+ },
+ spec: { contextId: 'uncertified-context', mcpServers: ['delta-server'] },
+ })
+
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ } finally {
+ pendingFullSafetyPass.resolve(undefined)
+ await watcher.stop()
+ await server.stop()
+ }
+ })
+
+ it('does not let a Context delta certify an outstanding McpServer revision', async () => {
+ const pendingFullSafetyPass = deferred()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('server-revision-context-delta-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('unrelated-context', {
+ name: 'unrelated-context',
+ namespace: 'mcp-server',
+ uid: 'unrelated-context-uid',
+ spec: { contextId: 'unrelated-context', mcpServers: [] },
+ })
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).mcpServerDesiredRevision += 1
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+
+ try {
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ await contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'unrelated-context',
+ namespace: 'mcp-server',
+ uid: 'unrelated-context-uid',
+ },
+ spec: { contextId: 'unrelated-context', mcpServers: ['delta-server'] },
+ })
+
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ } finally {
+ pendingFullSafetyPass.resolve(undefined)
+ await watcher.stop()
+ await server.stop()
+ }
+ })
+
+ it('does not let one Context delta certify another Context revision still in flight', async () => {
+ const pendingFullSafetyPass = deferred()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string; uid: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)
+
+ const watcher = new McpServerWatcher()
+ await (watcher as any).startContextWatch('parallel-context-delta-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 1
+ ;(watcher as any).contexts.set('first-context', {
+ name: 'first-context',
+ namespace: 'mcp-server',
+ uid: 'first-context-uid',
+ spec: { contextId: 'first-context', mcpServers: [] },
+ })
+ ;(watcher as any).contexts.set('second-context', {
+ name: 'second-context',
+ namespace: 'mcp-server',
+ uid: 'second-context-uid',
+ spec: { contextId: 'second-context', mcpServers: [] },
+ })
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ // This represents a prior Context change whose global safety sweep is
+ // pending. It invalidates the old certificate before the scoped delta.
+ ;(watcher as any).contextDesiredRevision += 1
+ const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+
+ try {
+ await contextWatchCallback!('MODIFIED', {
+ metadata: {
+ name: 'second-context',
+ namespace: 'mcp-server',
+ uid: 'second-context-uid',
+ },
+ spec: { contextId: 'second-context', mcpServers: ['delta-server'] },
+ })
+
+ expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
+ } finally {
+ pendingFullSafetyPass.resolve(undefined)
+ await watcher.stop()
+ await server.stop()
+ }
+ })
+
+ it.each([
+ ['Context', 'contextDesiredRevision'],
+ ['McpServer', 'mcpServerDesiredRevision'],
+ ] as const)(
+ 'does not restore readiness from a stale %s delta safety certificate',
+ async (_kind, revisionField) => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ const certificate = {
+ contextGeneration: (watcher as any).contextWatchGeneration,
+ serverGeneration: (watcher as any).mcpWatchGeneration,
+ contextRevision: (watcher as any).contextDesiredRevision,
+ serverRevision: (watcher as any).mcpServerDesiredRevision,
+ }
+
+ ;(watcher as any)[revisionField] += 1
+
+ expect((watcher as any).recordNetworkPolicySafetyCertificate(certificate)).toBe(false)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ await watcher.stop()
+ }
+ )
+
+ it.each([
+ ['SharedFileSystem', 'LIST', 'sharedfilesystems', 'startSharedFileSystemWatch'],
+ ['SharedFileSystem', 'WATCH', 'sharedfilesystems', 'startSharedFileSystemWatch'],
+ ['GlobalFileSystem', 'LIST', 'globalfilesystems', 'startGlobalFileSystemWatch'],
+ ['GlobalFileSystem', 'WATCH', 'globalfilesystems', 'startGlobalFileSystemWatch'],
+ ] as const)(
+ 'does not let a pending %s %s delay NetworkPolicy safety, bootstrap, or readiness certification',
+ async (_kind, boundary, plural, watchMethod) => {
+ const pendingBoundary = deferred<unknown>()
+ const pendingAdditivePolicies = deferred()
+ let completeAuthoritativeRevocation: (() => void) | undefined
+ mocks.listNamespacedCustomObject.mockImplementation(
+ async ({ plural: requestedPlural }: { plural: string }) => {
+ if (requestedPlural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'background-lane-rv' }, items: [] }
+ }
+ if (boundary === 'LIST' && requestedPlural === plural) {
+ return pendingBoundary.promise
+ }
+ return { items: [] }
+ }
+ )
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown,
+ _servers: unknown,
+ options?: { onAuthoritativeRevocationComplete?: () => void }
+ ) => {
+ completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete
+ await pendingAdditivePolicies.promise
+ }
+ )
+
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ const sfsWatch = vi
+ .spyOn(watcher as any, 'startSharedFileSystemWatch')
+ .mockResolvedValue(undefined)
+ const gfsWatch = vi
+ .spyOn(watcher as any, 'startGlobalFileSystemWatch')
+ .mockResolvedValue(undefined)
+ const targetWatch = watchMethod === 'startSharedFileSystemWatch' ? sfsWatch : gfsWatch
+ if (boundary === 'WATCH') {
+ targetWatch.mockImplementationOnce(() => pendingBoundary.promise as Promise<void>)
+ }
+ const bootstrap = watcher.start()
+
+ try {
+ await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
+ await expect(bootstrap).resolves.toBeUndefined()
+
+ expect(completeAuthoritativeRevocation).toBeTypeOf('function')
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ if (boundary === 'LIST') {
+ expect(targetWatch).not.toHaveBeenCalled()
+ } else {
+ expect(targetWatch).toHaveBeenCalledOnce()
+ }
+
+ completeAuthoritativeRevocation?.()
+ await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
+ } finally {
+ pendingBoundary.resolve({ items: [] })
+ pendingAdditivePolicies.resolve(undefined)
+ await flushMicrotasks()
+ await watcher.stop()
+ }
+ }
+ )
+
+ it('starts every initial background convergence lane from the caches current after watches', async () => {
+ const staleServer = {
+ metadata: { name: 'stale-server', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const currentServer = {
+ name: 'current-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/current-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const staleContext = {
+ metadata: { name: 'stale-context', namespace: 'mcp-server' },
+ spec: { contextId: 'stale-context', mcpServers: [] },
+ }
+ const currentContext = {
+ name: 'current-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'current-context', mcpServers: [] },
+ }
+ const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
+ const currentSfs = { name: 'current-sfs', namespace: 'mcp-host', spec: {} }
+ const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }
+ const currentGfs = { name: 'current-gfs', namespace: 'mcp-host', spec: {} }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [staleServer] }
+ if (plural === 'contexts') return { items: [staleContext] }
+ if (plural === 'sharedfilesystems') return { items: [staleSfs] }
+ if (plural === 'globalfilesystems') return { items: [staleGfs] }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'current-cache-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer', () => {
+ ;(watcher as any).servers.clear()
+ ;(watcher as any).servers.set(currentServer.name, currentServer)
+ })
+ stubAuthoritativeInventoryWatch(watcher, 'Context', () => {
+ ;(watcher as any).contexts.clear()
+ ;(watcher as any).contexts.set(currentContext.name, currentContext)
+ })
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => {
+ ;(watcher as any).sharedFileSystems.clear()
+ ;(watcher as any).sharedFileSystems.set(currentSfs.name, currentSfs)
+ })
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => {
+ ;(watcher as any).globalFileSystems.clear()
+ ;(watcher as any).globalFileSystems.set(currentGfs.name, currentGfs)
+ })
+ const gfsFullReconcile = vi.spyOn((watcher as any).gfsReconciler, 'fullReconcile')
+
+ await watcher.start()
+
+ await vi.waitFor(() => {
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([currentServer], {
+ runEffect: expect.any(Function),
+ })
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(
+ [currentContext],
+ [currentServer],
+ expect.objectContaining({
+ serverInventoryComplete: true,
+ ensureDefaults: false,
+ contextInventoryAuthoritative: expect.any(Function),
+ serverInventoryAuthoritative: expect.any(Function),
+ })
+ )
+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([currentSfs])
+ expect(gfsFullReconcile).toHaveBeenCalledWith([currentGfs])
+ })
+
+ watcher.stop()
+ })
+
+ it('updates discovery without re-reconciling a status-only McpServer watch event', async () => {
+ const previousServer = {
+ name: 'status-server',
+ namespace: 'mcp-server',
+ uid: 'status-server-uid',
+ generation: 7,
+ annotations: { 'clerum.io/recipe': 'status-recipe' },
+ labels: { 'clerum.io/workload': 'status-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/status-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).servers.set(previousServer.name, previousServer)
+ const changed = vi.fn()
+ watcher.onChange(changed)
+ const reconciler = (watcher as any).reconciler
+ const netPol = (watcher as any).netPolReconciler
+ const callback = (watcher as any).getMcpServerWatchCallback()
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
+
+ await callback('MODIFIED', {
+ metadata: {
+ name: previousServer.name,
+ namespace: previousServer.namespace,
+ uid: previousServer.uid,
+ generation: previousServer.generation,
+ annotations: previousServer.annotations,
+ labels: previousServer.labels,
+ },
+ spec: previousServer.spec,
+ status: { conditions: [{ type: 'Ready', status: 'True' }] },
+ })
+
+ expect(watcher.getAllServers()[0]?.status).toEqual({
+ conditions: [{ type: 'Ready', status: 'True' }],
+ })
+ expect(changed).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
+
+ await callback('MODIFIED', {
+ metadata: {
+ name: previousServer.name,
+ namespace: previousServer.namespace,
+ uid: previousServer.uid,
+ generation: 8,
+ annotations: previousServer.annotations,
+ labels: previousServer.labels,
+ },
+ spec: { ...previousServer.spec, image: 'clerum/status-server:next' },
+ })
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)
+ await watcher.stop()
+ })
+
+ it('does not advance desired revision for the controller-owned network-ready handshake', async () => {
+ const previousServer = {
+ name: 'network-ready-server',
+ namespace: 'mcp-server',
+ uid: 'network-ready-server-uid',
+ generation: 7,
+ annotations: { 'clerum.io/pre-deploy': 'true' },
+ labels: { 'clerum.io/workload': 'network-ready-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/network-ready-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).servers.set(previousServer.name, previousServer)
+ const callback = (watcher as any).getMcpServerWatchCallback()
+
+ await callback('MODIFIED', {
+ metadata: {
+ name: previousServer.name,
+ namespace: previousServer.namespace,
+ uid: previousServer.uid,
+ generation: previousServer.generation,
+ annotations: {
+ ...previousServer.annotations,
+ 'clerum.io/network-ready': 'true',
+ },
+ labels: previousServer.labels,
+ },
+ spec: previousServer.spec,
+ })
+
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
+ expect((watcher as any).reconciler.reconcile).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('advances McpServer desired revision for owner addition and deletion', async () => {
+ const serverObject = {
+ metadata: {
+ name: 'revision-server',
+ namespace: 'mcp-server',
+ uid: 'revision-server-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/revision-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ const callback = (watcher as any).getMcpServerWatchCallback()
+
+ expect((watcher as any).mcpServerDesiredRevision).toBe(0)
+ await callback('ADDED', serverObject)
+ expect((watcher as any).mcpServerDesiredRevision).toBe(1)
+ await callback('DELETED', serverObject)
+ expect((watcher as any).mcpServerDesiredRevision).toBe(2)
+
+ await watcher.stop()
+ })
+
+ it('advances Context desired revision only when the authoritative desired cache changes', async () => {
+ const watcher = new McpServerWatcher()
+ let contextWatchCallback:
+ | ((
+ type: string,
+ apiObj: {
+ metadata: { name: string; namespace: string }
+ spec: { contextId: string; mcpServers: string[] }
+ }
+ ) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startContextWatch('context-desired-revision-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+
+ const added = {
+ metadata: { name: 'revision-context', namespace: 'mcp-server' },
+ spec: { contextId: 'revision-context', mcpServers: [] as string[] },
+ }
+ expect((watcher as any).contextDesiredRevision).toBe(0)
+ await contextWatchCallback!('ADDED', added)
+ expect((watcher as any).contextDesiredRevision).toBe(1)
+
+ await contextWatchCallback!('MODIFIED', added)
+ expect((watcher as any).contextDesiredRevision).toBe(1)
+
+ const modified = {
+ ...added,
+ spec: { ...added.spec, mcpServers: ['revision-server'] },
+ }
+ await contextWatchCallback!('MODIFIED', modified)
+ expect((watcher as any).contextDesiredRevision).toBe(2)
+
+ await contextWatchCallback!('DELETED', modified)
+ expect((watcher as any).contextDesiredRevision).toBe(3)
+
+ await watcher.stop()
+ })
+
+ it('uses only the bounded fleet pass for the cold-start Host snapshot', async () => {
+ const initialHosts = Array.from({ length: 50 }, (_, index) => ({
+ metadata: {
+ name: `initial-host-${index}`,
+ namespace: 'mcp-host',
+ uid: `initial-host-uid-${index}`,
+ generation: 1,
+ },
+ spec: {
+ host: `initial-host-${index}`,
+ contextRef: 'default',
+ secretRef: 'host-secret',
+ },
+ }))
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ return {
+ metadata: { resourceVersion: 'initial-host-rv' },
+ items: initialHosts,
+ }
+ }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'initial-cc-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ const urgentReconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile')
+
+ await watcher.start()
+ await vi.waitFor(() =>
+ expect(mocks.hostFullReconcile).toHaveBeenCalledWith(
+ initialHosts.map(host =>
+ expect.objectContaining({
+ name: host.metadata.name,
+ generation: host.metadata.generation,
+ })
+ )
+ )
+ )
+ await flushMicrotasks()
+
+ expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
+ expect(urgentReconcile).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('admits a McpServer watch reconciliation while initial convergence is blocked on another server', async () => {
+ const staleServer = {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'stale.example', port: 443 }],
+ },
+ }
+ const currentServer = {
+ metadata: { name: 'current-server', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/current-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const reconciliationOrder: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(staleServer.name, staleServer)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, async () => {
+ reconciliationOrder.push('initial')
+ })
+ )
+ )
+ })
+ const reconciler = (watcher as any).reconciler
+ reconciler.reconcile.mockImplementationOnce(async () => {
+ reconciliationOrder.push('watch')
+ })
+
+ const convergence = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ const watchConvergence = (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)
+
+ await flushMicrotasks()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'current-server' }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ releaseEgress.resolve(undefined)
+ await Promise.all([convergence, watchConvergence])
+
+ expect(reconciliationOrder).toEqual(['watch'])
+ expect(reconciler.reconcile).toHaveBeenLastCalledWith(
+ expect.objectContaining({ name: 'current-server' }),
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ })
+
+ it('continues a same-server reconciliation lane after a rejected operation', async () => {
+ const watcher = new McpServerWatcher()
+ const server = { name: 'retry-safe-server', namespace: 'mcp-server' }
+ const order: string[] = []
+ const firstStarted = deferred()
+ const rejectFirst = deferred()
+
+ const rejected = (watcher as any).enqueueMcpServerReconciliation(server, async () => {
+ order.push('rejected')
+ firstStarted.resolve(undefined)
+ await rejectFirst.promise
+ throw new Error('transient lane failure')
+ }) as Promise<void>
+ await firstStarted.promise
+ const recovered = (watcher as any).enqueueMcpServerReconciliation(server, async () => {
+ order.push('recovered')
+ }) as Promise<void>
+ rejectFirst.resolve(undefined)
+
+ await expect(rejected).rejects.toThrow('transient lane failure')
+ await expect(recovered).resolves.toBeUndefined()
+ expect(order).toEqual(['rejected', 'recovered'])
+ expect((watcher as any).mcpServerReconciliationQueues.size).toBe(0)
+ await watcher.stop()
+ })
+
+ it('revalidates env-Secret matches inside each server lane before reconciling', async () => {
+ const currentServer = {
+ name: 'current-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/current-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const staleServer = {
+ name: 'stale-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const unrelatedServer = {
+ name: 'unrelated-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/unrelated-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'other-env' },
+ },
+ }
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const reconciler = (watcher as any).reconciler
+ for (const server of [currentServer, staleServer, unrelatedServer]) {
+ ;(watcher as any).servers.set(server.name, server)
+ }
+
+ const blocker = (watcher as any).enqueueMcpServerReconciliation(staleServer, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ })
+ await blockerStarted.promise
+
+ const convergence = watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+ ;(watcher as any).servers.set(staleServer.name, {
+ ...staleServer,
+ spec: { ...staleServer.spec, envSecret: { name: 'replacement-env' } },
+ })
+ await vi.waitFor(() =>
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: currentServer.name }),
+ { isCurrent: expect.any(Function) }
+ )
+ )
+ releaseBlocker.resolve(undefined)
+ await Promise.all([blocker, convergence])
+
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ expect(
+ reconciler.reconcile.mock.calls.some(
+ ([server]: [McpServerCRD]) => server.name === staleServer.name
+ )
+ ).toBe(false)
+ expect(
+ reconciler.reconcile.mock.calls.some(
+ ([server]: [McpServerCRD]) => server.name === unrelatedServer.name
+ )
+ ).toBe(false)
+ await watcher.stop()
+ })
+
+ it('defers env-Secret reconciliation until the McpServer inventory is authoritative', async () => {
+ const selected = {
+ name: 'authority-fenced-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/authority-fenced-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ ;(watcher as any).servers.set(selected.name, selected)
+ ;(watcher as any).mcpServerCacheSynced = false
+
+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ markMcpServerInventoryAuthoritative(watcher)
+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: selected.name }),
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ })
+
+ it('fences Context-triggered SharedFileSystem convergence to the current Context lease', async () => {
+ const context = {
+ name: 'sfs-context',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'sfs-context',
+ sharedFileSystems: [{ name: 'shared-data', mountPath: '/shared' }],
+ },
+ }
+ const sharedFileSystem = {
+ name: 'shared-data',
+ namespace: 'mcp-host',
+ spec: {},
+ }
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).sharedFileSystemReconciler
+ ;(watcher as any).sharedFileSystems.set(sharedFileSystem.name, sharedFileSystem)
+
+ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(
+ undefined,
+ context,
+ () => false
+ )
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ await (watcher as any).reconcileSharedFileSystemsReferencedByContext(
+ undefined,
+ context,
+ () => true
+ )
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: sharedFileSystem.name })
+ )
+ await watcher.stop()
+ })
+
+ it('fences Context-triggered Host convergence to the current Context lease', async () => {
+ const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
+ ;(watcher as any).hosts.set('context-host', {
+ name: 'context-host',
+ namespace: 'mcp-host',
+ spec: {
+ host: 'context-host',
+ contextRef: 'context-a',
+ secretRef: 'host-secret',
+ },
+ })
+ const hostReconciler = (watcher as any).hostReconciler
+
+ await (watcher as any).reconcileHostsReferencingContext('context-a', () => false)
+
+ expect(hostReconciler.reconcile).not.toHaveBeenCalled()
+
+ await (watcher as any).reconcileHostsReferencingContext('context-a', () => true)
+
+ expect(hostReconciler.reconcile).toHaveBeenCalledOnce()
+ expect(hostReconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'context-host' })
+ )
+ await watcher.stop()
+ })
+
+ it('keeps env-Secret reconciliation a no-op after stop', async () => {
+ const selected = {
+ name: 'stopped-secret-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stopped-secret-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ envSecret: { name: 'shared-env' },
+ },
+ }
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ ;(watcher as any).servers.set(selected.name, selected)
+
+ await watcher.stop()
+ await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ })
+
+ it('admits an initial server without egress while another initial server waits on DNS', async () => {
+ const dnsBlockedServer = {
+ name: 'dns-blocked-server',
+ namespace: 'mcp-server',
+ uid: 'dns-blocked-server-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/dns-blocked-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'blocked.example', port: 443 }],
+ },
+ }
+ const independentServer = {
+ name: 'independent-server',
+ namespace: 'mcp-server',
+ uid: 'independent-server-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/independent-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(dnsBlockedServer.name, dnsBlockedServer)
+ ;(watcher as any).servers.set(independentServer.name, independentServer)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementationOnce(
+ async (
+ servers: Array<typeof dnsBlockedServer | typeof independentServer>,
+ options: { runEffect: (serverName: string, work: () => Promise<void>) => Promise<void> }
+ ) => {
+ await Promise.all(
+ servers.map(server => options.runEffect(server.name, () => reconciler.reconcile(server)))
+ )
+ }
+ )
+
+ const convergence = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ await flushMicrotasks()
+
+ expect(reconciler.reconcile).toHaveBeenCalledWith(independentServer)
+ expect(reconciler.reconcile).not.toHaveBeenCalledWith(dnsBlockedServer)
+
+ releaseEgress.resolve(undefined)
+ await convergence
+
+ expect(reconciler.reconcile).toHaveBeenCalledWith(dnsBlockedServer)
+ await watcher.stop()
+ })
+
+ it('admits a Context watch reconciliation while initial policy convergence is blocked', async () => {
+ const staleContext = {
+ name: 'stale-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'stale-context', mcpServers: [] },
+ }
+ const currentContext = {
+ metadata: { name: 'current-context', namespace: 'mcp-server' },
+ spec: { contextId: 'current-context', mcpServers: [] },
+ }
+ const fullReconcileStarted = deferred()
+ const releaseFullReconcile = deferred()
+ const reconciliationOrder: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).contexts.set(staleContext.name, staleContext)
+ mocks.netPolFullReconcile.mockImplementationOnce(async () => {
+ fullReconcileStarted.resolve(undefined)
+ await releaseFullReconcile.promise
+ reconciliationOrder.push('initial')
+ })
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementationOnce(async () => {
+ reconciliationOrder.push('watch')
+ })
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof currentContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startContextWatch('context-queue-rv')
+
+ const convergence = (watcher as any).runInitialNetworkPolicyConvergence(true)
+ await fullReconcileStarted.promise
+ const watchConvergence = contextWatchCallback!('MODIFIED', currentContext)
+
+ await flushMicrotasks()
+ expect(netPol.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'current-context' }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+
+ releaseFullReconcile.resolve(undefined)
+ await Promise.all([convergence, watchConvergence])
+
+ expect(reconciliationOrder).toEqual(['watch', 'initial'])
+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
+ expect.objectContaining({ name: 'current-context' }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+ await watcher.stop()
+ })
+
+ it('makes a recreated Context the last writer after same-contextId orphan cleanup', async () => {
+ const contextName = 'recreated-context-resource'
+ const contextId = 'recreated-context'
+ const recreatedContext = {
+ metadata: { name: contextName, namespace: 'mcp-server' },
+ spec: {
+ contextId,
+ description: 'recreated',
+ mcpServers: [],
+ },
+ }
+ const orphanCleanupStarted = deferred()
+ const releaseOrphanCleanup = deferred()
+ const order: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown[],
+ _servers: unknown[],
+ options: {
+ runContextEffect: (effectContextId: string, work: () => Promise<void>) => Promise<void>
+ }
+ ) => {
+ await options.runContextEffect(contextId, async () => {
+ orphanCleanupStarted.resolve(undefined)
+ await releaseOrphanCleanup.promise
+ order.push('orphan-delete')
+ })
+ }
+ )
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementationOnce(async () => {
+ order.push('live-recreate')
+ })
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof recreatedContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startContextWatch('recreated-context-rv')
+
+ const convergence = (watcher as any).runInitialNetworkPolicyConvergence()
+ await orphanCleanupStarted.promise
+ const recreation = contextWatchCallback!('ADDED', recreatedContext)
+ await flushMicrotasks()
+
+ expect(netPol.reconcileContext).not.toHaveBeenCalled()
+
+ releaseOrphanCleanup.resolve(undefined)
+ await Promise.all([convergence, recreation])
+
+ expect(order).toEqual(['orphan-delete', 'live-recreate'])
+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ name: contextName,
+ spec: expect.objectContaining({ contextId, description: 'recreated' }),
+ }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+ await watcher.stop()
+ })
+
+ it('retires old contextId policies before reconciling a mutable Context identity', async () => {
+ const oldContext = {
+ name: 'mutable-context-resource',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'old-context-id',
+ mcpServers: ['redis-tools'],
+ },
+ }
+ const newContext = {
+ metadata: { name: oldContext.name, namespace: oldContext.namespace },
+ spec: {
+ contextId: 'new-context-id',
+ mcpServers: ['redis-tools'],
+ },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ return {
+ metadata: { resourceVersion: 'mutable-context-rv' },
+ items: [newContext],
+ }
+ }
+ return { items: [] }
+ })
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof newContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contexts.set(oldContext.name, oldContext)
+ ;(watcher as any).contextCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const order: string[] = []
+ netPol.reconcileDeleteContext.mockImplementation(async (contextId: string) => {
+ order.push(`delete:${contextId}`)
+ })
+ netPol.reconcileContext.mockImplementation(async (context: typeof oldContext) => {
+ order.push(`reconcile:${context.spec.contextId}`)
+ })
+ await (watcher as any).startContextWatch('mutable-context-rv')
+
+ await contextWatchCallback!('MODIFIED', newContext)
+
+ expect(order).toEqual(['delete:old-context-id', 'reconcile:new-context-id'])
+ expect(netPol.reconcileDeleteContext).toHaveBeenCalledWith(
+ 'old-context-id',
+ expect.any(Function)
+ )
+ await watcher.stop()
+ })
+
+ it('retires queued Context policy work immediately when its watch ends', async () => {
+ const context = {
+ metadata: { name: 'retired-context-resource', namespace: 'mcp-server' },
+ spec: { contextId: 'retired-context-id', mcpServers: [] },
+ }
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined
+ let contextWatchDone: ((error: Error | null) => void) | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {
+ contextWatchCallback = callback
+ contextWatchDone = done
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ await (watcher as any).startContextWatch('retired-context-rv')
+ const blocker = (watcher as any).enqueueContextReconciliation(
+ context.spec.contextId,
+ async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }
+ )
+ await blockerStarted.promise
+
+ const event = contextWatchCallback!('MODIFIED', context)
+ contextWatchDone!(null)
+ releaseBlocker.resolve(undefined)
+ await Promise.all([blocker, event])
+
+ expect(netPol.reconcileContext).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('serializes McpServer-driven Context policy work with a newer Context watch revision', async () => {
+ const server = {
+ metadata: {
+ name: 'context-policy-server',
+ namespace: 'mcp-server',
+ uid: 'context-policy-server-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'context-policy',
+ image: 'clerum/context-policy-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const oldContext = {
+ name: 'context-policy-resource',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'context-policy',
+ description: 'old',
+ mcpServers: [server.metadata.name],
+ },
+ }
+ const newContext = {
+ metadata: { name: oldContext.name, namespace: oldContext.namespace },
+ spec: {
+ ...oldContext.spec,
+ description: 'new',
+ },
+ }
+ const oldStarted = deferred()
+ const releaseOld = deferred()
+ const order: string[] = []
+ let oldLease: (() => boolean) | undefined
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof newContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contexts.set(oldContext.name, oldContext)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementation(
+ async (context: typeof oldContext, options: { isCurrent: () => boolean }) => {
+ if (context.spec.description === 'old') {
+ oldLease = options.isCurrent
+ oldStarted.resolve(undefined)
+ await releaseOld.promise
+ order.push('old')
+ return
+ }
+ order.push('new')
+ }
+ )
+ await (watcher as any).startContextWatch('context-policy-rv')
+
+ const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)
+ await oldStarted.promise
+ const contextEvent = contextWatchCallback!('MODIFIED', newContext)
+ await flushMicrotasks()
+
+ expect(netPol.reconcileContext).toHaveBeenCalledTimes(1)
+ expect(oldLease?.()).toBe(false)
+ releaseOld.resolve(undefined)
+ await Promise.all([serverEvent, contextEvent])
+
+ expect(order).toEqual(['old', 'new'])
+ expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ spec: expect.objectContaining({ description: 'new' }),
+ }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+ await watcher.stop()
+ })
+
+ it('retires a blocked Context policy lease when MODIFIED replaces it in the same watch', async () => {
+ const oldContext = {
+ metadata: { name: 'same-stream-context', namespace: 'mcp-server' },
+ spec: {
+ contextId: 'same-stream-context',
+ description: 'old',
+ mcpServers: ['old-server'],
+ },
+ }
+ const newContext = {
+ metadata: oldContext.metadata,
+ spec: {
+ ...oldContext.spec,
+ description: 'new',
+ mcpServers: ['new-server'],
+ },
+ }
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ let oldLease: (() => boolean) | undefined
+ let contextWatchCallback:
+ | ((type: string, apiObj: typeof oldContext) => Promise<void>)
+ | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileContext.mockImplementationOnce(
+ async (_context: unknown, options: { isCurrent: () => boolean }) => {
+ oldLease = options.isCurrent
+ mutationStarted.resolve()
+ await releaseMutation.promise
+ }
+ )
+ await (watcher as any).startContextWatch('same-stream-context-rv')
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+
+ const oldReconcile = contextWatchCallback!('MODIFIED', oldContext)
+ await mutationStarted.promise
+ expect(oldLease?.()).toBe(true)
+
+ const newReconcile = contextWatchCallback!('MODIFIED', newContext)
+ expect(oldLease?.()).toBe(false)
+
+ releaseMutation.resolve()
+ await Promise.all([oldReconcile, newReconcile])
+ await watcher.stop()
+ })
+
+ it('keeps live control-plane events responsive during a periodic external-egress fleet resync', async () => {
+ const server = {
+ name: 'periodic-egress-server',
+ namespace: 'mcp-server',
+ uid: 'periodic-egress-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/periodic-egress:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'periodic.example', port: 443 }],
+ },
+ }
+ const context = {
+ metadata: { name: 'live-context', namespace: 'mcp-server' },
+ spec: { contextId: 'live-context', mcpServers: [] },
+ }
+ const resyncStarted = deferred()
+ const releaseResync = deferred()
+ let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
+ contextWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0)
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ ;(watcher as any).servers.set(server.name, server)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ resyncStarted.resolve(undefined)
+ await releaseResync.promise
+ })
+ await (watcher as any).startContextWatch('live-context-rv')
+
+ const firstResync = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
+ const overlappingResync = (
+ watcher as any
+ ).externalEgressCoordinator.runResync() as Promise<void>
+ expect(overlappingResync).toBe(firstResync)
+ await resyncStarted.promise
+
+ const liveEvent = contextWatchCallback!('MODIFIED', context)
+ await flushMicrotasks()
+ expect(netPol.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'live-context' }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+
+ releaseResync.resolve(undefined)
+ await Promise.all([firstResync, liveEvent])
+ await watcher.stop()
+ randomSpy.mockRestore()
+ })
+
+ it('retires a blocked periodic egress lease when MODIFIED replaces it in the same watch', async () => {
+ const oldServer = {
+ name: 'same-stream-egress',
+ namespace: 'mcp-server',
+ uid: 'same-stream-egress-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/same-stream-egress:old',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'old.example', port: 443 }],
+ },
+ }
+ const newServer = {
+ metadata: {
+ name: oldServer.name,
+ namespace: oldServer.namespace,
+ uid: oldServer.uid,
+ generation: 2,
+ },
+ spec: {
+ ...oldServer.spec,
+ image: 'clerum/same-stream-egress:new',
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ },
+ }
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ let oldLease: (() => boolean) | undefined
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ ;(watcher as any).servers.set(oldServer.name, oldServer)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(
+ async (_server: unknown, options: { isCurrent: () => boolean }) => {
+ oldLease = options.isCurrent
+ mutationStarted.resolve()
+ await releaseMutation.promise
+ }
+ )
+ vi.spyOn(Math, 'random').mockReturnValue(0)
+
+ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
+ await mutationStarted.promise
+ expect(oldLease?.()).toBe(true)
+
+ const replacement = (watcher as any).getMcpServerWatchCallback()('MODIFIED', newServer)
+ expect(oldLease?.()).toBe(false)
+
+ releaseMutation.resolve()
+ await Promise.all([periodic, replacement])
+ await watcher.stop()
+ })
+
+ it('uses the latest cached McpServer after periodic external-egress jitter', async () => {
+ vi.useFakeTimers()
+ const staleServer = {
+ name: 'periodic-egress-server',
+ namespace: 'mcp-server',
+ uid: 'periodic-egress-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/periodic-egress:old',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'old.example', port: 443 }],
+ },
+ }
+ const currentServer = {
+ metadata: {
+ name: staleServer.name,
+ namespace: staleServer.namespace,
+ uid: staleServer.uid,
+ generation: 2,
+ },
+ spec: {
+ ...staleServer.spec,
+ image: 'clerum/periodic-egress:new',
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ },
+ }
+ const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5)
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).servers.set(staleServer.name, staleServer)
+ const netPol = (watcher as any).netPolReconciler
+
+ const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
+ await flushMicrotasks()
+ await (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)
+ await vi.advanceTimersByTimeAsync(2500)
+ await periodic
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ name: staleServer.name,
+ generation: 2,
+ spec: expect.objectContaining({
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ }),
+ }),
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ randomSpy.mockRestore()
+ })
+
+ it('queues same-object SharedFileSystem and GlobalFileSystem watch effects behind initial passes', async () => {
+ const staleSfs = { name: 'stale-sfs', namespace: 'mcp-host', spec: {} }
+ const currentSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
+ const staleGfs = { name: 'stale-gfs', namespace: 'mcp-host', spec: {} }
+ const currentGfs = {
+ metadata: { name: 'stale-gfs', namespace: 'mcp-host' },
+ spec: {},
+ status: {
+ phase: 'Ready',
+ pvcName: 'gfs-drive',
+ serviceName: 'gfsc',
+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',
+ },
+ }
+ const sfsInitialStarted = deferred()
+ const releaseSfsInitial = deferred()
+ const gfsInitialStarted = deferred()
+ const releaseGfsInitial = deferred()
+ const sfsOrder: string[] = []
+ const gfsOrder: string[] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).sharedFileSystems.set(staleSfs.name, staleSfs)
+ ;(watcher as any).globalFileSystems.set(staleGfs.name, staleGfs)
+
+ mocks.sfsFullReconcile.mockImplementationOnce(async () => {
+ sfsInitialStarted.resolve(undefined)
+ await releaseSfsInitial.promise
+ sfsOrder.push('initial')
+ })
+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler
+ sfsReconciler.reconcile.mockImplementationOnce(async () => {
+ sfsOrder.push('watch')
+ })
+ const gfsReconciler = (watcher as any).gfsReconciler
+ vi.spyOn(gfsReconciler, 'fullReconcile').mockImplementationOnce(async () => {
+ gfsInitialStarted.resolve(undefined)
+ await releaseGfsInitial.promise
+ gfsOrder.push('initial')
+ })
+ const gfsReconcile = vi.spyOn(gfsReconciler, 'reconcile').mockImplementationOnce(async () => {
+ gfsOrder.push('watch')
+ })
+
+ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined
+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined
+ mocks.watch
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ sfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ gfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startSharedFileSystemWatch()
+ await (watcher as any).startGlobalFileSystemWatch()
+
+ const sfsInitial = (watcher as any).runInitialSharedFileSystemConvergence()
+ const gfsInitial = (watcher as any).runInitialGlobalFileSystemConvergence()
+ await Promise.all([sfsInitialStarted.promise, gfsInitialStarted.promise])
+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)
+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)
+
+ await flushMicrotasks()
+ expect(sfsReconciler.reconcile).not.toHaveBeenCalled()
+ expect(gfsReconcile).not.toHaveBeenCalled()
+
+ releaseSfsInitial.resolve(undefined)
+ releaseGfsInitial.resolve(undefined)
+ await Promise.all([sfsInitial, gfsInitial, sfsWatch, gfsWatch])
+
+ expect(sfsOrder).toEqual(['initial', 'watch'])
+ expect(gfsOrder).toEqual(['initial', 'watch'])
+ expect(gfsReconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ status: currentGfs.status })
+ )
+ await watcher.stop()
+ })
+
+ it('does not let a periodic SFS or GFS inventory overwrite a newer watch cache entry', async () => {
+ const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
+ const currentSfs = { metadata: { name: 'current-sfs', namespace: 'mcp-host' }, spec: {} }
+ const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }
+ const currentGfs = { metadata: { name: 'current-gfs', namespace: 'mcp-host' }, spec: {} }
+ const sfsListStarted = deferred()
+ const releaseSfsList = deferred<unknown>()
+ const gfsListStarted = deferred()
+ const releaseGfsList = deferred<unknown>()
+ const watcher = new McpServerWatcher()
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'sharedfilesystems') {
+ sfsListStarted.resolve(undefined)
+ return releaseSfsList.promise
+ }
+ if (plural === 'globalfilesystems') {
+ gfsListStarted.resolve(undefined)
+ return releaseGfsList.promise
+ }
+ return { items: [] }
+ })
+ const gfsReconciler = (watcher as any).gfsReconciler
+ vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)
+
+ let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined
+ let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined
+ mocks.watch
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ sfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ .mockImplementationOnce(async (_path, _options, callback) => {
+ gfsWatchCallback = callback
+ return { abort: vi.fn() }
+ })
+ await (watcher as any).startSharedFileSystemWatch()
+ await (watcher as any).startGlobalFileSystemWatch()
+
+ const sfsResync = (watcher as any).runSfsResync()
+ const gfsResync = (watcher as any).runGfsResync()
+ await Promise.all([sfsListStarted.promise, gfsListStarted.promise])
+ const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)
+ const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)
+ releaseSfsList.resolve({ items: [staleSfs] })
+ releaseGfsList.resolve({ items: [staleGfs] })
+ await Promise.all([sfsResync, gfsResync, sfsWatch, gfsWatch])
+
+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'current-sfs' }),
+ ])
+ expect(gfsReconciler.fullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'current-gfs' }),
+ ])
+ expect((watcher as any).sharedFileSystems.get('current-sfs')).toEqual(
+ expect.objectContaining({ name: 'current-sfs' })
+ )
+ expect((watcher as any).sharedFileSystems.has('stale-sfs')).toBe(false)
+ expect((watcher as any).globalFileSystems.get('current-gfs')).toEqual(
+ expect.objectContaining({ name: 'current-gfs' })
+ )
+ expect((watcher as any).globalFileSystems.has('stale-gfs')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('refreshes SFS and GFS caches and effects from an uncontested periodic inventory', async () => {
+ const listedSfs = {
+ metadata: { name: 'listed-sfs', namespace: 'mcp-host' },
+ spec: { capacity: '1Gi' },
+ }
+ const listedGfs = {
+ metadata: { name: 'listed-gfs', namespace: 'gfs' },
+ spec: { capacity: '10Gi' },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'sharedfilesystems') return { items: [listedSfs] }
+ if (plural === 'globalfilesystems') return { items: [listedGfs] }
+ return { items: [] }
+ })
+ const watcher = new McpServerWatcher()
+ const gfsReconciler = (watcher as any).gfsReconciler
+ const gfsFullReconcile = vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)
+
+ await Promise.all([(watcher as any).runSfsResync(), (watcher as any).runGfsResync()])
+
+ expect((watcher as any).sharedFileSystems.get('listed-sfs')).toEqual(
+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec })
+ )
+ expect((watcher as any).globalFileSystems.get('listed-gfs')).toEqual(
+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec })
+ )
+ expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec }),
+ ])
+ expect(gfsFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec }),
+ ])
+ await watcher.stop()
+ })
+
+ it('does not run queued fleet effects after the watcher stops', async () => {
+ const initialServer = {
+ name: 'initial-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/initial-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'initial.example', port: 443 }],
+ },
+ }
+ const queuedServer = {
+ metadata: { name: 'queued-server', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/queued-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(initialServer.name, initialServer)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ const reconciler = (watcher as any).reconciler
+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, () => reconciler.reconcile(selected))
+ )
+ )
+ })
+
+ const initial = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ const queuedWatch = (watcher as any).getMcpServerWatchCallback()('MODIFIED', queuedServer)
+
+ await watcher.stop()
+ releaseEgress.resolve(undefined)
+ await Promise.all([initial, queuedWatch])
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ })
+
+ it('coalesces overlapping McpServer fleet passes into one trailing current-cache pass', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const snapshots: string[][] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set('first-server', {
+ name: 'first-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/first-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ mocks.serverFullReconcile
+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {
+ snapshots.push(servers.map(server => server.name))
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ })
+ .mockImplementationOnce(async (servers: Array<{ name: string }>) => {
+ snapshots.push(servers.map(server => server.name))
+ })
+
+ const first = (watcher as any).runInitialMcpServerConvergence()
+ await firstPassStarted.promise
+ ;(watcher as any).servers.set('second-server', {
+ name: 'second-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/second-server:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ const second = (watcher as any).runInitialMcpServerConvergence()
+ const third = (watcher as any).runInitialMcpServerConvergence()
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+ releaseFirstPass.resolve(undefined)
+ await Promise.all([first, second, third])
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
+ expect(snapshots).toEqual([['first-server'], ['first-server', 'second-server']])
+ await watcher.stop()
+ })
+
+ it('coalesces overlapping NetworkPolicy fleet passes into one trailing current-cache pass', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const snapshots: string[][] = []
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contexts.set('first-context', {
+ name: 'first-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'first-context', mcpServers: [] },
+ })
+ mocks.netPolFullReconcile
+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {
+ snapshots.push(contexts.map(context => context.name))
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ })
+ .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {
+ snapshots.push(contexts.map(context => context.name))
+ })
+
+ const first = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ ;(watcher as any).contexts.set('second-context', {
+ name: 'second-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'second-context', mcpServers: [] },
+ })
+ const second = (watcher as any).runInitialNetworkPolicyConvergence()
+ const third = (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
+ releaseFirstPass.resolve(undefined)
+ await Promise.all([first, second, third])
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
+ expect(snapshots).toEqual([['first-context'], ['first-context', 'second-context']])
+ await watcher.stop()
+ })
+
+ it('does not lose an initial convergence request queued in the settlement microtask', async () => {
+ const watcher = new McpServerWatcher()
+ let trailing: Promise<void> | undefined
+ const convergenceCore = vi
+ .fn()
+ .mockImplementationOnce(
+ () =>
+ ({
+ then: (resolve: () => void) => {
+ resolve()
+ queueMicrotask(() => {
+ trailing = (watcher as any).runInitialMcpServerConvergence()
+ })
+ },
+ }) as PromiseLike<void>
+ )
+ .mockResolvedValueOnce(undefined)
+ ;(watcher as any).runInitialMcpServerConvergenceCore = convergenceCore
+
+ const first = (watcher as any).runInitialMcpServerConvergence()
+ await first
+ await vi.waitFor(() => expect(trailing).toBeDefined())
+ await trailing
+
+ expect(convergenceCore).toHaveBeenCalledTimes(2)
+ await watcher.stop()
+ })
+
+ it('retains one keyed queue until every same-key waiter has drained', async () => {
+ const watcher = new McpServerWatcher()
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const effects: string[] = []
+ const queues = (watcher as any).contextReconciliationQueues as Map<
+ string,
+ { references: number }
+ >
+
+ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ effects.push('first')
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ })
+ await firstStarted.promise
+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ effects.push('second')
+ })
+
+ expect(queues.get('shared')?.references).toBe(2)
+ expect(effects).toEqual(['first'])
+ releaseFirst.resolve(undefined)
+ await Promise.all([first, second])
+ expect(effects).toEqual(['first', 'second'])
+ expect(queues.size).toBe(0)
+ await watcher.stop()
+ })
+
+ it('does not execute newly enqueued keyed work after stop', async () => {
+ const watcher = new McpServerWatcher()
+ const work = vi.fn()
+ ;(watcher as any).stopped = true
+
+ await (watcher as any).enqueueContextReconciliation('stopped', work)
+
+ expect(work).not.toHaveBeenCalled()
+ expect((watcher as any).contextReconciliationQueues.size).toBe(0)
+ })
+
+ it('acquires multi-context identity keys in one deterministic sorted order', async () => {
+ const watcher = new McpServerWatcher()
+ const acquired: string[] = []
+ vi.spyOn(watcher as any, 'enqueueContextReconciliation').mockImplementation(
+ async (...args: unknown[]) => {
+ const [key, work] = args as [string, () => Promise<void>]
+ acquired.push(key)
+ await work()
+ }
+ )
+
+ await (watcher as any).enqueueContextIdentityReconciliation(
+ ['z-context', 'a-context', 'm-context', 'z-context'],
+ async () => undefined
+ )
+
+ expect(acquired).toEqual(['a-context', 'm-context', 'z-context'])
+ await watcher.stop()
+ })
+
+ it('fences an McpServer fleet pass when its LIST-to-WATCH authority is retired', async () => {
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 3)
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:v1',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'stale.example', port: 443 }],
+ },
+ })
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, () => reconciler.reconcile(selected))
+ )
+ )
+ })
+
+ const stalePass = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ ;(watcher as any).mcpServerCacheSynced = false
+ ;(watcher as any).mcpWatchGeneration = 8
+ const trailing = (watcher as any).runInitialMcpServerConvergence()
+ releaseEgress.resolve(undefined)
+ await Promise.all([stalePass, trailing])
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ generation: 2,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/stale-server:v2',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'current.example', port: 443 }],
+ },
+ })
+ ;(watcher as any).mcpServerCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ await (watcher as any).runInitialMcpServerConvergence()
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({
+ generation: 2,
+ spec: expect.objectContaining({ image: 'clerum/stale-server:v2' }),
+ })
+ )
+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('fences NetworkPolicy positive effects when their inventory leases are retired', async () => {
+ const firstPassStarted = deferred()
+ const releaseFirstPass = deferred()
+ const appliedContexts: string[] = []
+ const appliedServers: string[] = []
+ let pass = 0
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 11
+ ;(watcher as any).mcpWatchGeneration = 13
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
+ ;(watcher as any).contexts.set('stale-context', {
+ name: 'stale-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },
+ })
+ ;(watcher as any).servers.set('stale-server', {
+ name: 'stale-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'stale-context',
+ image: 'clerum/stale-server:v1',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
+ pass += 1
+ if (pass === 1) {
+ firstPassStarted.resolve(undefined)
+ await releaseFirstPass.promise
+ }
+ await Promise.all(
+ contexts.map((context: { spec: { contextId: string } }) =>
+ options.runContextEffect(context.spec.contextId, async () => {
+ appliedContexts.push(context.spec.contextId)
+ })
+ )
+ )
+ await Promise.all(
+ servers.map((server: { name: string }) =>
+ options.runServerEffect(server.name, async () => {
+ appliedServers.push(server.name)
+ })
+ )
+ )
+ })
+
+ const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()
+ await firstPassStarted.promise
+ ;(watcher as any).contextCacheSynced = false
+ ;(watcher as any).mcpServerCacheSynced = false
+ ;(watcher as any).contextWatchGeneration = 12
+ ;(watcher as any).mcpWatchGeneration = 14
+ const trailing = (watcher as any).runInitialNetworkPolicyConvergence()
+ releaseFirstPass.resolve(undefined)
+ await Promise.all([stalePass, trailing])
+
+ expect(appliedContexts).toEqual([])
+ expect(appliedServers).toEqual([])
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)
+ ;(watcher as any).contexts.clear()
+ ;(watcher as any).servers.clear()
+ ;(watcher as any).contexts.set('current-context', {
+ name: 'current-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'current-context', mcpServers: ['current-server'] },
+ })
+ ;(watcher as any).servers.set('current-server', {
+ name: 'current-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'current-context',
+ image: 'clerum/current-server:v2',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ })
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(appliedContexts).toEqual(['current-context'])
+ expect(appliedServers).toEqual(['current-server'])
+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 21
+ ;(watcher as any).mcpWatchGeneration = 34
+ mocks.netPolFullReconcile.mockImplementationOnce(async (_contexts, _servers, options) => {
+ ;(watcher as any).contextCacheSynced = false
+ options.onAuthoritativeRevocationComplete()
+ })
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect((watcher as any).networkPolicyRevocationContextGeneration).not.toBe(21)
+ expect((watcher as any).networkPolicyRevocationServerGeneration).not.toBe(34)
+ await watcher.stop()
+ })
+
+ it('wires onExternalEgressRevoked to schedule a MODIFIED external-egress retry (B3)', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 21
+ ;(watcher as any).mcpWatchGeneration = 34
+ const server = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+ const retrySpy = vi
+ .spyOn((watcher as any).externalEgressCoordinator, 'scheduleRetry')
+ .mockImplementation(() => undefined)
+ let captured: { onExternalEgressRevoked?: (s: unknown) => void } | undefined
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (_contexts: unknown[], _servers: unknown[], options: typeof captured) => {
+ captured = options
+ }
+ )
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')
+ captured!.onExternalEgressRevoked!(server)
+ expect(retrySpy).toHaveBeenCalledWith('MODIFIED', server)
+ await watcher.stop()
+ })
+
+ it('does not start runtime fleet convergence without a current NetworkPolicy safety certificate', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+
+ await (watcher as any).runInitialMcpServerConvergence()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
+
+ await watcher.stop()
+ })
+
+ it('does not start runtime fleet convergence while the safety inventory is uncertified', async () => {
+ // M1: with the revocation counters aligned, the ONLY thing that can null the
+ // certificate is the lost safety fence. Fence lost -> cert null -> the fleet
+ // lane must not run; fence restored -> it runs exactly once.
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ try {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+ await (watcher as any).runInitialMcpServerConvergence()
+ expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
+
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ await (watcher as any).runInitialMcpServerConvergence()
+ expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
+ } finally {
+ // vi.clearAllMocks() clears calls, not implementations — restore explicitly
+ // or every later test would re-gate on a stale false.
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ await watcher.stop()
+ }
+ })
+
+ it('invalidates readiness when either desired policy revision advances', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ ;(watcher as any).mcpServerDesiredRevision += 1
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ ;(watcher as any).contextDesiredRevision += 1
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ await watcher.stop()
+ })
+
+ it('retires readiness and requests a safety pass before a changed McpServer effect settles', async () => {
+ const previous = {
+ metadata: {
+ name: 'restrictive-change',
+ namespace: 'mcp-server',
+ uid: 'restrictive-change-uid',
+ generation: 1,
+ },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/restrictive-change:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' as const }],
+ },
+ }
+ const current = {
+ ...previous,
+ metadata: { ...previous.metadata, generation: 2 },
+ spec: {
+ ...previous.spec,
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' as const }],
+ },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).hostCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 4
+ ;(watcher as any).mcpWatchGeneration = 7
+ ;(watcher as any).servers.set(previous.metadata.name, {
+ name: previous.metadata.name,
+ namespace: previous.metadata.namespace,
+ uid: previous.metadata.uid,
+ generation: previous.metadata.generation,
+ spec: previous.spec,
+ })
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ const safetyPass = vi
+ .spyOn(watcher as any, 'runInitialNetworkPolicyConvergence')
+ .mockResolvedValue(undefined)
+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockRejectedValueOnce(
+ new Error('replacement policy unavailable')
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback(7)('MODIFIED', current)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect(safetyPass).toHaveBeenCalled()
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it('fences runtime effects when either generation retires the safety certificate', async () => {
+ const selected = {
+ name: 'certificate-fenced-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/certificate-fenced:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'certificate-fenced.example', port: 443 }],
+ },
+ }
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const runtimeEffect = vi.fn()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 11
+ ;(watcher as any).mcpWatchGeneration = 13
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).servers.set(selected.name, selected)
+ ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
+ await Promise.all(
+ servers.map((server: { name: string }) => options.runEffect(server.name, runtimeEffect))
+ )
+ })
+
+ const convergence = (watcher as any).runInitialMcpServerConvergence()
+ await egressStarted.promise
+ ;(watcher as any).contextWatchGeneration = 12
+ releaseEgress.resolve(undefined)
+ await convergence
+
+ expect(runtimeEffect).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it.each([
+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],
+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],
+ ] as const)(
+ 'does not admit queued %s effects after the watcher stops',
+ async (_, enqueueMethod) => {
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const watcher = new McpServerWatcher()
+ let queuedEffectRan = false
+
+ const first = (watcher as any)[enqueueMethod]('same-object', async () => {
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ }) as Promise<void>
+ await firstStarted.promise
+ const queued = (watcher as any)[enqueueMethod]('same-object', async () => {
+ queuedEffectRan = true
+ }) as Promise<void>
+
+ await watcher.stop()
+ releaseFirst.resolve(undefined)
+ await Promise.all([first, queued])
+
+ expect(queuedEffectRan).toBe(false)
+ }
+ )
+
+ it.each([
+ ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],
+ ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],
+ ] as const)(
+ 'serializes the same %s identity without blocking an unrelated fleet member',
+ async (_, enqueueMethod) => {
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const secondStarted = deferred()
+ const unrelatedStarted = deferred()
+ const watcher = new McpServerWatcher()
+ let secondRan = false
+
+ const first = (watcher as any)[enqueueMethod]('slow', async () => {
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ }) as Promise<void>
+ await firstStarted.promise
+ const second = (watcher as any)[enqueueMethod]('slow', async () => {
+ secondRan = true
+ secondStarted.resolve(undefined)
+ }) as Promise<void>
+ const unrelated = (watcher as any)[enqueueMethod]('independent', async () => {
+ unrelatedStarted.resolve(undefined)
+ }) as Promise<void>
+
+ await unrelatedStarted.promise
+ expect(secondRan).toBe(false)
+ releaseFirst.resolve(undefined)
+ await Promise.all([first, second, unrelated])
+ await expect(secondStarted.promise).resolves.toBeUndefined()
+ await watcher.stop()
+ }
+ )
+
+ it('keeps a keyed lane alive until a third waiter has consumed the same queue entry', async () => {
+ const firstStarted = deferred()
+ const releaseFirst = deferred()
+ const secondStarted = deferred()
+ const releaseSecond = deferred()
+ const thirdStarted = deferred()
+ let thirdRan = false
+ const watcher = new McpServerWatcher()
+
+ const first = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ firstStarted.resolve(undefined)
+ await releaseFirst.promise
+ })
+ await firstStarted.promise
+ const second = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ secondStarted.resolve(undefined)
+ await releaseSecond.promise
+ })
+ const third = (watcher as any).enqueueContextReconciliation('shared', async () => {
+ thirdRan = true
+ thirdStarted.resolve(undefined)
+ })
+
+ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(3)
+ releaseFirst.resolve(undefined)
+ await secondStarted.promise
+ expect(thirdRan).toBe(false)
+ expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(2)
+ releaseSecond.resolve(undefined)
+ await Promise.all([first, second, third])
+ expect((watcher as any).contextReconciliationQueues.has('shared')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('caps initial convergence retry delay after attempts exceed the schedule', async () => {
+ vi.useFakeTimers()
+ const watcher = new McpServerWatcher()
+ const convergence = vi
+ .spyOn(watcher as any, 'runInitialMcpServerConvergence')
+ .mockResolvedValue(undefined)
+ ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 99)
+ ;(watcher as any).scheduleInitialConvergenceRetry('McpServer')
+ await vi.advanceTimersByTimeAsync(299_999)
+ expect(convergence).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(convergence).toHaveBeenCalledOnce()
+ await watcher.stop()
+ })
+
+ it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = false
+ ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 2)
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)
+ ;(watcher as any).mcpServerCacheSynced = true
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
+ [],
+ [],
+ expect.objectContaining({
+ serverInventoryComplete: true,
+ ensureDefaults: false,
+ contextInventoryAuthoritative: expect.any(Function),
+ serverInventoryAuthoritative: expect.any(Function),
+ })
+ )
+ expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
+ await watcher.stop()
+ })
+
+ it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextWatchGeneration = 17
+ ;(watcher as any).mcpWatchGeneration = 23
+ let options:
+ | {
+ contextInventoryAuthoritative: () => boolean
+ serverInventoryAuthoritative: () => boolean
+ }
+ | undefined
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown[],
+ _servers: unknown[],
+ received: {
+ contextInventoryAuthoritative: () => boolean
+ serverInventoryAuthoritative: () => boolean
+ }
+ ) => {
+ options = received
+ }
+ )
+
+ await (watcher as any).runInitialNetworkPolicyConvergence()
+
+ expect(options?.contextInventoryAuthoritative()).toBe(true)
+ expect(options?.serverInventoryAuthoritative()).toBe(true)
+
+ // A recovered watch may already be authoritative again, but it is a
+ // different snapshot generation and cannot authorize cleanup by this pass.
+ ;(watcher as any).contextWatchGeneration = 18
+ ;(watcher as any).mcpWatchGeneration = 24
+
+ expect(options?.contextInventoryAuthoritative()).toBe(false)
+ expect(options?.serverInventoryAuthoritative()).toBe(false)
+ await watcher.stop()
+ })
+
+ it('escalates and resets failed initial McpServer convergence without delaying bootstrap', async () => {
+ vi.useFakeTimers()
+ const server = {
+ metadata: { name: 'initial-runtime', namespace: 'mcp-server' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/initial-runtime:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'mcp-retry-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ mocks.serverFullReconcile
+ .mockRejectedValueOnce(new Error('initial runtime reconciliation failed'))
+ .mockRejectedValueOnce(new Error('initial runtime reconciliation still unavailable'))
+ .mockResolvedValueOnce(undefined)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const retryMetricBefore = await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_retries_total',
+ 'McpServer'
+ )
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await watcher.start()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
+
+ await vi.advanceTimersByTimeAsync(14_999)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(3)
+ expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)
+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(false)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_retries_total',
+ 'McpServer'
+ )
+ ).toBe(retryMetricBefore + 2)
+ expect(
+ await readInitialConvergenceMetric(
+ 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ 'McpServer'
+ )
+ ).toBeGreaterThan(0)
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it('cancels a pending initial convergence retry when the watcher stops', async () => {
+ vi.useFakeTimers()
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ mocks.serverFullReconcile.mockRejectedValueOnce(
+ new Error('initial runtime reconciliation unavailable')
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ await (watcher as any).runInitialMcpServerConvergence()
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+ expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(1)
+ expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(true)
+
+ await watcher.stop()
+ await vi.advanceTimersByTimeAsync(300_000)
+
+ expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
+ expect((watcher as any).initialConvergenceRetryAttempts.size).toBe(0)
+ expect((watcher as any).initialConvergenceRetryTimers.size).toBe(0)
+ errorSpy.mockRestore()
+ })
+
+ it('retries failed initial NetworkPolicy convergence from the current caches', async () => {
+ vi.useFakeTimers()
+ const context = {
+ metadata: { name: 'initial-context', namespace: 'mcp-server' },
+ spec: { contextId: 'initial-context', mcpServers: [] },
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [] }
+ if (plural === 'contexts') return { items: [context] }
+ if (plural === 'communicationchannels') {
+ return { metadata: { resourceVersion: 'netpol-retry-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ mocks.netPolFullReconcile
+ .mockRejectedValueOnce(new Error('initial NetworkPolicy reconciliation failed'))
+ .mockImplementationOnce(
+ async (
+ _contexts: unknown,
+ _servers: unknown,
+ options?: { onAuthoritativeRevocationComplete?: () => void }
+ ) => {
+ options?.onAuthoritativeRevocationComplete?.()
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await watcher.start()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
+ [expect.objectContaining({ name: 'initial-context' })],
+ [],
+ expect.objectContaining({
+ serverInventoryComplete: true,
+ ensureDefaults: false,
+ contextInventoryAuthoritative: expect.any(Function),
+ serverInventoryAuthoritative: expect.any(Function),
+ })
+ )
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it('does not report a post-certification additive failure as a readiness safety failure', async () => {
+ vi.useFakeTimers()
+ const additiveFailure = new AggregateError(
+ [new Error('Context policy API unavailable')],
+ 'One or more additive Context NetworkPolicy reconciliations failed'
+ )
+ mocks.netPolFullReconcile.mockImplementationOnce(
+ async (
+ _contexts: unknown,
+ _servers: unknown,
+ options?: { onAuthoritativeRevocationComplete?: () => void }
+ ) => {
+ options?.onAuthoritativeRevocationComplete?.()
+ throw additiveFailure
+ }
+ )
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await watcher.start()
+ await vi.advanceTimersByTimeAsync(0)
+
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect(errorSpy).toHaveBeenCalledWith(
+ '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',
+ additiveFailure
+ )
+ expect(errorSpy).not.toHaveBeenCalledWith(
+ '[K8s] Initial NetworkPolicy background reconciliation failed:',
+ additiveFailure
+ )
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ })
+
+ it.each([
+ ['McpServer', 'startMcpServerWatch'],
+ ['Context', 'startContextWatch'],
+ ] as const)(
+ 'keeps safe bootstrap authority-gated when the %s watch cannot be established',
+ async (_kind, method) => {
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ if (method === 'startMcpServerWatch') {
+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))
+ } else {
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ }
+ if (method === 'startContextWatch') {
+ vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))
+ } else if (method !== 'startMcpServerWatch') {
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ }
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+ const recoveryTimerField =
+ method === 'startMcpServerWatch'
+ ? 'mcpServerCacheRecoveryTimer'
+ : 'contextCacheRecoveryTimer'
+
+ try {
+ await expect(watcher.start()).resolves.toBeUndefined()
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
+ expect((watcher as any)[recoveryTimerField]).not.toBeNull()
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ } finally {
+ await watcher.stop()
+ errorSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ }
+ )
+
+ it.each([
+ ['SharedFileSystem', 'startSharedFileSystemWatch'],
+ ['GlobalFileSystem', 'startGlobalFileSystemWatch'],
+ ] as const)(
+ 'isolates, logs, and retries an initial %s background watch rejection',
+ async (kind, method) => {
+ vi.useFakeTimers()
+ const failure = new Error(`${method} unavailable`)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const watcher = new McpServerWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ const rejectedWatch = vi
+ .spyOn(watcher as any, method)
+ .mockRejectedValueOnce(failure)
+ .mockResolvedValueOnce(undefined)
+ const otherMethod =
+ method === 'startSharedFileSystemWatch'
+ ? 'startGlobalFileSystemWatch'
+ : 'startSharedFileSystemWatch'
+ vi.spyOn(watcher as any, otherMethod).mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
+
+ await expect(watcher.start()).resolves.toBeUndefined()
+ await vi.advanceTimersByTimeAsync(0)
+
+ expect(errorSpy).toHaveBeenCalledWith(
+ `[K8s] ${kind} background watch failed to start:`,
+ failure
+ )
+ expect(rejectedWatch).toHaveBeenCalledOnce()
+ expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
+
+ await vi.advanceTimersByTimeAsync(4_999)
+ expect(rejectedWatch).toHaveBeenCalledOnce()
+ await vi.advanceTimersByTimeAsync(1)
+ expect(rejectedWatch).toHaveBeenCalledTimes(2)
+
+ await watcher.stop()
+ errorSpy.mockRestore()
+ }
+ )
+
+ it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') return { items: [] }
+ return { items: [] }
+ })
+
+ await expect(listAllHosts()).resolves.toEqual([])
+
+ expect(mocks.hostListCallOptions).toHaveBeenCalledWith(
+ expect.objectContaining({
+ middleware: [expect.objectContaining({ pre: expect.any(Function) })],
+ middlewareMergeStrategy: 'append',
+ })
+ )
+ })
+
+ it('preserves Host UID from the Kubernetes inventory boundary', async () => {
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'uid-host',
+ namespace: 'mcp-host',
+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
+ generation: 4,
+ },
+ spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+
+ await expect(listAllHosts()).resolves.toEqual([
+ expect.objectContaining({
+ name: 'uid-host',
+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
+ generation: 4,
+ }),
+ ])
+ })
+
+ it('preserves observed GlobalFileSystem status from the Kubernetes inventory boundary', async () => {
+ const status = {
+ phase: 'Ready',
+ pvcName: 'gfs-drive',
+ serviceName: 'gfsc',
+ serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',
+ }
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'globalfilesystems') {
+ return {
+ items: [
+ {
+ metadata: { name: 'gfs', namespace: 'gfs' },
+ spec: {},
+ status,
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+
+ await expect(listAllGlobalFileSystems()).resolves.toEqual([
+ expect.objectContaining({ name: 'gfs', namespace: 'gfs', status }),
+ ])
+ })
+
+ it('reconciles startup external egress before runtime full reconciliation', async () => {
+ const eventLog: string[] = []
+ const server = {
+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
+ spec: {
+ contextRef: 'wf-research',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
+ },
+ }
+
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: '1' }, items: [] }
+ return { items: [] }
+ })
+ mocks.ensureDefaultPolicies.mockImplementation(async () => {
+ eventLog.push('defaults')
+ })
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, async () => {
+ eventLog.push('runtime')
+ await reconciler.reconcile(selected)
+ })
+ )
+ )
+ })
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementation(async () => {
+ eventLog.push('egress')
+ })
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
+
+ await watcher.start()
+
+ await vi.waitFor(() => expect(eventLog).toContain('runtime'))
+ expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(
+ [expect.objectContaining({ name: 'web-search' })],
+ { runEffect: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {
+ vi.useFakeTimers()
+ const server = {
+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
+ spec: {
+ contextRef: 'wf-research',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
+ },
+ }
+
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: '1' }, items: [] }
+ return { items: [] }
+ })
+
+ const watcher = new McpServerWatcher()
+ const reconciler = (watcher as any).reconciler
+ mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
+ await Promise.all(
+ servers.map((selected: { name: string }) =>
+ options.runEffect(selected.name, () => reconciler.reconcile(selected))
+ )
+ )
+ })
+ const netPol = (watcher as any).netPolReconciler
+ const dnsFailure = new Error('dns resolution failed')
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(dnsFailure)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
+
+ await watcher.start()
+
+ await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith(
+ [expect.objectContaining({ name: 'web-search' })],
+ { runEffect: expect.any(Function) }
+ )
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
+ expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)
+ expect(errorSpy).not.toHaveBeenCalledWith(
+ '[K8s] Initial NetworkPolicy background reconciliation failed:',
+ dnsFailure
+ )
watcher.stop()
errorSpy.mockRestore()
@@ -448,6 +4627,29 @@ describe('McpServerWatcher startup', () => {
})
})
+describe('getContext error semantics', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ })
+
+ it('preserves not-found semantics only for an authoritative 404', async () => {
+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(
+ Object.assign(new Error('context absent'), { code: 404 })
+ )
+
+ await expect(getContext('missing-context')).resolves.toBeNull()
+ })
+
+ it('propagates non-404 API failures instead of converting discovery to an empty fleet', async () => {
+ const unavailable = Object.assign(new Error('apiserver unavailable'), {
+ response: { statusCode: 503 },
+ })
+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)
+
+ await expect(getContext('production-context')).rejects.toBe(unavailable)
+ })
+})
+
describe('McpServerWatcher external egress retries', () => {
const serverObject = {
metadata: { name: 'redis-tools', namespace: 'mcp-server' },
@@ -459,42 +4661,195 @@ describe('McpServerWatcher external egress retries', () => {
},
}
- beforeEach(() => {
- vi.useFakeTimers()
- })
+ beforeEach(() => {
+ vi.clearAllMocks()
+ vi.useFakeTimers()
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+ })
+
+ it('retries failed external egress reconciliation from an ADDED event', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const serverWithBindings = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ annotations: {
+ 'clerum.io/recipe-bindings': JSON.stringify([
+ { source: 'redis-tools', target: 'downstream' },
+ ]),
+ },
+ },
+ }
+ ;(watcher as any).contexts.set('context1', {
+ name: 'context1',
+ namespace: 'mcp-server',
+ spec: { contextId: 'context1', mcpServers: ['redis-tools'] },
+ })
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverWithBindings)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()
+ expect(netPol.reconcileContext).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'redis-tools' }),
+ { isCurrent: expect.any(Function) }
+ )
+ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(1)
+ expect(netPol.reconcileContext).toHaveBeenCalledWith(
+ expect.objectContaining({
+ spec: expect.objectContaining({ contextId: 'context1' }),
+ }),
+ expect.objectContaining({ isCurrent: expect.any(Function) })
+ )
+
+ watcher.stop()
+ })
+
+ it('retires a binding-policy lease when a same-watch MODIFIED supersedes the server', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const bindingStarted = deferred()
+ const releaseBinding = deferred()
+ bindingReconciler.reconcileBindings.mockImplementationOnce(async () => {
+ bindingStarted.resolve(undefined)
+ await releaseBinding.promise
+ })
+ const annotations = {
+ 'clerum.io/recipe-bindings': JSON.stringify([
+ { from: 'redis-tools', to: 'downstream', port: 443 },
+ ]),
+ }
+ const original = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ uid: 'redis-tools-uid',
+ generation: 1,
+ annotations,
+ },
+ }
+ const replacement = {
+ ...original,
+ metadata: { ...original.metadata, generation: 2 },
+ spec: { ...original.spec, image: 'redis-tools:v2' },
+ }
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const originalEffect = watchCallback('ADDED', original)
+ await bindingStarted.promise
+ const replacementEffect = watchCallback('MODIFIED', replacement)
+
+ const originalOptions = bindingReconciler.reconcileBindings.mock.calls[0][4]
+ expect(originalOptions).toEqual({ isCurrent: expect.any(Function) })
+ expect(originalOptions.isCurrent()).toBe(false)
+
+ releaseBinding.resolve(undefined)
+ await Promise.all([originalEffect, replacementEffect])
+
+ expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(2)
+ expect(bindingReconciler.reconcileBindings).toHaveBeenLastCalledWith(
+ 'redis-tools',
+ expect.any(Array),
+ 'redis-tools',
+ 'redis-tools',
+ { isCurrent: expect.any(Function) }
+ )
+ await watcher.stop()
+ })
+
+ it('retires binding cleanup when a same-name McpServer is recreated during DELETE', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const cleanupStarted = deferred()
+ const releaseCleanup = deferred()
+ bindingReconciler.cleanupBindings.mockImplementationOnce(async () => {
+ cleanupStarted.resolve(undefined)
+ await releaseCleanup.promise
+ })
+ const deleted = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ uid: 'redis-tools-old-uid',
+ generation: 1,
+ },
+ }
+ const recreated = {
+ ...serverObject,
+ metadata: {
+ ...serverObject.metadata,
+ uid: 'redis-tools-new-uid',
+ generation: 1,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },
+ }
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const deleteEffect = watchCallback('DELETED', deleted)
+ await cleanupStarted.promise
+ const recreateEffect = watchCallback('ADDED', recreated)
- afterEach(() => {
- vi.useRealTimers()
- vi.restoreAllMocks()
+ const cleanupOptions = bindingReconciler.cleanupBindings.mock.calls[0][1]
+ expect(cleanupOptions).toEqual({ deleteAllowed: expect.any(Function) })
+ await expect(cleanupOptions.deleteAllowed()).resolves.toBe(false)
+
+ releaseCleanup.resolve(undefined)
+ await Promise.all([deleteEffect, recreateEffect])
+
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledTimes(1)
+ await watcher.stop()
})
- it('retries failed external egress reconciliation from an ADDED event', async () => {
+ it('retries the complete pipeline when a no-egress runtime reconcile fails', async () => {
const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
+ markMcpServerInventoryAuthoritative(watcher)
const reconciler = (watcher as any).reconciler
- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
- netPol.reconcileExternalEgress.mockResolvedValue(undefined)
+ const noEgressServer = {
+ ...serverObject,
+ spec: {
+ ...serverObject.spec,
+ egressBindings: undefined,
+ },
+ }
+ reconciler.reconcile
+ .mockRejectedValueOnce(new Error('deployment temporarily unavailable'))
+ .mockResolvedValue(undefined)
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await watchCallback('ADDED', serverObject)
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ await (watcher as any).getMcpServerWatchCallback()('ADDED', noEgressServer)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5000)
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'redis-tools' })
- )
-
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
watcher.stop()
})
it('retries runtime reconciliation when external egress is ready but runtime reconcile fails', async () => {
const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockResolvedValue(undefined)
@@ -514,7 +4869,8 @@ describe('McpServerWatcher external egress retries', () => {
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenLastCalledWith(
- expect.objectContaining({ name: 'redis-tools' })
+ expect.objectContaining({ name: 'redis-tools' }),
+ { isCurrent: expect.any(Function) }
)
watcher.stop()
@@ -522,6 +4878,7 @@ describe('McpServerWatcher external egress retries', () => {
it('keeps retry attempts bounded when runtime fails after an external egress retry succeeds', async () => {
const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
@@ -547,14 +4904,16 @@ describe('McpServerWatcher external egress retries', () => {
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(3)
expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenLastCalledWith(
- expect.objectContaining({ name: 'redis-tools' })
+ expect.objectContaining({ name: 'redis-tools' }),
+ { isCurrent: expect.any(Function) }
)
watcher.stop()
})
- it('does not let periodic external egress resync cancel a pending runtime retry', async () => {
+ it('continues a pending runtime pipeline during periodic external egress resync', async () => {
const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockResolvedValue(undefined)
@@ -563,6 +4922,7 @@ describe('McpServerWatcher external egress retries', () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'log').mockImplementation(() => {})
+ vi.spyOn(Math, 'random').mockReturnValue(0)
const watchCallback = (watcher as any).getMcpServerWatchCallback()
await watchCallback('ADDED', serverObject)
@@ -570,18 +4930,152 @@ describe('McpServerWatcher external egress retries', () => {
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
- await (watcher as any).runExternalEgressResync()
+ await (watcher as any).externalEgressCoordinator.runResync()
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(5000)
-
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
watcher.stop()
})
+ it('keeps retrying an ADDED pipeline at the capped delay until it recovers', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress
+ .mockRejectedValueOnce(new Error('initial outage'))
+ .mockRejectedValueOnce(new Error('outage after 5s'))
+ .mockRejectedValueOnce(new Error('outage after 15s'))
+ .mockRejectedValueOnce(new Error('outage after 30s'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback()('ADDED', serverObject)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.advanceTimersByTimeAsync(15000)
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(4)
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(5)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ watcher.stop()
+ })
+
+ it('keeps retrying a partial DELETE pipeline at the capped delay until it recovers', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ netPol.cleanupExternalEgress
+ .mockRejectedValueOnce(new Error('initial cleanup outage'))
+ .mockRejectedValueOnce(new Error('cleanup outage after 5s'))
+ .mockRejectedValueOnce(new Error('cleanup outage after 15s'))
+ .mockRejectedValueOnce(new Error('cleanup outage after 30s'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ await (watcher as any).getMcpServerWatchCallback()('DELETED', serverObject)
+ await vi.advanceTimersByTimeAsync(5000)
+ await vi.advanceTimersByTimeAsync(15000)
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(4)
+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(30000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(5)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+ watcher.stop()
+ })
+
+ it('bounds concurrent durable retries across McpServers', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const reconciler = (watcher as any).reconciler
+ const release = deferred()
+ let active = 0
+ let maxActive = 0
+ reconciler.reconcile.mockImplementation(async () => {
+ active += 1
+ maxActive = Math.max(maxActive, active)
+ await release.promise
+ active -= 1
+ })
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ for (let index = 0; index < 11; index += 1) {
+ const server = {
+ name: `server-${index}`,
+ namespace: 'mcp-server',
+ spec: {
+ ...serverObject.spec,
+ egressBindings: undefined,
+ },
+ }
+ ;(watcher as any).servers.set(server.name, server)
+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', server)
+ }
+
+ await vi.advanceTimersByTimeAsync(5000)
+ await flushMicrotasks()
+
+ expect(maxActive).toBe(10)
+
+ release.resolve(undefined)
+ await flushMicrotasks()
+ watcher.stop()
+ })
+
+ it('replays a retry against the watcher generation current when the timer executes', async () => {
+ const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
+ const selected = {
+ name: 'generation-fenced-server',
+ namespace: 'mcp-server',
+ spec: {
+ ...serverObject.spec,
+ egressBindings: undefined,
+ },
+ }
+ ;(watcher as any).servers.set(selected.name, selected)
+ ;(watcher as any).mcpWatchGeneration = 41
+ const replay = vi
+ .spyOn(watcher as any, 'reconcileMcpServerWatchEvent')
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ ;(watcher as any).scheduleExternalEgressRetry('ADDED', selected)
+ ;(watcher as any).mcpWatchGeneration = 42
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(replay).toHaveBeenCalledWith(
+ 'MODIFIED',
+ expect.objectContaining({ name: selected.name }),
+ 42,
+ expect.objectContaining({
+ isCurrent: expect.any(Function),
+ complete: expect.any(Function),
+ })
+ )
+ watcher.stop()
+ })
+
it('reconciles external egress before runtime reconciliation on ADDED events', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
@@ -611,36 +5105,337 @@ describe('McpServerWatcher external egress retries', () => {
vi.spyOn(console, 'warn').mockImplementation(() => {})
const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await watchCallback('ADDED', serverObject)
+ await watchCallback('ADDED', serverObject)
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+
+ watcher.stop()
+ })
+
+ it('retries failed external egress cleanup from a DELETED event', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))
+ netPol.cleanupExternalEgress.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('DELETED', serverObject)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+ expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith(
+ 'redis-tools',
+ 'mcp-server',
+ undefined,
+ expect.any(Function)
+ )
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+
+ watcher.stop()
+ })
+
+ it('retries the full DELETE pipeline when authoritative absence is temporarily unavailable', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ const unavailable = Object.assign(new Error('apiserver temporarily unavailable'), {
+ response: { statusCode: 503 },
+ })
+ mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()
+
+ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
+ expect(netPol.cleanupExternalEgress).not.toHaveBeenCalled()
+ expect(reconciler.reconcileDelete).not.toHaveBeenCalled()
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+ expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(4)
+
+ watcher.stop()
+ })
+
+ it('retries the full DELETE pipeline when inventory authority is lost between cleanup stages', async () => {
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ reconciler.reconcileDelete.mockImplementationOnce(async () => {
+ ;(watcher as any).mcpServerCacheSynced = false
+ })
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(1)
+ expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
+ ;(watcher as any).mcpServerCacheSynced = true
+ await vi.advanceTimersByTimeAsync(14_999)
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ await vi.advanceTimersByTimeAsync(1)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(2)
+ expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
+ deleteAllowed: expect.any(Function),
+ })
+
+ watcher.stop()
+ })
+
+ it('retires queued McpServer work immediately when its watch ends', async () => {
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ let watchCallback: ((type: string, apiObj: typeof serverObject) => Promise<void>) | undefined
+ let watchDone: ((error: Error | null) => void) | undefined
+ mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {
+ watchCallback = callback
+ watchDone = done
+ return { abort: vi.fn() }
+ })
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ await (watcher as any).startMcpServerWatch('mcp-watch-rv')
+ const blocker = (watcher as any).enqueueMcpServerReconciliation(
+ { name: 'redis-tools', namespace: 'mcp-server' },
+ async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }
+ )
+ await blockerStarted.promise
+
+ const event = watchCallback!('ADDED', serverObject)
+ watchDone!(null)
+ releaseBlocker.resolve(undefined)
+ await Promise.all([blocker, event])
+
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ await watcher.stop()
+ })
+
+ it('retries the recreated server when an older DELETE retry timer already owns the key', async () => {
+ const deletedServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-old-uid',
+ generation: 1,
+ },
+ spec: serverObject.spec,
+ }
+ const recreatedServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-new-uid',
+ generation: 1,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:recreated' },
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('cleanup unavailable'))
+ netPol.reconcileExternalEgress
+ .mockRejectedValueOnce(new Error('recreated policy unavailable'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ await watchCallback('DELETED', deletedServer)
+ await watchCallback('ADDED', recreatedServer)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({
+ uid: 'redis-tools-new-uid',
+ generation: 1,
+ spec: expect.objectContaining({ image: 'redis-tools:recreated' }),
+ }),
+ { isCurrent: expect.any(Function) }
+ )
+
+ watcher.stop()
+ })
+
+ it('retries the latest DELETE cleanup when an older ADD retry timer already owns the key', async () => {
+ const server = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 3,
+ },
+ spec: serverObject.spec,
+ }
+ const watcher = new McpServerWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('policy unavailable'))
+ netPol.cleanupExternalEgress
+ .mockRejectedValueOnce(new Error('cleanup unavailable'))
+ .mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ await watchCallback('ADDED', server)
+ await watchCallback('DELETED', server)
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+
+ watcher.stop()
+ })
+
+ it('does not continue to runtime or bindings after stop releases a live DNS hold', async () => {
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const server = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 1,
+ annotations: {
+ 'clerum.io/recipe-bindings': JSON.stringify([
+ { source: 'redis-tools', target: 'downstream' },
+ ]),
+ },
+ },
+ spec: serverObject.spec,
+ }
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ const bindingReconciler = (watcher as any).bindingReconciler
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+
+ const reconcile = watchCallback('ADDED', server)
+ await egressStarted.promise
+ await watcher.stop()
+ releaseEgress.resolve(undefined)
+ await reconcile
+
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
+ expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()
+ })
+
+ it('does not run an older runtime revision after newer desired state arrives during egress', async () => {
+ const egressStarted = deferred()
+ const releaseEgress = deferred()
+ const oldServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 1,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:old' },
+ }
+ const newServer = {
+ metadata: {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ uid: 'redis-tools-uid',
+ generation: 2,
+ },
+ spec: { ...serverObject.spec, image: 'redis-tools:new' },
+ }
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const reconciler = (watcher as any).reconciler
+ netPol.reconcileExternalEgress
+ .mockImplementationOnce(async () => {
+ egressStarted.resolve(undefined)
+ await releaseEgress.promise
+ })
+ .mockResolvedValue(undefined)
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
- expect(reconciler.reconcile).not.toHaveBeenCalled()
+ const oldReconcile = watchCallback('ADDED', oldServer)
+ await egressStarted.promise
+ const newReconcile = watchCallback('MODIFIED', newServer)
+ releaseEgress.resolve(undefined)
+ await Promise.all([oldReconcile, newReconcile])
+
+ expect(reconciler.reconcile).toHaveBeenCalledOnce()
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({
+ generation: 2,
+ spec: expect.objectContaining({ image: 'redis-tools:new' }),
+ }),
+ { isCurrent: expect.any(Function) }
+ )
watcher.stop()
})
- it('retries failed external egress cleanup from a DELETED event', async () => {
+ it('does not resume periodic external egress work from jitter after stop', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
- netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))
- netPol.cleanupExternalEgress.mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await watchCallback('DELETED', serverObject)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
-
- await vi.advanceTimersByTimeAsync(5000)
+ ;(watcher as any).servers.set('redis-tools', {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ })
+ vi.spyOn(Math, 'random').mockReturnValue(0.5)
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
- expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith('redis-tools', 'mcp-server')
+ const resync = (watcher as any).externalEgressCoordinator.runResync()
+ await flushMicrotasks()
+ await watcher.stop()
+ await vi.advanceTimersByTimeAsync(2500)
+ await resync
- watcher.stop()
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
})
it('periodic resync reuses external egress reconciliation for cached servers with bindings', async () => {
const watcher = new McpServerWatcher()
+ markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
netPol.reconcileExternalEgress.mockResolvedValue(undefined)
vi.spyOn(Math, 'random').mockReturnValue(0)
@@ -650,7 +5445,7 @@ describe('McpServerWatcher external egress retries', () => {
spec: serverObject.spec,
})
- await (watcher as any).runExternalEgressResync()
+ await (watcher as any).externalEgressCoordinator.runResync()
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
@@ -660,55 +5455,92 @@ describe('McpServerWatcher external egress retries', () => {
it('does not run parallel external egress reconciles for the same McpServer', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
const server = {
name: 'redis-tools',
namespace: 'mcp-server',
spec: serverObject.spec,
}
;(watcher as any).servers.set('redis-tools', server)
- ;(watcher as any).externalEgressInFlight.set(
- 'mcp-server/redis-tools',
- new Promise(() => undefined)
- )
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
+ })
vi.spyOn(Math, 'random').mockReturnValue(0)
- await (watcher as any).runExternalEgressResync()
+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await mutationStarted.promise
+ await (watcher as any).externalEgressCoordinator.runResync()
- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+ releaseMutation.resolve(undefined)
+ await active
watcher.stop()
})
it('waits for already in-flight external egress before reconciling the current server', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
const server = {
name: 'redis-tools',
namespace: 'mcp-server',
spec: serverObject.spec,
}
- let releaseExisting!: () => void
- const existing = new Promise<void>(resolve => {
- releaseExisting = resolve
+ ;(watcher as any).servers.set(server.name, server)
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
})
- ;(watcher as any).externalEgressInFlight.set('mcp-server/redis-tools', existing)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const run = (watcher as any).runExternalEgressOnce('MODIFIED', server)
- await Promise.resolve()
+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await mutationStarted.promise
+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await flushMicrotasks()
- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
expect(warn).toHaveBeenCalledWith(
'[K8s] Waiting for external egress reconcile for mcp-server/redis-tools; already in flight'
)
- releaseExisting()
- await expect(run).resolves.toBeUndefined()
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
+ releaseMutation.resolve(undefined)
+ await Promise.all([active, waiting])
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
warn.mockRestore()
watcher.stop()
})
+
+ it('does not start another external egress mutation after stop releases an in-flight wait', async () => {
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ const server = {
+ name: 'redis-tools',
+ namespace: 'mcp-server',
+ spec: serverObject.spec,
+ }
+ ;(watcher as any).servers.set(server.name, server)
+ netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
+ })
+
+ const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await mutationStarted.promise
+ const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await flushMicrotasks()
+ await watcher.stop()
+ releaseMutation.resolve(undefined)
+ await Promise.all([active, waiting])
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+ })
})
describe('McpServerWatcher.countCommunicationChannels', () => {
@@ -779,9 +5611,139 @@ describe('McpServerWatcher wires CC counter into HostReconciler', () => {
})
})
+describe('McpServerWatcher Host mutation dependency wiring', () => {
+ it('H2: wires a uid-guarded cache reflector into HostReconciler', () => {
+ const watcher = new McpServerWatcher()
+ const reflect = (
+ watcher.getHostReconciler() as unknown as {
+ _reflectHostOutcomeFn:
+ | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
+ | null
+ }
+ )._reflectHostOutcomeFn
+ if (!reflect) {
+ throw new Error('setReflectHostOutcome was never wired by the McpServerWatcher constructor')
+ }
+ const entry = {
+ name: 'h2-host',
+ namespace: 'mcp-host',
+ uid: 'uid-1',
+ spec: { host: 'h2-host', contextRef: 'c', secretRef: 's' },
+ } as unknown as HostCRD
+ ;(watcher as unknown as { hosts: Map<string, HostCRD> }).hosts.set('h2-host', entry)
+ const apply = vi.fn()
+ reflect('h2-host', 'uid-1', apply) // positive: current entry, matching uid
+ expect(apply).toHaveBeenCalledTimes(1)
+ expect(apply).toHaveBeenCalledWith(entry)
+ apply.mockClear()
+ reflect('h2-host', 'uid-2', apply) // recreation: same name, different uid → skip
+ expect(apply).not.toHaveBeenCalled()
+ reflect('h2-host', undefined, apply) // fail-closed on missing uid
+ expect(apply).not.toHaveBeenCalled()
+ reflect('absent-host', 'uid-1', apply) // absent entry → skip
+ expect(apply).not.toHaveBeenCalled()
+ })
+
+ it('tracks only the selected Context, referenced SFS state, and Host-scoped channels', () => {
+ const watcher = new McpServerWatcher()
+ const host = {
+ name: 'alpha-host',
+ namespace: 'mcp-host',
+ spec: { contextRef: 'alpha-context' },
+ } as HostCRD
+ const context = {
+ name: 'alpha-context',
+ namespace: 'mcp-server',
+ spec: {
+ contextId: 'alpha-context',
+ sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
+ },
+ }
+ const documentsV1 = {
+ name: 'documents',
+ namespace: 'mcp-host',
+ spec: { storageClassName: 'fast-v1' },
+ }
+ const alphaChannelV1 = {
+ name: 'telegram-alpha',
+ namespace: 'channels',
+ spec: { hostRef: 'alpha-host' },
+ }
+ ;(watcher as any).contexts.set(context.name, context)
+ ;(watcher as any).sharedFileSystems.set(documentsV1.name, documentsV1)
+ ;(watcher as any).communicationChannels.set(alphaChannelV1.name, alphaChannelV1)
+ ;(watcher as any).ccCacheSynced = true
+ ;(watcher as any).ccWatchGeneration = 7
+ const sfsReconciler = (watcher as any).sharedFileSystemReconciler
+ sfsReconciler.isMountable.mockReturnValue(true)
+ const hostReconciler = watcher.getHostReconciler() as any
+ const resolveDependencies = hostReconciler._hostMutationDependenciesFn
+
+ const original = resolveDependencies(host)
+ expect(original).toEqual([
+ { name: context.name, namespace: context.namespace, spec: context.spec },
+ true,
+ 7,
+ { name: documentsV1.name, namespace: documentsV1.namespace, spec: documentsV1.spec },
+ true,
+ {
+ name: alphaChannelV1.name,
+ namespace: alphaChannelV1.namespace,
+ spec: alphaChannelV1.spec,
+ },
+ ])
+ ;(watcher as any).sharedFileSystems.set('unrelated', {
+ name: 'unrelated',
+ namespace: 'mcp-host',
+ spec: {},
+ })
+ ;(watcher as any).communicationChannels.set('telegram-beta', {
+ name: 'telegram-beta',
+ namespace: 'channels',
+ spec: { hostRef: 'beta-host' },
+ })
+ const afterUnrelatedChanges = resolveDependencies(host)
+ expect(afterUnrelatedChanges).toEqual(original)
+
+ const documentsV2 = {
+ ...documentsV1,
+ spec: { storageClassName: 'fast-v2' },
+ }
+ ;(watcher as any).sharedFileSystems.set(documentsV2.name, documentsV2)
+ const afterSfsRevision = resolveDependencies(host)
+ expect(afterSfsRevision[3]).toEqual({
+ name: documentsV2.name,
+ namespace: documentsV2.namespace,
+ spec: documentsV2.spec,
+ })
+ expect(afterSfsRevision[3]).not.toEqual(original[3])
+
+ sfsReconciler.isMountable.mockReturnValue(false)
+ const afterMountabilityChange = resolveDependencies(host)
+ expect(afterMountabilityChange[4]).toBe(false)
+
+ const alphaChannelV2 = {
+ ...alphaChannelV1,
+ spec: {
+ ...alphaChannelV1.spec,
+ credentialsSecretRef: { name: 'telegram-alpha-v2' },
+ },
+ }
+ ;(watcher as any).communicationChannels.set(alphaChannelV2.name, alphaChannelV2)
+ const afterChannelRevision = resolveDependencies(host)
+ expect(afterChannelRevision.at(-1)).toEqual({
+ name: alphaChannelV2.name,
+ namespace: alphaChannelV2.namespace,
+ spec: alphaChannelV2.spec,
+ })
+ expect(afterChannelRevision.at(-1)).not.toEqual(original.at(-1))
+ })
+})
+
describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('re-reconciles the Host whose name matches the CC hostRef', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>
hosts.set('marketing', {
name: 'marketing',
@@ -800,6 +5762,7 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('is a no-op when the hostRef does not match any cached Host', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const reconcileSpy = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -811,6 +5774,7 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('reports reconcile errors without throwing so the watch can schedule recovery', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>
hosts.set('marketing', {
name: 'marketing',
@@ -828,6 +5792,7 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('schedules lifecycle convergence when a targeted CC event reconcile fails', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
;(watcher as any).hosts.set('marketing', {
name: 'marketing',
namespace: 'mcp-host',
@@ -896,8 +5861,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {
})
const watcher = new McpServerWatcher()
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
const ccWatchSpy = vi.spyOn(watcher as any, 'startCommunicationChannelWatch')
@@ -936,8 +5901,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {
const watcher = new McpServerWatcher()
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
const ccWatchSpy = vi
@@ -1005,7 +5970,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('partial-channel', {
name: 'partial-channel',
@@ -1045,7 +6010,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
if (plural === 'hosts') return { items: [host] }
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('watch-observed-channel', {
name: 'watch-observed-channel',
@@ -1086,7 +6051,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostReconcileHosts
.mockImplementationOnce(() => firstFleetPass.promise)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const failClosedPass = (watcher as any).requestHostFleetReconcile(
@@ -1162,7 +6127,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
})
const fleetFailure = new Error('fleet did not converge')
mocks.hostReconcileHosts.mockRejectedValueOnce(fleetFailure).mockResolvedValueOnce(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -1204,7 +6169,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
if (hostListAttempts === 1) throw inventoryFailure
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -1214,14 +6179,12 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
lifecycleGeneration
)
expect((watcher as any).ccAppliedLifecycleGeneration).not.toBe(lifecycleGeneration)
- expect((watcher as any).ccFleetRetryTimer).not.toBeNull()
expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(5000)
await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
expect(hostListAttempts).toBe(2)
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
- expect((watcher as any).ccFleetRetryTimer).toBeNull()
} finally {
errorSpy.mockRestore()
watcher.stop()
@@ -1237,7 +6200,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
mocks.hostFullReconcile.mockRejectedValueOnce(new HostFleetReconcileError([], [cleanupFailure]))
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -1249,7 +6212,6 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
)
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
- expect((watcher as any).ccFleetRetryTimer).toBeNull()
await vi.advanceTimersByTimeAsync(300000)
expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
} finally {
@@ -1270,7 +6232,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostFullReconcile
.mockRejectedValueOnce(new HostFleetReconcileError([hostFailure], [cleanupFailure]))
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
;(watcher as any).ccCacheSynced = true
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -1287,7 +6249,6 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
await vi.advanceTimersByTimeAsync(5000)
await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
- expect((watcher as any).ccFleetRetryTimer).toBeNull()
await (watcher as any).performHostResync()
expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2)
@@ -1312,7 +6273,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostReconcileHosts
.mockRejectedValueOnce(new Error('lifecycle pass failed'))
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
;(watcher as any).ccCacheSynced = true
@@ -1357,7 +6318,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
mocks.hostReconcileHosts.mockRejectedValue(new Error('persistent fleet failure'))
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -1403,15 +6364,13 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostReconcileHosts
.mockRejectedValueOnce(new Error('old generation failed'))
.mockResolvedValueOnce(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
await (watcher as any).requestHostFleetReconcile('fail-closed', staleGeneration)
- expect((watcher as any).ccFleetRetryTimer).not.toBeNull()
const currentGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
- expect((watcher as any).ccFleetRetryTimer).toBeNull()
await (watcher as any).requestHostFleetReconcile('recovered', currentGeneration)
expect(mocks.hostReconcileHosts).toHaveBeenCalledTimes(2)
@@ -1439,7 +6398,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
}
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('previous-channel', {
name: 'previous-channel',
@@ -1480,7 +6439,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const scheduleSpy = vi
.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery')
.mockImplementation(() => {})
@@ -1538,7 +6497,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
.mockImplementationOnce(() => firstHostConvergence.promise)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const firstRecovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>
try {
@@ -1596,7 +6555,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
}
)
.mockResolvedValue({ abort: vi.fn() })
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
try {
await expect((watcher as any).recoverCommunicationChannelCache()).resolves.toBe(false)
@@ -1629,7 +6588,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
mocks.watch.mockImplementationOnce(() => pendingWatch.promise)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const recovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>
try {
@@ -1674,7 +6633,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const scheduleSpy = vi
.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery')
.mockImplementation(() => {})
@@ -1736,9 +6695,9 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
maxActivePasses = Math.max(maxActivePasses, activePasses)
activePasses -= 1
})
- const watcher = new McpServerWatcher()
- vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
+ const watcher = newContextAuthoritativeWatcher()
+ stubAuthoritativeInventoryWatch(watcher, 'McpServer')
+ stubAuthoritativeInventoryWatch(watcher, 'Context')
vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(
@@ -1758,8 +6717,11 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()
expect(maxActivePasses).toBe(1)
- initialPass.resolve(undefined)
+ // Startup now resolves after safe bootstrap; the serialized fleet tail
+ // keeps running independently and must still drain the queued lifecycle pass.
await start
+ initialPass.resolve(undefined)
+ await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce()
@@ -1786,7 +6748,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { abort }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
;(watcher as any).ccCacheSynced = true
await (watcher as any).startCommunicationChannelWatch('504')
@@ -1820,7 +6782,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return requests[callbacks.length - 1]
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const scheduleSpy = vi
.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery')
.mockImplementation(() => {})
@@ -1871,7 +6833,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
const abort = vi.fn()
const watchRequest = { abort }
mocks.watch.mockReset().mockResolvedValue(watchRequest)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
await (watcher as any).startHostWatch('stream-rv')
await vi.advanceTimersByTimeAsync(30_001)
@@ -1906,9 +6868,11 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => activePass)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
;(watcher as any).ccCacheSynced = true
- ;(watcher as any).ccAppliedLifecycleGeneration = (watcher as any).ccLifecycleGeneration
+ ;(watcher as any).hostFleetScheduler.markLifecycleApplied(
+ (watcher as any).ccLifecycleGeneration
+ )
const first = (watcher as any).runHostResync() as Promise<void>
await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))
@@ -1927,9 +6891,53 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)
})
+ it('queues a trailing full Host pass when Context recovery lands during an active pass', async () => {
+ vi.clearAllMocks()
+ const activePass = deferred()
+ mocks.hostFullReconcile
+ .mockReset()
+ .mockImplementationOnce(() => activePass.promise)
+ .mockResolvedValue(undefined)
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'contexts') {
+ return {
+ metadata: { resourceVersion: 'recovered-context-rv' },
+ items: [
+ {
+ metadata: { name: 'recovered-context', namespace: 'mcp-server' },
+ spec: {
+ contextId: 'recovered-context',
+ mcpServers: [],
+ sharedFileSystems: [{ name: 'recovered-sfs' }],
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ })
+ mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
+ const watcher = newContextAuthoritativeWatcher()
+ ;(watcher as any).hostCacheSynced = true
+
+ const first = (watcher as any).requestHostFleetReconcile(
+ 'before Context recovery'
+ ) as Promise<void>
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))
+
+ await expect((watcher as any).recoverContextInventoryAndWatch()).resolves.toBe(true)
+ expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)
+
+ activePass.resolve(undefined)
+ await first
+ await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2))
+
+ await watcher.stop()
+ })
+
it('does not invent another fleet transition during a failed periodic cache recovery', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
;(watcher as any).ccCacheSynced = false
const recover = vi
.spyOn(watcher as any, 'recoverCommunicationChannelCache')
@@ -1945,6 +6953,33 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
watcher.stop()
})
+ it('coalesces the newest Host input revision into the serialized trailing pass', async () => {
+ vi.clearAllMocks()
+ const watcher = newContextAuthoritativeWatcher()
+ const activePass = deferred()
+ const requests: Array<{ inputRevision: number }> = []
+ vi.spyOn(watcher as any, 'performHostFleetReconcile').mockImplementation(
+ async (...args: unknown[]) => {
+ const request = args[0] as { inputRevision: number }
+ requests.push({ ...request })
+ if (requests.length === 1) await activePass.promise
+ }
+ )
+ ;(watcher as any).hostFleetScheduler.bumpInputRevision()
+ const first = (watcher as any).requestHostFleetReconcile('active') as Promise<void>
+ ;(watcher as any).hostFleetScheduler.bumpInputRevision()
+ const second = (watcher as any).requestHostFleetReconcile('pending-v2') as Promise<void>
+ ;(watcher as any).hostFleetScheduler.bumpInputRevision()
+ const third = (watcher as any).requestHostFleetReconcile('pending-v3') as Promise<void>
+
+ activePass.resolve(undefined)
+ await Promise.all([first, second, third])
+
+ expect(requests).toHaveLength(2)
+ expect(requests[1].inputRevision).toBe(3)
+ watcher.stop()
+ })
+
it('collapses overlapping fleet requests into one serialized trailing pass', async () => {
vi.clearAllMocks()
const fleetV1 = [
@@ -1988,7 +7023,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
maxActivePasses = Math.max(maxActivePasses, activePasses)
activePasses -= 1
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const first = (watcher as any).requestHostFleetReconcile(
'first',
@@ -2041,7 +7076,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
it('uses fresh lifecycle inventory without replacing newer Host watch-cache entries', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const initialHost = {
name: 'stateless-primary',
namespace: 'mcp-host',
@@ -2106,7 +7141,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => lifecyclePass)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()
const lifecycle = (watcher as any).requestHostFleetReconcile(
'CommunicationChannel interruption',
@@ -2151,7 +7186,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => lifecyclePass)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()
const lifecycle = (watcher as any).requestHostFleetReconcile(
'CommunicationChannel recovery',
@@ -2196,7 +7231,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => lifecyclePass)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const active = (watcher as any).requestHostFleetReconcile(
'CommunicationChannel interruption',
@@ -2243,7 +7278,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
})
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const full = (watcher as any).requestHostFleetReconcile(
'initial Host reconciliation',
@@ -2295,7 +7330,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return { items: [] }
})
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
@@ -2380,13 +7415,14 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return { items: [] }
})
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('transient event failure'))
.mockResolvedValue(undefined)
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
await hostWatchCallback('MODIFIED', {
metadata: { name: 'retry-before-list', namespace: 'mcp-host', generation: 1 },
@@ -2395,6 +7431,9 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
expect(reconcile).toHaveBeenCalledTimes(1)
expect((watcher as any).hostWatchRetryTimers.size).toBe(1)
+ // The active Host watch is then lost. A full request must recover a fresh
+ // snapshot and retire the failed event's pending retry.
+ ;(watcher as any).hostCacheSynced = false
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
expect((watcher as any).hostWatchRetryTimers.size).toBe(0)
@@ -2447,7 +7486,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
await fullReconcile.promise
order.push('full:end')
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const hostReconciler = watcher.getHostReconciler()
const reconcile = vi.spyOn(hostReconciler, 'reconcile').mockImplementation(async host => {
order.push(`watch:${host.name}:${host.generation}`)
@@ -2500,7 +7539,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
}
)
mocks.listNamespacedCustomObject.mockImplementation(async () => ({ items: [] }))
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const hostReconciler = watcher.getHostReconciler()
// The reconcile()/reconcileDelete() spies stand in for the per-Host chain;
// the actual serializeByHost non-overlap guarantee is proven in
@@ -2551,7 +7590,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const order: string[] = []
const timeout = new HostK8sRequestTimeoutError('AppsV1Api.readNamespacedDeployment', 30_000)
const reconcile = vi
@@ -2562,6 +7601,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
})
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
const first = hostWatchCallback('MODIFIED', {
metadata: { name: 'host-a', namespace: 'mcp-host', generation: 1 },
@@ -2597,7 +7637,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
if (plural === 'hosts') return { items: [] }
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -2646,7 +7686,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
// wins). Block that full pass so it stays running until shutdown unblocks
// the promoted trailing waiter — the semantic this test protects.
mocks.hostFullReconcile.mockReset().mockImplementation(() => trailingPass)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const activeRequest = (watcher as any).requestHostFleetReconcile(
'startup',
@@ -2692,7 +7732,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
resolveActivePass = resolve
})
mocks.hostReconcileHosts.mockReset().mockImplementationOnce(() => activePass)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const activeRequest = (watcher as any).requestHostFleetReconcile(
'active',
@@ -2731,7 +7771,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return inventory
})
mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const staleRequest = (watcher as any).requestHostFleetReconcile(
'stale transition',
@@ -2785,7 +7825,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
if (plural === 'hosts') return hostInventory
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const run = (watcher as any).requestHostFleetReconcile('shutdown race') as Promise<void>
await vi.waitFor(() => {
@@ -2810,7 +7850,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
it('releases the periodic single-flight guard after a rejected pass', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const failure = new Error('transient resync failure')
const performSpy = vi
.spyOn(watcher as any, 'performHostResync')
@@ -2843,12 +7883,189 @@ describe('McpServerWatcher Host watch generation', () => {
vi.useRealTimers()
})
+ it('folds a newer lifecycle generation into an in-flight Host inventory recovery', async () => {
+ const hostListStarted = deferred()
+ const hostList = deferred<{
+ metadata: { resourceVersion: string }
+ items: unknown[]
+ }>()
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListStarted.resolve(undefined)
+ return hostList.promise
+ }
+ return { items: [] }
+ })
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+
+ const recovery = (watcher as any).recoverHostInventoryAndWatch(
+ 'Host watch recovery convergence',
+ 1,
+ 'watch-recovery'
+ ) as Promise<unknown>
+ await hostListStarted.promise
+ const joined = (watcher as any).recoverHostInventoryAndWatch(
+ 'initial Host reconciliation',
+ 2,
+ 'cold-start'
+ ) as Promise<unknown>
+
+ expect(joined).toBe(recovery)
+ hostList.resolve({ metadata: { resourceVersion: 'joined-host-rv' }, items: [] })
+ await Promise.all([recovery, joined])
+
+ expect(fleetRequest).toHaveBeenCalledOnce()
+ expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')
+ await watcher.stop()
+ })
+
+ it('adopts a pending cold-start intent when a direct Host recovery starts first', async () => {
+ vi.useFakeTimers()
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ return { metadata: { resourceVersion: 'adopted-host-rv' }, items: [] }
+ }
+ return { items: [] }
+ })
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+
+ ;(watcher as any).scheduleHostCacheRecovery({
+ convergenceReason: 'initial Host reconciliation',
+ ccLifecycleGeneration: 2,
+ cause: 'cold-start',
+ })
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+
+ await (watcher as any).recoverHostInventoryAndWatch(
+ 'Host watch recovery convergence',
+ 1,
+ 'watch-recovery'
+ )
+
+ expect(mocks.listNamespacedCustomObject).toHaveBeenCalledTimes(1)
+ expect(fleetRequest).toHaveBeenCalledOnce()
+ expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
+ await watcher.stop()
+ })
+
+ it('folds a recovery intent scheduled while Host LIST is in flight into the successful pass', async () => {
+ vi.useFakeTimers()
+ const hostListStarted = deferred()
+ const hostList = deferred<{
+ metadata: { resourceVersion: string }
+ items: unknown[]
+ }>()
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural === 'hosts') {
+ hostListStarted.resolve(undefined)
+ return hostList.promise
+ }
+ return { items: [] }
+ })
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+
+ const recovery = (watcher as any).recoverHostInventoryAndWatch(
+ 'Host watch recovery convergence',
+ 1,
+ 'watch-recovery'
+ ) as Promise<unknown>
+ await hostListStarted.promise
+ ;(watcher as any).scheduleHostCacheRecovery({
+ convergenceReason: 'CommunicationChannel event Host convergence',
+ ccLifecycleGeneration: 3,
+ cause: 'cold-start',
+ })
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+ hostList.resolve({ metadata: { resourceVersion: 'in-flight-intent-rv' }, items: [] })
+ await recovery
+
+ expect(fleetRequest).toHaveBeenCalledOnce()
+ expect(fleetRequest).toHaveBeenCalledWith(
+ 'CommunicationChannel event Host convergence',
+ 3,
+ 'full'
+ )
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
+ await watcher.stop()
+ })
+
+ it('preserves an in-flight recovery intent across failure and the scheduled retry', async () => {
+ vi.useFakeTimers()
+ const firstHostListStarted = deferred()
+ const firstHostList = deferred<{
+ metadata: { resourceVersion: string }
+ items: unknown[]
+ }>()
+ let hostListCalls = 0
+ mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
+ if (plural !== 'hosts') return { items: [] }
+ hostListCalls += 1
+ if (hostListCalls === 1) {
+ firstHostListStarted.resolve(undefined)
+ return firstHostList.promise
+ }
+ return { metadata: { resourceVersion: 'retry-preserved-rv' }, items: [] }
+ })
+ const watcher = newContextAuthoritativeWatcher()
+ const fleetRequest = vi
+ .spyOn(watcher as any, 'requestHostFleetReconcile')
+ .mockResolvedValue(undefined)
+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+
+ const recovery = (watcher as any).recoverHostInventoryAndWatch(
+ 'Host watch recovery convergence',
+ 1,
+ 'watch-recovery'
+ ) as Promise<unknown>
+ await firstHostListStarted.promise
+ ;(watcher as any).scheduleHostCacheRecovery({
+ convergenceReason: 'CommunicationChannel event Host convergence',
+ ccLifecycleGeneration: 3,
+ cause: 'cold-start',
+ })
+ firstHostList.reject(new Error('transient Host LIST failure'))
+ await expect(recovery).rejects.toThrow('transient Host LIST failure')
+
+ expect((watcher as any).hostCacheRecoveryIntent).toEqual({
+ convergenceReason: 'CommunicationChannel event Host convergence',
+ ccLifecycleGeneration: 3,
+ cause: 'cold-start',
+ })
+ expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+
+ await vi.advanceTimersByTimeAsync(5_000)
+ await vi.waitFor(() => expect(fleetRequest).toHaveBeenCalledOnce())
+
+ expect(hostListCalls).toBe(2)
+ expect(fleetRequest).toHaveBeenCalledWith(
+ 'CommunicationChannel event Host convergence',
+ 3,
+ 'full'
+ )
+ expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
+ expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
+ errorSpy.mockRestore()
+ await watcher.stop()
+ })
+
it('rejects a Host snapshot that cannot anchor a continuing watch', async () => {
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'hosts') return { metadata: {}, items: [] }
return { items: [] }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
@@ -2884,7 +8101,7 @@ describe('McpServerWatcher Host watch generation', () => {
if (watchQueries.length === 1) throw new Error('watch connection failed')
return { abort: vi.fn() }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
@@ -2928,7 +8145,7 @@ describe('McpServerWatcher Host watch generation', () => {
if (path.endsWith('/hosts')) return lateRequest.promise
return { abort: vi.fn() }
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const requested = (watcher as any).requestHostFleetReconcile(
'Periodic resync',
@@ -2937,7 +8154,8 @@ describe('McpServerWatcher Host watch generation', () => {
) as Promise<void>
await vi.waitFor(() => expect(mocks.watch).toHaveBeenCalledOnce())
- const internal = (watcher as any).hostFleetReconcileInFlight.promise as Promise<void>
+ const internal = (watcher as any).hostFleetScheduler.hostFleetReconcileInFlight
+ .promise as Promise<void>
expect((watcher as any).hosts.has('snapshot-host')).toBe(true)
watcher.stop()
@@ -2970,7 +8188,7 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
await (watcher as any).startHostWatch('initial-rv')
;(watcher as any).hostCacheSynced = true
@@ -3010,7 +8228,7 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('transient event failure'))
@@ -3087,7 +8305,7 @@ describe('McpServerWatcher Host watch generation', () => {
mocks.hostFullReconcile
.mockImplementationOnce(() => firstFullReconcile.promise)
.mockResolvedValue(undefined)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const full = (watcher as any).requestHostFleetReconcile(
'Periodic resync',
undefined,
@@ -3153,7 +8371,7 @@ describe('McpServerWatcher Host watch generation', () => {
],
}
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
await (watcher as any).startHostWatch('test-host-rv')
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
@@ -3194,7 +8412,7 @@ describe('McpServerWatcher Host watch generation', () => {
],
}
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -3248,7 +8466,7 @@ describe('McpServerWatcher Host watch generation', () => {
],
}
})
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcileDelete = vi
.spyOn(watcher.getHostReconciler(), 'reconcileDelete')
.mockResolvedValue(undefined)
@@ -3270,7 +8488,8 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery diff enqueues an immediate per-Host delete for a disappeared Host (#827 item 1)', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
+ markHostInventoryAuthoritative(watcher)
const reconcileDelete = vi
.spyOn(watcher.getHostReconciler(), 'reconcileDelete')
.mockResolvedValue(undefined)
@@ -3301,7 +8520,8 @@ describe('McpServerWatcher Host watch generation', () => {
// short-circuits on snapshotNames.has(name) and never dispatches at all.
it('recovery delete carries a live-cache fence for a Host recreated during admission (#827 F2 parity)', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
+ markHostInventoryAuthoritative(watcher)
const reconcileDelete = vi
.spyOn(watcher.getHostReconciler(), 'reconcileDelete')
.mockResolvedValue(undefined)
@@ -3335,7 +8555,8 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery diff reconciles (does NOT delete) a same-name Host recreated with a new uid (#827 items 1+2)', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
+ markHostInventoryAuthoritative(watcher)
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -3371,7 +8592,7 @@ describe('McpServerWatcher Host watch generation', () => {
})
it('suppresses a watch event whose identity (uid) differs from the cached Host (#827 item 2)', () => {
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
;(watcher as any).hosts.set('x', {
name: 'x',
namespace: 'mcp-host',
@@ -3400,7 +8621,8 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery urgently admits a wake-pending Host that is otherwise unchanged (§10.2 step 7 wake branch)', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
+ markHostInventoryAuthoritative(watcher)
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -3442,7 +8664,7 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery does NOT urgently dispatch a fully-unchanged Host with no wake annotation', async () => {
vi.clearAllMocks()
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -3490,13 +8712,14 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('transient'))
.mockResolvedValue(undefined)
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
await hostWatchCallback('MODIFIED', {
metadata: { name: 'retry-lane-host', namespace: 'mcp-host', generation: 2 },
@@ -3532,7 +8755,7 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const unrelatedHost = {
name: 'unrelated-host',
namespace: 'mcp-host',
@@ -3547,6 +8770,7 @@ describe('McpServerWatcher Host watch generation', () => {
await (watcher as any).startHostWatch('test-host-rv')
const watchGeneration = (watcher as any).hostWatchGeneration
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
await hostWatchCallback('ADDED', {
metadata: { name: 'retry-host', namespace: 'mcp-host', generation: 1 },
@@ -3582,12 +8806,13 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValue(new Error('persistent failure'))
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
await hostWatchCallback('ADDED', {
metadata: { name: 'bounded-host', namespace: 'mcp-host', generation: 1 },
@@ -3617,13 +8842,14 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = new McpServerWatcher()
+ const watcher = newContextAuthoritativeWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('stale event failed'))
.mockResolvedValue(undefined)
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
+ markHostInventoryAuthoritative(watcher)
await hostWatchCallback('ADDED', {
metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },
@@ -3645,6 +8871,7 @@ describe('McpServerWatcher Host watch generation', () => {
describe('McpServerWatcher.startCommunicationChannelWatch', () => {
it('caches ADDED events and triggers Host reconcile', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
name: 'marketing',
@@ -3668,6 +8895,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
it('removes from cache on DELETED and triggers Host reconcile', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-marketing', {
name: 'cc-marketing',
@@ -3696,6 +8924,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-1', {
name: 'cc-1',
@@ -3760,6 +8989,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revision rolls', () => {
it('on CC ADDED, patches the new host channel-reader revision annotation', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
name: 'marketing',
@@ -3782,6 +9012,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('preserves credentialsSecretRef from the watch event into the cache', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
@@ -3812,6 +9043,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('on CC MODIFIED with hostRef change, patches BOTH old and new hosts', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-1', {
name: 'cc-1',
@@ -3846,6 +9078,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('on CC DELETED, patches the (former) host channel-reader revision annotation', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-marketing', {
name: 'cc-marketing',
@@ -3874,6 +9107,7 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('swallows patchChannelReaderRevisionAnnotation errors without breaking the watch loop', async () => {
const watcher = new McpServerWatcher()
+ markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
name: 'marketing',
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 7fe7ceaec..1dc96adcf 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -14,9 +14,19 @@ import {
} from './administrativeOutcomeReporter'
import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'
import { config } from './config'
+import {
+ ExternalEgressConvergenceCoordinator,
+ type ExternalEgressRetryHandle,
+ type ExternalEgressWatchEventType,
+} from './externalEgressConvergenceCoordinator'
import { gfsDefaultFactoryConfig } from './gfsConfig'
import { GfsReconciler } from './gfsReconciler'
import { ControlApiGfsSeedClient } from './gfsSeedClient'
+import {
+ type HostFleetReconcileMode,
+ type HostFleetReconcileRequest,
+ HostFleetScheduler,
+} from './hostFleetScheduler'
import {
HostFleetReconcileError,
type HostReconcileSource,
@@ -30,8 +40,19 @@ import {
import { K8sGfsApi } from './k8s/gfsK8sApi'
import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'
import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'
-import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'
-import { NetworkPolicyReconciler } from './networkPolicyReconciler'
+import {
+ confirmAuthoritativeMcpServerAbsence,
+ isMcpServerStatusOnlyUpdate,
+ sameMcpServerDesiredRevision,
+} from './mcpServerSafety'
+import {
+ hostDeleteCleanupTotal,
+ hostFleetRequestsTotal,
+ hostWatchRecoverySeconds,
+ initialConvergenceLastSuccessTimestampSeconds,
+ initialConvergenceRetriesTotal,
+} from './metrics'
+import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'
import { McpServerReconciler } from './reconciler'
import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'
import {
@@ -48,6 +69,7 @@ import {
SharedFileSystemCRD,
SharedFileSystemSpec,
} from './types'
+import { getErrorCode } from './utils'
// Only initialize K8s client if not in dev mode
let customObjectsApi: k8s.CustomObjectsApi | null = null
@@ -83,30 +105,13 @@ const PLURAL_HOSTS = 'hosts'
const PLURAL_SHAREDFILESYSTEMS = 'sharedfilesystems'
const PLURAL_GLOBALFILESYSTEMS = 'globalfilesystems'
const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'
-const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000]
-const EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY = 10
-const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000
+// Before readiness was decoupled, a failed initial McpServer or NetworkPolicy
+// sweep made provider.start() fail and Kubernetes restarted HCC. These retries
+// retain that convergence guarantee now that the sweeps run in the background.
+const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
+const INVENTORY_CACHE_RECOVERY_RETRY_MS = 5000
-/**
- * External-egress resync delay (ms), H2 (issue #299). The configured interval is
- * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a
- * rotating low-TTL host is sampled every rotation. The floor is the hard lower
- * bound to avoid a hot-loop. Pure and unit-testable.
- */
-export function externalEgressResyncDelayMs(
- intervalSec: number,
- floorSec: number,
- minObservedTtlMs = Infinity
-): number {
- const floorMs = floorSec * 1000
- const configuredMs = intervalSec * 1000
- const ttlAwareMs = Number.isFinite(minObservedTtlMs)
- ? Math.min(configuredMs, minObservedTtlMs / 2)
- : configuredMs
- return Math.max(ttlAwareMs, floorMs)
-}
const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000
-const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
const HOST_CACHE_RECOVERY_RETRY_MS = 5000
const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]
// Wake-pending Hosts get immediate per-Host admission after watch recovery
@@ -118,57 +123,92 @@ type CommunicationChannelSnapshot = {
resourceVersion?: string
}
+type McpServerSnapshot = {
+ servers: McpServerCRD[]
+ resourceVersion?: string
+}
+
+type ContextSnapshot = {
+ contexts: ContextCRD[]
+ resourceVersion?: string
+}
+
type HostSnapshot = {
hosts: HostCRD[]
resourceVersion?: string
}
-type HostFleetReconcileMode = 'full' | 'lifecycle'
+type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'
type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
+type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'
-type HostFleetReconcileRequest = {
- reason: string
- mode: HostFleetReconcileMode
- ccLifecycleGeneration?: number
+type NetworkPolicySafetyCertificate = {
+ contextGeneration: number
+ serverGeneration: number
+ contextRevision: number
+ serverRevision: number
}
-type PendingHostFleetReconcile = HostFleetReconcileRequest & {
+type ActiveInitialConvergenceRun = {
+ trailingRequested: boolean
promise: Promise<void>
- resolve: () => void
}
-type ActiveHostFleetReconcile = HostFleetReconcileRequest & { promise: Promise<void> }
+type HostInventoryRecoveryRequest = {
+ convergenceReason: string
+ ccLifecycleGeneration?: number
+ cause: HostInventoryRecoveryCause
+}
+
+type ActiveHostInventoryRecovery = {
+ request: HostInventoryRecoveryRequest
+ promise: Promise<HostCRD[]>
+}
-function hostFleetRequestCovers(
- active: HostFleetReconcileRequest,
- requested: HostFleetReconcileRequest
-): boolean {
- const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'
- if (!modeCovered) return false
- if (requested.ccLifecycleGeneration === undefined) return true
- return active.ccLifecycleGeneration === requested.ccLifecycleGeneration
+function mergeHostInventoryRecoveryRequest(
+ target: HostInventoryRecoveryRequest,
+ requested: HostInventoryRecoveryRequest
+): void {
+ // Cold start is the stronger admission mode: its snapshot is consumed only
+ // by the bounded full pass, whereas watch recovery also dispatches urgent
+ // per-Host work. Preserve that mode when either joined caller requires it.
+ if (requested.cause === 'cold-start' || target.cause !== 'cold-start') {
+ target.convergenceReason = requested.convergenceReason
+ }
+ if (target.cause === 'cold-start' || requested.cause === 'cold-start') {
+ target.cause = 'cold-start'
+ }
+ if (requested.ccLifecycleGeneration !== undefined) {
+ target.ccLifecycleGeneration =
+ target.ccLifecycleGeneration === undefined
+ ? requested.ccLifecycleGeneration
+ : Math.max(target.ccLifecycleGeneration, requested.ccLifecycleGeneration)
+ }
}
-function mergeHostFleetRequests(
- pending: HostFleetReconcileRequest,
- requested: HostFleetReconcileRequest
-): HostFleetReconcileRequest {
- const pendingGeneration = pending.ccLifecycleGeneration
- const requestedGeneration = requested.ccLifecycleGeneration
- return {
- reason: requested.reason,
- mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',
- ccLifecycleGeneration:
- pendingGeneration === undefined
- ? requestedGeneration
- : requestedGeneration === undefined
- ? pendingGeneration
- : Math.max(pendingGeneration, requestedGeneration),
+/**
+ * Serializes a reconciliation domain without turning a rejected operation into
+ * a permanent queue failure. Kubernetes Watch invokes async callbacks without
+ * awaiting the preceding callback, while a full reconciliation can delete
+ * orphaned resources owned by the same domain. A single ordered effect stream
+ * therefore makes a later watch event the final writer after an older sweep.
+ */
+class SerializedReconciliationQueue {
+ private tail: Promise<void> = Promise.resolve()
+
+ enqueue(work: () => Promise<void>): Promise<void> {
+ const result = this.tail.then(work)
+ this.tail = result.then(
+ () => undefined,
+ () => undefined
+ )
+ return result
}
}
-function delay(ms: number): Promise<void> {
- return new Promise(resolve => setTimeout(resolve, ms))
+type KeyedReconciliationQueue = {
+ queue: SerializedReconciliationQueue
+ references: number
}
type McpServerWatchObject = {
@@ -205,7 +245,7 @@ export interface McpServerProvider {
/**
* List all McpServer CRDs in the namespace.
*/
-export async function listAllMcpServers(): Promise<McpServerCRD[]> {
+async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
@@ -221,6 +261,7 @@ export async function listAllMcpServers(): Promise<McpServerCRD[]> {
})
const list = response as {
+ metadata?: { resourceVersion?: string }
items: Array<{
metadata: {
name: string
@@ -247,17 +288,21 @@ export async function listAllMcpServers(): Promise<McpServerCRD[]> {
}))
console.log(`[K8s] Found ${servers.length} McpServer(s)`)
- return servers
+ return { servers, resourceVersion: list.metadata?.resourceVersion }
} catch (error) {
console.error('[K8s] Failed to list McpServers:', error)
throw error
}
}
+export async function listAllMcpServers(): Promise<McpServerCRD[]> {
+ return (await listMcpServerSnapshot()).servers
+}
+
/**
* List all Context CRDs in the namespace.
*/
-export async function listAllContexts(): Promise<ContextCRD[]> {
+async function listContextSnapshot(): Promise<ContextSnapshot> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
@@ -273,8 +318,14 @@ export async function listAllContexts(): Promise<ContextCRD[]> {
})
const list = response as {
+ metadata?: { resourceVersion?: string }
items: Array<{
- metadata: { name: string; namespace?: string }
+ metadata: {
+ name: string
+ namespace?: string
+ uid?: string
+ generation?: number
+ }
spec: ContextSpec
}>
}
@@ -282,17 +333,23 @@ export async function listAllContexts(): Promise<ContextCRD[]> {
const contexts = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.namespace,
+ uid: item.metadata.uid,
+ generation: item.metadata.generation,
spec: item.spec,
}))
console.log(`[K8s] Found ${contexts.length} Context(s)`)
- return contexts
+ return { contexts, resourceVersion: list.metadata?.resourceVersion }
} catch (error) {
console.error('[K8s] Failed to list Contexts:', error)
throw error
}
}
+export async function listAllContexts(): Promise<ContextCRD[]> {
+ return (await listContextSnapshot()).contexts
+}
+
/**
* List all Host CRDs in host namespace.
*/
@@ -399,12 +456,16 @@ export async function getAuthToken(
* SharedFileSystems are allowed to live in, per CRD validation).
*/
export async function listAllSharedFileSystems(): Promise<SharedFileSystemCRD[]> {
- if (!customObjectsApi) {
+ if (!hostCustomObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all SharedFileSystems in namespace ${config.hostNamespace}`)
- const response = await customObjectsApi.listNamespacedCustomObject({
+ // Deadline-bearing client, like listHostSnapshot: the cold-start Host fleet
+ // pass waits on this inventory, so an apiserver that never answers would
+ // otherwise strand the fleet behind an already-certified readiness — and
+ // leave the SharedFileSystem watch, which starts after this await, dead.
+ const response = await hostCustomObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.hostNamespace,
@@ -446,12 +507,17 @@ export async function listAllGlobalFileSystems(): Promise<GlobalFileSystemCRD[]>
plural: PLURAL_GLOBALFILESYSTEMS,
})
const list = response as {
- items: Array<{ metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }>
+ items: Array<{
+ metadata: { name: string; namespace?: string }
+ spec: GlobalFileSystemSpec
+ status?: GlobalFileSystemCRD['status']
+ }>
}
return list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || namespace,
spec: item.spec,
+ status: item.status,
}))
} catch (error) {
console.error('[K8s] Failed to list GlobalFileSystems:', error)
@@ -552,7 +618,7 @@ export async function getContext(contextId: string): Promise<ContextCRD | null>
return null
}
console.error(`[K8s] Failed to read Context CRD:`, error)
- return null
+ throw error
}
}
@@ -572,7 +638,26 @@ export class McpServerWatcher implements McpServerProvider {
private hosts: Map<string, HostCRD> = new Map()
private contexts: Map<string, ContextCRD> = new Map()
private sharedFileSystems: Map<string, SharedFileSystemCRD> = new Map()
+ private globalFileSystems: Map<string, GlobalFileSystemCRD> = new Map()
private communicationChannels: Map<string, CommunicationChannelCRD> = new Map()
+ private mcpWatchGeneration = 0
+ private contextWatchGeneration = 0
+ private mcpServerDesiredRevision = 0
+ private contextDesiredRevision = 0
+ // Readiness covers authoritative revocation, not additive fleet completion.
+ // Generations bind the completed safety sweep to the exact LIST -> WATCH
+ // pair that supplied its absence decisions. A recovered watch invalidates
+ // this marker until the new inventory's orphan-allow sweep completes.
+ private networkPolicyRevocationContextGeneration = 0
+ private networkPolicyRevocationServerGeneration = 0
+ private networkPolicyRevocationContextRevision = 0
+ private networkPolicyRevocationServerRevision = 0
+ private mcpServerCacheSynced = false
+ private contextCacheSynced = false
+ private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
+ private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
+ private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null
+ private contextCacheRecoveryInFlight: Promise<boolean> | null = null
// A watch callback can still settle after its stream reports completion.
// Host reconciles change the mcp-host pod template, so stale callbacks must
// never replay an older Host spec after a replacement watch is active.
@@ -585,11 +670,18 @@ export class McpServerWatcher implements McpServerProvider {
private readonly hostWatchRetryAttempts = new Map<string, number>()
private hostCacheSynced = false
private hostCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
+ private sfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null
+ private gfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null
// Host watch LIST-to-WATCH recovery is a dedicated, deduplicated operation
// (§10.2). Concurrent recovery signals reuse the in-flight promise so exactly
// one LIST + WATCH is installed. Independent per-Host events no longer share a
// process-wide convergence tail — they enter the per-Host serializer directly.
- private hostRecoveryInFlight: Promise<HostCRD[]> | null = null
+ private hostRecoveryInFlight: ActiveHostInventoryRecovery | null = null
+ private hostCacheRecoveryIntent: HostInventoryRecoveryRequest | null = null
+ // Only the cold-start Host fleet pass waits for the initial SFS inventory.
+ // The inventory itself remains asynchronous with respect to provider.start
+ // and NetworkPolicy readiness certification.
+ private initialHostFleetSfsInventory: Promise<boolean> | null = null
// B2: tracks whether the CC snapshot is paired with a continuing watch.
// Set to true only while a complete snapshot is paired with a live watch;
// used by HostReconciler to make fail-closed lifecycle decisions.
@@ -597,16 +689,16 @@ export class McpServerWatcher implements McpServerProvider {
private ccWatchGeneration = 0
private ccCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private ccCacheRecoveryInFlight: Promise<boolean> | null = null
- private ccLifecycleGeneration = 0
- private ccAppliedLifecycleGeneration = -1
- private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null
- private ccFleetRetryAttempt = 0
- private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null
- private hostFleetReconcilePending: PendingHostFleetReconcile | null = null
- private resolveHostFleetShutdown: () => void = () => {}
- private readonly hostFleetShutdown = new Promise<void>(resolve => {
- this.resolveHostFleetShutdown = resolve
+ private readonly hostFleetScheduler = new HostFleetScheduler({
+ perform: request => this.performHostFleetReconcile(request),
+ recordRequest: result => hostFleetRequestsTotal.inc({ result }),
})
+ private get ccLifecycleGeneration(): number {
+ return this.hostFleetScheduler.currentLifecycleGeneration
+ }
+ private get ccAppliedLifecycleGeneration(): number {
+ return this.hostFleetScheduler.appliedLifecycleGeneration
+ }
private hostResyncInFlight: Promise<void> | null = null
private changeCallback?: () => void
private stopped = false
@@ -618,15 +710,46 @@ export class McpServerWatcher implements McpServerProvider {
private gfsReconciler: GfsReconciler
private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
private readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter
- private readonly externalEgressRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()
- private readonly externalEgressRetryAttempts = new Map<string, number>()
- private readonly externalEgressInFlight = new Map<string, Promise<void>>()
+ private readonly externalEgressCoordinator = new ExternalEgressConvergenceCoordinator({
+ listServers: () => [...this.servers.values()],
+ getCurrentServer: name => this.servers.get(name),
+ inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,
+ sameDesiredRevision: sameMcpServerDesiredRevision,
+ enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),
+ mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),
+ replay: (type, server, retry) =>
+ this.reconcileMcpServerWatchEvent(type, server, this.mcpWatchGeneration, retry),
+ externalEgressRefreshMinTtlMs: () => this.netPolReconciler.externalEgressRefreshMinTtlMs,
+ })
+ private readonly initialConvergenceRetryTimers = new Map<
+ InitialConvergenceLane,
+ ReturnType<typeof setTimeout>
+ >()
+ private readonly initialConvergenceRetryAttempts = new Map<InitialConvergenceLane, number>()
+ private readonly initialConvergenceRuns = new Map<
+ InitialConvergenceLane,
+ ActiveInitialConvergenceRun
+ >()
+ // Every resource effect serializes by resource identity, so one slow fleet
+ // member cannot block unrelated live events while same-resource updates
+ // still retain last-writer ordering.
+ private readonly mcpServerReconciliationQueues = new Map<string, KeyedReconciliationQueue>()
+ private readonly contextReconciliationQueues = new Map<string, KeyedReconciliationQueue>()
+ private readonly sharedFileSystemReconciliationQueues = new Map<
+ string,
+ KeyedReconciliationQueue
+ >()
+ private readonly globalFileSystemReconciliationQueues = new Map<
+ string,
+ KeyedReconciliationQueue
+ >()
+ private sharedFileSystemCacheRevision = 0
+ private globalFileSystemCacheRevision = 0
// Periodic resync timer: K8s watches can drop events on long disconnects;
// running fullReconcile every N minutes guarantees mcp-host-runtime-token Secret
// rotation eventually catches up even after a missed MODIFIED event.
// Disabled when interval <= 0 (tests).
private resyncTimer: ReturnType<typeof setInterval> | null = null
- private externalEgressResyncTimer: ReturnType<typeof setTimeout> | null = null
// Periodic SharedFileSystem resync (#592): the SFS watch fires only on SFS CRD
// changes, not on PVC binding / wfc pod readiness, so a SharedFileSystem that
// reported Initializing/Degraded needs a periodic re-reconcile to converge to a
@@ -644,6 +767,11 @@ export class McpServerWatcher implements McpServerProvider {
}
this.watch = new k8s.Watch(kc)
this.reconciler = new McpServerReconciler(kc)
+ this.reconciler.setInventoryAuthority(() => ({
+ known: this.mcpServerCacheSynced,
+ generation: this.mcpWatchGeneration,
+ }))
+ this.reconciler.setResolveCurrentServer(name => this.servers.get(name))
const infrastructureTelemetryReporter = config.controlApiBaseUrl
? createInfrastructureTelemetryReporter(config.governedTracingEnabled, {
baseUrl: config.controlApiBaseUrl,
@@ -693,6 +821,14 @@ export class McpServerWatcher implements McpServerProvider {
// entries a delete/newer event has since removed), and orphan cleanup
// compares candidates with the CURRENT inventory rather than a pass snapshot.
this.hostReconciler.setResolveCurrentHost(name => this.hosts.get(name))
+ // H2: reflect committed lifecycle outcomes onto the CURRENT cache entry —
+ // but never onto a same-name recreation. uid === undefined fails closed
+ // (production watch objects always carry uid).
+ this.hostReconciler.setReflectHostOutcome((name, uid, apply) => {
+ const cached = this.hosts.get(name)
+ if (!cached || uid === undefined || cached.uid !== uid) return
+ apply(cached)
+ })
// Orphan cleanup requires known watch authority and a stable watch
// generation; expose both so cleanup fail-closes while authority is unknown
// or the watch has been retired mid-pass.
@@ -700,6 +836,40 @@ export class McpServerWatcher implements McpServerProvider {
known: this.hostCacheSynced,
generation: this.hostWatchGeneration,
}))
+ this.hostReconciler.setHostMutationAuthority(() => ({
+ known: this.hostCacheSynced && this.contextCacheSynced,
+ hostGeneration: this.hostWatchGeneration,
+ contextGeneration: this.contextWatchGeneration,
+ }))
+ this.hostReconciler.setResolveHostMutationDependencies(host => {
+ const context = this.contexts.get(host.spec.contextRef)
+ // Lease desired semantics, not cache-object identity or observed status.
+ // Kubernetes watch events replace objects and status-only changes may
+ // legitimately advance resourceVersion without changing Host inputs.
+ const desiredRevision = <T extends { name: string; namespace: string; spec: unknown }>(
+ resource: T | undefined
+ ) =>
+ resource
+ ? { name: resource.name, namespace: resource.namespace, spec: resource.spec }
+ : undefined
+ const dependencies: unknown[] = [
+ desiredRevision(context),
+ this.ccCacheSynced,
+ this.ccWatchGeneration,
+ ]
+ for (const ref of context?.spec.sharedFileSystems ?? []) {
+ const sfs = this.sharedFileSystems.get(ref.name)
+ dependencies.push(
+ desiredRevision(sfs),
+ sfs ? this.sharedFileSystemReconciler.isMountable(sfs) : false
+ )
+ }
+ const channels = this.findCommunicationChannelsByHostRef(host.name).sort((a, b) =>
+ a.name.localeCompare(b.name)
+ )
+ dependencies.push(...channels.map(channel => desiredRevision(channel)))
+ return dependencies
+ })
// Per-CC credentials Secret migration: HostReconciler computes the
// channel-reader credentials-revision annotation by hashing the Secret(s)
// referenced by all CCs of a host. Wire both finders so
@@ -728,65 +898,487 @@ export class McpServerWatcher implements McpServerProvider {
return this.ccCacheSynced
}
- private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {
- this.communicationChannels.clear()
- for (const channel of snapshot.channels) {
- this.communicationChannels.set(channel.name, channel)
+ /**
+ * Readiness authority covers the inventories that back HCC discovery,
+ * Context policy decisions, and Host API/runtime ownership. Communication
+ * Channel loss is intentionally excluded: that subsystem already fails safe
+ * by preserving replicas and disabling stateless suspension until recovery.
+ * SFS/GFS retain their established eventual-resync contract.
+ */
+ isReadinessInventoryAuthoritative(): boolean {
+ return (
+ !this.stopped &&
+ this.mcpServerCacheSynced &&
+ this.contextCacheSynced &&
+ this.hostCacheSynced &&
+ // The generation/revision equalities below prove no newer desired state
+ // is outstanding; they cannot prove the last pass actually revoked what
+ // it set out to revoke. Losing a delete fence bumps no generation and
+ // moves no revision, so it is invisible to every other term here while
+ // leaving live an allow the pass classified as stale. The reconciler
+ // degrades this fence at the point of loss; readiness reads it so the
+ // gate re-closes for the same reason it opened.
+ this.netPolReconciler.hasCertifiedSafetyInventory() &&
+ this.networkPolicyRevocationContextGeneration === this.contextWatchGeneration &&
+ this.networkPolicyRevocationServerGeneration === this.mcpWatchGeneration &&
+ this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&
+ this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision
+ )
+ }
+
+ /**
+ * Host-managed runtime mutation requires a LIST snapshot paired with its
+ * continuing WATCH. This narrower predicate is also used by non-HTTP effect
+ * producers while the broader readiness predicate additionally covers
+ * McpServer and Context discovery authority.
+ */
+ isHostInventoryAuthoritative(): boolean {
+ return !this.stopped && this.hostCacheSynced
+ }
+
+ /**
+ * Host runtime state is derived from both the Host and Context inventories:
+ * Context owns the SharedFileSystem mount set rendered into each Host pod.
+ * Mutation must therefore stop when either LIST -> WATCH pair is unavailable.
+ */
+ private isHostEffectInventoryAuthoritative(): boolean {
+ return this.isHostInventoryAuthoritative() && this.contextCacheSynced
+ }
+
+ /**
+ * Fail closed at every Host-effect admission boundary while recovery owns
+ * inventory authority. Preserve the reason/generation in the retry intent so
+ * the successful LIST -> WATCH recovery's full pass applies the deferred
+ * effect from current state instead of acting on the stale cache.
+ */
+ private admitHostDependentEffects(
+ convergenceReason: string,
+ ccLifecycleGeneration?: number
+ ): boolean {
+ if (this.isHostEffectInventoryAuthoritative()) return true
+ if (!this.stopped && !this.hostCacheSynced) {
+ this.scheduleHostCacheRecovery({
+ convergenceReason,
+ ccLifecycleGeneration,
+ cause: 'watch-recovery',
+ })
}
+ return false
}
- private beginCommunicationChannelLifecycleTransition(): number {
- this.ccLifecycleGeneration += 1
- this.ccFleetRetryAttempt = 0
- if (this.ccFleetRetryTimer) {
- clearTimeout(this.ccFleetRetryTimer)
- this.ccFleetRetryTimer = null
+ private retireMcpServerWatch(expectedGeneration?: number): boolean {
+ if (expectedGeneration !== undefined && expectedGeneration !== this.mcpWatchGeneration) {
+ return false
}
- return this.ccLifecycleGeneration
+ this.mcpServerCacheSynced = false
+ this.mcpWatchGeneration += 1
+ this.mcpWatchRequest?.abort()
+ this.mcpWatchRequest = null
+ return true
}
- private markCommunicationChannelLifecycleApplied(generation: number): void {
- if (generation !== this.ccLifecycleGeneration) return
- this.ccAppliedLifecycleGeneration = generation
- this.ccFleetRetryAttempt = 0
- if (this.ccFleetRetryTimer) {
- clearTimeout(this.ccFleetRetryTimer)
- this.ccFleetRetryTimer = null
+ private retireContextWatch(expectedGeneration?: number): boolean {
+ if (expectedGeneration !== undefined && expectedGeneration !== this.contextWatchGeneration) {
+ return false
}
+ this.contextCacheSynced = false
+ this.contextWatchGeneration += 1
+ this.ctxWatchRequest?.abort()
+ this.ctxWatchRequest = null
+ return true
}
- private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {
- const generation = request.ccLifecycleGeneration
- if (generation === undefined) return
+ private async enqueueKeyedReconciliation(
+ queues: Map<string, KeyedReconciliationQueue>,
+ key: string,
+ work: () => Promise<void>
+ ): Promise<void> {
+ let entry = queues.get(key)
+ if (!entry) {
+ entry = { queue: new SerializedReconciliationQueue(), references: 0 }
+ queues.set(key, entry)
+ }
+ entry.references++
+ try {
+ await entry.queue.enqueue(async () => {
+ if (this.stopped) return
+ await work()
+ })
+ } finally {
+ entry.references--
+ if (entry.references === 0 && queues.get(key) === entry) queues.delete(key)
+ }
+ }
+
+ private enqueueMcpServerReconciliation(
+ server: Pick<McpServerCRD, 'name' | 'namespace'>,
+ work: () => Promise<void>
+ ): Promise<void> {
+ return this.enqueueKeyedReconciliation(
+ this.mcpServerReconciliationQueues,
+ `${server.namespace}/${server.name}`,
+ work
+ )
+ }
+
+ private enqueueContextReconciliation(
+ contextId: string,
+ work: () => Promise<void>
+ ): Promise<void> {
+ return this.enqueueKeyedReconciliation(this.contextReconciliationQueues, contextId, work)
+ }
+
+ private enqueueContextIdentityReconciliation(
+ contextIds: string[],
+ work: () => Promise<void>
+ ): Promise<void> {
+ const keys = [...new Set(contextIds)].sort()
+ const enter = (index: number): Promise<void> => {
+ const key = keys[index]
+ if (!key) return work()
+ return this.enqueueContextReconciliation(key, () => enter(index + 1))
+ }
+ return enter(0)
+ }
+
+ private hasMcpServerInventoryAuthority(watchGeneration: number): boolean {
+ return this.mcpServerCacheSynced && this.mcpWatchGeneration === watchGeneration
+ }
+
+ private hasContextInventoryAuthority(watchGeneration: number): boolean {
+ return this.contextCacheSynced && this.contextWatchGeneration === watchGeneration
+ }
+
+ private async mcpServerAbsentForDelete(
+ name: string,
+ namespace: string,
+ watchGeneration: number
+ ): Promise<boolean> {
+ if (!customObjectsApi) return false
+ return confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => this.hasMcpServerInventoryAuthority(watchGeneration),
+ resolveCurrent: () => this.servers.get(name),
+ readCurrent: () =>
+ customObjectsApi.getNamespacedCustomObject({
+ group: GROUP,
+ version: VERSION,
+ namespace,
+ plural: PLURAL_MCPSERVERS,
+ name,
+ }),
+ })
+ }
+
+ private async authorizeMcpServerDeleteOrRetry(
+ server: McpServerCRD,
+ watchGeneration: number
+ ): Promise<boolean> {
+ try {
+ const authorized = await this.mcpServerAbsentForDelete(
+ server.name,
+ server.namespace,
+ watchGeneration
+ )
+ if (!authorized && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
+ console.warn(
+ `[K8s] McpServer inventory authority changed while deleting ${server.name}; ` +
+ 'full cleanup deferred until retry'
+ )
+ this.scheduleExternalEgressRetry('DELETED', server)
+ this.changeCallback?.()
+ }
+ return authorized
+ } catch (error) {
+ console.error(
+ `[K8s] Authoritative absence check failed for deleted McpServer ${server.name}; ` +
+ 'cleanup blocked until retry:',
+ error
+ )
+ this.scheduleExternalEgressRetry('DELETED', server)
+ this.changeCallback?.()
+ return false
+ }
+ }
+
+ private async contextAbsentForDelete(
+ name: string,
+ namespace: string,
+ watchGeneration: number
+ ): Promise<boolean> {
+ if (
+ !this.contextCacheSynced ||
+ this.contextWatchGeneration !== watchGeneration ||
+ this.contexts.has(name) ||
+ !customObjectsApi
+ ) {
+ return false
+ }
+ try {
+ await customObjectsApi.getNamespacedCustomObject({
+ group: GROUP,
+ version: VERSION,
+ namespace,
+ plural: PLURAL_CONTEXTS,
+ name,
+ })
+ return false
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404) throw error
+ }
+ return (
+ this.contextCacheSynced &&
+ this.contextWatchGeneration === watchGeneration &&
+ !this.contexts.has(name)
+ )
+ }
+
+ private async contextIdAbsentForDelete(
+ contextId: string,
+ watchGeneration: number
+ ): Promise<boolean> {
+ const inventoryAuthoritative = () =>
+ this.contextCacheSynced && this.contextWatchGeneration === watchGeneration
+ const resolveCurrent = () =>
+ [...this.contexts.values()].find(context => context.spec.contextId === contextId)
+ if (!customObjectsApi || !inventoryAuthoritative() || resolveCurrent()) return false
+
+ const response = (await customObjectsApi.listNamespacedCustomObject({
+ group: GROUP,
+ version: VERSION,
+ namespace: config.namespace,
+ plural: PLURAL_CONTEXTS,
+ })) as { items?: Array<{ spec?: { contextId?: string } }> }
+ if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false
+
+ return inventoryAuthoritative() && !resolveCurrent()
+ }
+
+ private enqueueSharedFileSystemReconciliation(
+ name: string,
+ work: () => Promise<void>
+ ): Promise<void> {
+ return this.enqueueKeyedReconciliation(this.sharedFileSystemReconciliationQueues, name, work)
+ }
+
+ private enqueueGlobalFileSystemReconciliation(
+ name: string,
+ work: () => Promise<void>
+ ): Promise<void> {
+ return this.enqueueKeyedReconciliation(this.globalFileSystemReconciliationQueues, name, work)
+ }
+
+ /**
+ * Fleet passes acquire every selected object lane in stable order. This
+ * preserves their snapshot-wide ordering without making two unrelated live
+ * watch events wait behind one another.
+ */
+ private enqueueKeyedFleetReconciliation(
+ queues: Map<string, KeyedReconciliationQueue>,
+ keys: string[],
+ work: () => Promise<void>
+ ): Promise<void> {
+ const orderedKeys = [...new Set(keys)].sort()
+ const enter = (index: number): Promise<void> => {
+ const key = orderedKeys[index]
+ if (!key) return work()
+ return this.enqueueKeyedReconciliation(queues, key, () => enter(index + 1))
+ }
+ return enter(0)
+ }
+
+ private enqueueSharedFileSystemFleetReconciliation(
+ names: string[],
+ work: () => Promise<void>
+ ): Promise<void> {
+ return this.enqueueKeyedFleetReconciliation(
+ this.sharedFileSystemReconciliationQueues,
+ names,
+ work
+ )
+ }
+
+ private enqueueGlobalFileSystemFleetReconciliation(
+ names: string[],
+ work: () => Promise<void>
+ ): Promise<void> {
+ return this.enqueueKeyedFleetReconciliation(
+ this.globalFileSystemReconciliationQueues,
+ names,
+ work
+ )
+ }
+
+ private requireInventoryResourceVersion(
+ kind: 'McpServer' | 'Context',
+ resourceVersion: string | undefined
+ ): string {
+ if (!resourceVersion) {
+ throw new Error(`${kind} snapshot missing resourceVersion`)
+ }
+ return resourceVersion
+ }
+
+ private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {
+ this.servers.clear()
+ for (const server of snapshot.servers) {
+ this.servers.set(server.name, server)
+ }
+ }
+
+ private installContextSnapshot(snapshot: ContextSnapshot): void {
+ this.contexts.clear()
+ for (const context of snapshot.contexts) {
+ this.contexts.set(context.name, context)
+ }
+ }
+
+ private async restartMcpServerWatch(snapshot: McpServerSnapshot): Promise<number> {
+ const resourceVersion = this.requireInventoryResourceVersion(
+ 'McpServer',
+ snapshot.resourceVersion
+ )
+ this.mcpServerCacheSynced = false
+ this.installMcpServerSnapshot(snapshot)
+ const watchGeneration = await this.startMcpServerWatch(resourceVersion)
if (
this.stopped ||
- generation !== this.ccLifecycleGeneration ||
- generation === this.ccAppliedLifecycleGeneration ||
- this.ccFleetRetryTimer
+ watchGeneration !== this.mcpWatchGeneration ||
+ this.mcpWatchRequest === null
) {
- return
+ throw new Error('McpServer snapshot could not be paired with an active watch')
}
- const delayIndex = Math.min(
- this.ccFleetRetryAttempt,
- COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1
+ this.mcpServerCacheSynced = true
+ return watchGeneration
+ }
+
+ private async restartContextWatch(snapshot: ContextSnapshot): Promise<number> {
+ const resourceVersion = this.requireInventoryResourceVersion(
+ 'Context',
+ snapshot.resourceVersion
)
- const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]
- this.ccFleetRetryAttempt += 1
- this.ccFleetRetryTimer = setTimeout(() => {
- this.ccFleetRetryTimer = null
- if (
- this.stopped ||
- generation !== this.ccLifecycleGeneration ||
- generation === this.ccAppliedLifecycleGeneration
- ) {
- return
+ this.contextCacheSynced = false
+ this.installContextSnapshot(snapshot)
+ const watchGeneration = await this.startContextWatch(resourceVersion)
+ if (
+ this.stopped ||
+ watchGeneration !== this.contextWatchGeneration ||
+ this.ctxWatchRequest === null
+ ) {
+ throw new Error('Context snapshot could not be paired with an active watch')
+ }
+ this.contextCacheSynced = true
+ return watchGeneration
+ }
+
+ private recoverMcpServerInventoryAndWatch(): Promise<boolean> {
+ if (this.stopped) return Promise.resolve(false)
+ if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlight
+
+ const recovery = (async () => {
+ try {
+ const snapshot = await listMcpServerSnapshot()
+ if (this.stopped) return false
+ await this.restartMcpServerWatch(snapshot)
+ if (this.stopped) return false
+ this.changeCallback?.()
+ if (this.contextCacheSynced) {
+ void this.runInitialNetworkPolicyConvergence()
+ }
+ return true
+ } catch (error) {
+ this.mcpServerCacheSynced = false
+ console.error('[K8s] McpServer cache recovery failed:', error)
+ return false
}
- void this.requestHostFleetReconcile(
- 'CommunicationChannel lifecycle convergence retry',
- generation,
- request.mode
- )
- }, retryDelay)
+ })()
+
+ this.mcpServerCacheRecoveryInFlight = recovery
+ void recovery.finally(() => {
+ if (this.mcpServerCacheRecoveryInFlight === recovery) {
+ this.mcpServerCacheRecoveryInFlight = null
+ }
+ })
+ return recovery
+ }
+
+ private recoverContextInventoryAndWatch(): Promise<boolean> {
+ if (this.stopped) return Promise.resolve(false)
+ if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlight
+
+ const recovery = (async () => {
+ try {
+ const snapshot = await listContextSnapshot()
+ if (this.stopped) return false
+ await this.restartContextWatch(snapshot)
+ if (this.stopped) return false
+ this.hostFleetScheduler.bumpInputRevision()
+ void this.runInitialNetworkPolicyConvergence()
+ void this.runInitialSharedFileSystemConvergence()
+ if (this.hostCacheSynced) {
+ const pendingCcLifecycleGeneration =
+ this.ccLifecycleGeneration > this.ccAppliedLifecycleGeneration
+ ? this.ccLifecycleGeneration
+ : undefined
+ void this.requestHostFleetReconcile(
+ 'Context cache recovery',
+ pendingCcLifecycleGeneration,
+ 'full'
+ )
+ }
+ return true
+ } catch (error) {
+ this.contextCacheSynced = false
+ console.error('[K8s] Context cache recovery failed:', error)
+ return false
+ }
+ })()
+
+ this.contextCacheRecoveryInFlight = recovery
+ void recovery.finally(() => {
+ if (this.contextCacheRecoveryInFlight === recovery) {
+ this.contextCacheRecoveryInFlight = null
+ }
+ })
+ return recovery
+ }
+
+ private scheduleMcpServerCacheRecovery(): void {
+ if (this.stopped || this.mcpServerCacheRecoveryTimer) return
+ this.mcpServerCacheRecoveryTimer = setTimeout(() => {
+ this.mcpServerCacheRecoveryTimer = null
+ void this.recoverMcpServerInventoryAndWatch().then(recovered => {
+ if (!recovered) this.scheduleMcpServerCacheRecovery()
+ })
+ }, INVENTORY_CACHE_RECOVERY_RETRY_MS)
+ }
+
+ private scheduleContextCacheRecovery(): void {
+ if (this.stopped || this.contextCacheRecoveryTimer) return
+ this.contextCacheRecoveryTimer = setTimeout(() => {
+ this.contextCacheRecoveryTimer = null
+ void this.recoverContextInventoryAndWatch().then(recovered => {
+ if (!recovered) this.scheduleContextCacheRecovery()
+ })
+ }, INVENTORY_CACHE_RECOVERY_RETRY_MS)
+ }
+
+ private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {
+ this.communicationChannels.clear()
+ for (const channel of snapshot.channels) {
+ this.communicationChannels.set(channel.name, channel)
+ }
+ }
+
+ private beginCommunicationChannelLifecycleTransition(): number {
+ return this.hostFleetScheduler.beginLifecycleTransition()
+ }
+
+ private markCommunicationChannelLifecycleApplied(generation: number): void {
+ this.hostFleetScheduler.markLifecycleApplied(generation)
+ }
+
+ private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {
+ this.hostFleetScheduler.scheduleLifecycleRetry(request)
}
/**
@@ -884,19 +1476,45 @@ export class McpServerWatcher implements McpServerProvider {
* Hosts through their per-Host chains (step 7) and requests — without awaiting
* — a coalesced background full pass (step 8).
*/
- private recoverHostInventoryAndWatch(): Promise<HostCRD[]> {
+ private recoverHostInventoryAndWatch(
+ convergenceReason = 'Host watch recovery convergence',
+ ccLifecycleGeneration?: number,
+ cause: HostInventoryRecoveryCause = 'watch-recovery'
+ ): Promise<HostCRD[]> {
if (this.stopped) return Promise.resolve([])
- if (this.hostRecoveryInFlight) return this.hostRecoveryInFlight
- const recovery = this.performHostInventoryRecovery()
- this.hostRecoveryInFlight = recovery
- const clear = (): void => {
- if (this.hostRecoveryInFlight === recovery) this.hostRecoveryInFlight = null
- }
+ const requested = { convergenceReason, ccLifecycleGeneration, cause }
+ // A scheduled retry and an immediate caller are two entrances to the same
+ // monotonic recovery intent. Adopt the pending request before choosing or
+ // joining the active operation so a direct recovery cannot erase a stronger
+ // cold-start cause or newer lifecycle generation when it cancels the timer.
+ const request = this.hostCacheRecoveryIntent
+ ? { ...this.hostCacheRecoveryIntent }
+ : { ...requested }
+ if (this.hostCacheRecoveryIntent) {
+ mergeHostInventoryRecoveryRequest(request, requested)
+ this.hostCacheRecoveryIntent = null
+ if (this.hostCacheRecoveryTimer) {
+ clearTimeout(this.hostCacheRecoveryTimer)
+ this.hostCacheRecoveryTimer = null
+ }
+ }
+ if (this.hostRecoveryInFlight) {
+ mergeHostInventoryRecoveryRequest(this.hostRecoveryInFlight.request, request)
+ return this.hostRecoveryInFlight.promise
+ }
+ const recovery = this.performHostInventoryRecovery(request)
+ const active = { request, promise: recovery }
+ this.hostRecoveryInFlight = active
+ const clear = (): void => {
+ if (this.hostRecoveryInFlight === active) this.hostRecoveryInFlight = null
+ }
void recovery.then(clear, clear)
return recovery
}
- private async performHostInventoryRecovery(): Promise<HostCRD[]> {
+ private async performHostInventoryRecovery(
+ request: HostInventoryRecoveryRequest
+ ): Promise<HostCRD[]> {
const startedAt = Date.now()
// Capture the pre-recovery inventory so step 7 can distinguish genuinely
// new/changed Hosts from ones the fleet pass will converge anyway.
@@ -941,24 +1559,58 @@ export class McpServerWatcher implements McpServerProvider {
{ phase: 'total', outcome: 'success' },
(Date.now() - startedAt) / 1000
)
+ // Effect producers can discover newer work while LIST/WATCH recovery is
+ // already in flight. Fold that scheduled intent into this successful
+ // operation before cancelling its timer so no lifecycle generation,
+ // convergence reason, or stronger cold-start cause is discarded.
+ if (this.hostCacheRecoveryIntent) {
+ mergeHostInventoryRecoveryRequest(request, this.hostCacheRecoveryIntent)
+ }
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
- this.enqueueRecoveredUrgentHosts(
- snapshot.hosts,
- previousNames,
- previousGenerations,
- previousUids
- )
- // Addendum 4 (#827): a Host present before recovery and absent from the
- // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate
- // per-Host delete cleanup so a DELETE lost to watch retirement is never
- // silently dropped.
- this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)
+ this.hostCacheRecoveryIntent = null
+ if (request.cause === 'watch-recovery') {
+ this.enqueueRecoveredUrgentHosts(
+ snapshot.hosts,
+ previousNames,
+ previousGenerations,
+ previousUids
+ )
+ // Addendum 4 (#827): a Host present before recovery and absent from the
+ // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate
+ // per-Host delete cleanup so a DELETE lost to watch retirement is never
+ // silently dropped.
+ this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)
+ }
// Request a coalesced background full pass, but do NOT await it before
- // declaring watch recovery complete.
- void this.requestHostFleetReconcile('Host watch recovery convergence', undefined, 'full')
+ // declaring watch recovery complete. The cold-start pass alone waits for
+ // the concurrent SFS LIST: otherwise it can write every Host template
+ // without its declared mounts, then immediately roll the fleet again
+ // when SFS discovery finishes. NetworkPolicy readiness and start() stay
+ // independent from that finite inventory request.
+ const requestFleetPass = () => {
+ if (this.stopped) return
+ void this.requestHostFleetReconcile(
+ request.convergenceReason,
+ request.ccLifecycleGeneration,
+ 'full'
+ )
+ }
+ const initialSfsInventory =
+ request.cause === 'cold-start' ? this.initialHostFleetSfsInventory : null
+ if (initialSfsInventory) {
+ // Waiting for the inventory is deliberate: a Host fleet pass without it
+ // writes mount-less templates and immediately rerolls the whole fleet.
+ // The wait cannot strand the fleet, because the LIST underneath runs on
+ // the deadline-bearing client: an apiserver that never answers aborts
+ // the request, `startSharedFileSystemInventoryAndWatch` logs and
+ // resolves false, and this settles. One path, one pass.
+ void initialSfsInventory.then(() => requestFleetPass())
+ } else {
+ requestFleetPass()
+ }
return snapshot.hosts
} catch (error) {
// Keep authority unknown, record the failure, and use the existing
@@ -970,7 +1622,7 @@ export class McpServerWatcher implements McpServerProvider {
)
if (!this.stopped) {
console.error('[K8s] Host watch recovery failed:', error)
- this.scheduleHostCacheRecovery()
+ this.scheduleHostCacheRecovery(request)
}
throw error
}
@@ -1010,6 +1662,7 @@ export class McpServerWatcher implements McpServerProvider {
private async dispatchUrgentHostReconcile(name: string): Promise<void> {
if (this.stopped) return
+ if (!this.admitHostDependentEffects('Recovered urgent Host convergence')) return
const host = this.hosts.get(name)
if (!host) return
try {
@@ -1041,6 +1694,7 @@ export class McpServerWatcher implements McpServerProvider {
private async dispatchRecoveredHostDelete(name: string): Promise<void> {
if (this.stopped) return
+ if (!this.admitHostDependentEffects('Recovered Host deletion convergence')) return
// The fresh authoritative LIST already confirmed absence and the cache no
// longer holds this Host, so cleanup is authorized. Route it through the
// per-Host serializer (reconcileDelete → serializeByHost) so an older
@@ -1072,13 +1726,30 @@ export class McpServerWatcher implements McpServerProvider {
}
}
- private scheduleHostCacheRecovery(): void {
- if (this.stopped || this.hostCacheRecoveryTimer) return
+ private scheduleHostCacheRecovery(
+ requested: HostInventoryRecoveryRequest = {
+ convergenceReason: 'Host watch recovery convergence',
+ cause: 'watch-recovery',
+ }
+ ): void {
+ if (this.stopped) return
+ if (this.hostCacheRecoveryIntent) {
+ mergeHostInventoryRecoveryRequest(this.hostCacheRecoveryIntent, requested)
+ } else {
+ this.hostCacheRecoveryIntent = { ...requested }
+ }
+ if (this.hostCacheRecoveryTimer) return
this.hostCacheRecoveryTimer = setTimeout(() => {
this.hostCacheRecoveryTimer = null
+ const request = this.hostCacheRecoveryIntent ?? requested
+ this.hostCacheRecoveryIntent = null
// Watch recovery is an independent operation, no longer coupled to a full
// fleet pass. Recovery itself requests the background convergence pass.
- void this.recoverHostInventoryAndWatch().catch(() => undefined)
+ void this.recoverHostInventoryAndWatch(
+ request.convergenceReason,
+ request.ccLifecycleGeneration,
+ request.cause
+ ).catch(() => undefined)
}, HOST_CACHE_RECOVERY_RETRY_MS)
}
@@ -1123,6 +1794,7 @@ export class McpServerWatcher implements McpServerProvider {
source: HostReconcileSource = 'urgent'
): Promise<void> {
if (this.stopped) return
+ if (!this.admitHostDependentEffects('Host watch event convergence')) return
if (!this.isCurrentHostWatchEvent(type, host, eventRevision)) {
// #827: a DELETE that is no longer current because the cache now holds a
// (recreated) same-name Host is a superseded delete — record it and do
@@ -1217,6 +1889,7 @@ export class McpServerWatcher implements McpServerProvider {
await this.recoverHostInventoryAndWatch()
if (this.stopped) return
}
+ if (!this.isHostEffectInventoryAuthoritative()) return
// Reconcile the CURRENT cache. Fleet workers dispatch Host keys and
// resolve the freshest cached spec at execution (see collectHostReconcile
// Failures); only fullReconcile runs the authority-gated orphan cleanup.
@@ -1279,78 +1952,7 @@ export class McpServerWatcher implements McpServerProvider {
ccLifecycleGeneration?: number,
mode?: HostFleetReconcileMode
): Promise<void> {
- if (this.stopped) return Promise.resolve()
- const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')
- if (
- ccLifecycleGeneration !== undefined &&
- (ccLifecycleGeneration !== this.ccLifecycleGeneration ||
- (requestMode === 'lifecycle' &&
- ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))
- ) {
- return Promise.resolve()
- }
-
- const request: HostFleetReconcileRequest = {
- reason,
- mode: requestMode,
- ccLifecycleGeneration,
- }
- const active = this.hostFleetReconcileInFlight
- if (!active) {
- hostFleetRequestsTotal.inc({ result: 'started' })
- return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))
- }
- if (hostFleetRequestCovers(active, request)) {
- hostFleetRequestsTotal.inc({ result: 'coalesced' })
- return this.waitForHostFleetOrShutdown(active.promise)
- }
-
- if (!this.hostFleetReconcilePending) {
- let resolve!: () => void
- const promise = new Promise<void>(resolvePromise => {
- resolve = resolvePromise
- })
- this.hostFleetReconcilePending = {
- ...request,
- promise,
- resolve,
- }
- } else {
- const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)
- this.hostFleetReconcilePending.reason = merged.reason
- this.hostFleetReconcilePending.mode = merged.mode
- this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration
- }
- // The request did not start its own pass; it is queued behind the active one.
- hostFleetRequestsTotal.inc({ result: 'coalesced' })
- return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)
- }
-
- private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {
- return Promise.race([coverage, this.hostFleetShutdown])
- }
-
- private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {
- const promise = this.performHostFleetReconcile(request)
- const active: ActiveHostFleetReconcile = { ...request, promise }
- this.hostFleetReconcileInFlight = active
-
- const settle = () => {
- if (this.hostFleetReconcileInFlight !== active) return
- this.hostFleetReconcileInFlight = null
- const pending = this.hostFleetReconcilePending
- this.hostFleetReconcilePending = null
- if (!pending) return
- if (this.stopped) {
- pending.resolve()
- return
- }
- hostFleetRequestsTotal.inc({ result: 'trailing' })
- const trailing = this.startHostFleetReconcile(pending)
- void trailing.then(pending.resolve, pending.resolve)
- }
- void promise.then(settle, settle)
- return promise
+ return this.hostFleetScheduler.request(reason, ccLifecycleGeneration, mode)
}
private contextsReferencingSfs(sfsName: string): Array<{ namespace: string; name: string }> {
@@ -1410,14 +2012,35 @@ export class McpServerWatcher implements McpServerProvider {
return out
}
- private async reconcileSharedFileSystemsReferencedByContext(
+ private reconcileSharedFileSystemsReferencedByContext(
+ prevContext: ContextCRD | undefined,
+ nextContext: ContextCRD | undefined,
+ isCurrent: () => boolean = () => true
+ ): Promise<void> {
+ const names = [
+ ...(prevContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),
+ ...(nextContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),
+ ]
+ return this.enqueueSharedFileSystemFleetReconciliation(names, async () => {
+ if (!isCurrent()) return
+ await this.reconcileSharedFileSystemsReferencedByContextCore(
+ prevContext,
+ nextContext,
+ isCurrent
+ )
+ })
+ }
+
+ private async reconcileSharedFileSystemsReferencedByContextCore(
prevContext: ContextCRD | undefined,
- nextContext: ContextCRD | undefined
+ nextContext: ContextCRD | undefined,
+ isCurrent: () => boolean
): Promise<void> {
const names = new Set<string>()
for (const r of prevContext?.spec.sharedFileSystems ?? []) names.add(r.name)
for (const r of nextContext?.spec.sharedFileSystems ?? []) names.add(r.name)
for (const name of names) {
+ if (!isCurrent()) return
const sfs = this.sharedFileSystems.get(name)
if (!sfs) continue
try {
@@ -1434,6 +2057,9 @@ export class McpServerWatcher implements McpServerProvider {
* Deployment template picks up (or drops) the volume + volumeMount.
*/
private async reconcileHostsReferencingSfs(sfsName: string): Promise<void> {
+ if (!this.admitHostDependentEffects(`SharedFileSystem "${sfsName}" Host convergence`)) {
+ return
+ }
const affectedContexts = new Set<string>()
for (const ctx of this.contexts.values()) {
if (ctx.spec.sharedFileSystems?.some(r => r.name === sfsName)) {
@@ -1451,6 +2077,24 @@ export class McpServerWatcher implements McpServerProvider {
}
}
+ /**
+ * Reconcile Hosts affected by a Context change only while the Host cache is
+ * authoritative. A recovery full pass consumes the current Context cache, so
+ * deferral loses no effect and never acts on an unpaired Host LIST.
+ */
+ private async reconcileHostsReferencingContext(
+ contextName: string,
+ isCurrent: () => boolean = () => true
+ ): Promise<void> {
+ if (!isCurrent()) return
+ if (!this.admitHostDependentEffects(`Context "${contextName}" Host convergence`)) return
+ for (const host of this.hosts.values()) {
+ if (!isCurrent()) return
+ if (host.spec.contextRef !== contextName) continue
+ await this.hostReconciler.reconcile(host)
+ }
+ }
+
/**
* Re-reconcile the Host whose name matches the given hostRef. Called by the
* CommunicationChannel watch handler so per-Host channel-reader Deployment
@@ -1463,6 +2107,9 @@ export class McpServerWatcher implements McpServerProvider {
* its caller schedules bounded fleet convergence.
*/
private async reconcileHostsReferencingCC(hostRef: string): Promise<boolean> {
+ if (!this.admitHostDependentEffects(`CommunicationChannel Host "${hostRef}" convergence`)) {
+ return false
+ }
const host = this.hosts.get(hostRef)
if (!host) return true
try {
@@ -1554,9 +2201,28 @@ export class McpServerWatcher implements McpServerProvider {
* tracker resolves the heartbeat's hostRef through this).
*/
getHost(name: string): HostCRD | undefined {
+ if (!this.isHostInventoryAuthoritative()) return undefined
return this.hosts.get(name)
}
+ /**
+ * Route channel credentials-Secret events through the same Host authority
+ * boundary as watch, Context, SFS, and heartbeat effects.
+ */
+ async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {
+ const convergenceReason = `CommunicationChannel Secret "${secretNamespace}/${secretName}" Host convergence`
+ if (!this.ccCacheSynced) {
+ console.warn(
+ `[K8s] Deferring ${convergenceReason}; CommunicationChannel cache is not authoritative`
+ )
+ return
+ }
+ if (!this.admitHostDependentEffects(convergenceReason)) {
+ return
+ }
+ await this.hostReconciler.reconcileChannelReaderRevision(secretName, secretNamespace)
+ }
+
/**
* Trigger a reconcile for any cached McpServer whose `envSecret.name`
* matches the given Secret. Used by SecretInformer to react to Secret
@@ -1564,18 +2230,45 @@ export class McpServerWatcher implements McpServerProvider {
* event.
*/
async reconcileByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {
- for (const server of this.servers.values()) {
- if (server.namespace !== secretNamespace) continue
- if (server.spec.envSecret?.name !== secretName) continue
- try {
- console.log(
- `[K8s] Re-reconciling McpServer "${server.name}" after Secret "${secretName}" change`
- )
- await this.reconciler.reconcile(server)
- } catch (err) {
- console.error(`[K8s] Secret-triggered reconcile failed for "${server.name}":`, err)
- }
- }
+ const inventoryGeneration = this.mcpWatchGeneration
+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
+ const selected = [...this.servers.values()].filter(
+ server => server.namespace === secretNamespace && server.spec.envSecret?.name === secretName
+ )
+ await Promise.all(
+ selected.map(server =>
+ this.enqueueMcpServerReconciliation(server, async () => {
+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
+ const current = this.servers.get(server.name)
+ if (
+ !current ||
+ current.namespace !== secretNamespace ||
+ current.spec.envSecret?.name !== secretName
+ ) {
+ return
+ }
+ try {
+ console.log(
+ `[K8s] Re-reconciling McpServer "${current.name}" after Secret "${secretName}" change`
+ )
+ await this.reconciler.reconcile(current, {
+ isCurrent: () => {
+ if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return false
+ const latest = this.servers.get(current.name)
+ return (
+ latest !== undefined &&
+ latest.namespace === secretNamespace &&
+ latest.spec.envSecret?.name === secretName &&
+ sameMcpServerDesiredRevision(current, latest)
+ )
+ },
+ })
+ } catch (err) {
+ console.error(`[K8s] Secret-triggered reconcile failed for "${current.name}":`, err)
+ }
+ })
+ )
+ )
}
/**
@@ -1603,7 +2296,7 @@ export class McpServerWatcher implements McpServerProvider {
transport,
auth: server.spec.auth,
enabled: server.spec.enabled !== false,
- status: this.reconciler.getStatus(server.name),
+ status: this.reconciler.getStatus(server),
}
}
@@ -1658,61 +2351,55 @@ export class McpServerWatcher implements McpServerProvider {
* Start watching for changes and run initial reconciliation.
*/
async start(): Promise<void> {
- let initialServers: McpServerCRD[] = []
- let serverInventoryComplete = false
+ // L0/L1 isolation and removal of legacy permissive policies are mandatory
+ // bootstrap barriers. They are a fixed-size safety operation, independent
+ // of fleet size, and must succeed before HCC can watch or reconcile runtimes.
+ await this.netPolReconciler.ensureDefaultPolicies()
+
+ // Discovery and policy caches become authoritative only when each complete
+ // LIST snapshot is paired with a watch continuing from its opaque collection
+ // resourceVersion. Full fleet convergence happens later in the background.
+ // A transient LIST/WATCH failure on either inventory must not kill the
+ // retry that restart*Watch already armed (it sets *CacheSynced=false before
+ // it can throw). Mirror the Host lane below: log, keep the flag false, and
+ // schedule in-process recovery. The HTTP server stays live, dynamic
+ // readiness stays 503, and recovery promotes readiness only after a fresh
+ // LIST is paired with its continuing WATCH.
try {
- initialServers = await listAllMcpServers()
- serverInventoryComplete = true
- for (const server of initialServers) {
- this.servers.set(server.name, server)
- }
+ const initialServerSnapshot = await listMcpServerSnapshot()
+ await this.restartMcpServerWatch(initialServerSnapshot)
} catch (error) {
+ this.mcpServerCacheSynced = false
console.error(
- '[K8s] Skipping initial McpServer full reconciliation because server discovery failed:',
+ '[K8s] Initial McpServer inventory is unavailable; HCC remains unready while in-process recovery continues:',
error
)
}
-
- // Ensure baseline network policies exist even if initial Context discovery fails.
- try {
- await this.netPolReconciler.ensureDefaultPolicies()
- } catch (error) {
- console.error('[K8s] Failed to ensure default NetworkPolicies before startup:', error)
- }
-
- let initialExternalEgressFailures = new Set<string>()
- if (serverInventoryComplete) {
- initialExternalEgressFailures =
- await this.reconcileInitialExternalEgressBeforeRuntime(initialServers)
- }
-
- // Full McpServer reconciliation (Deployments + Services). Servers with
- // egressBindings are reconciled only after their external egress policies
- // have converged, matching the ADDED/MODIFIED watch-event contract.
- if (serverInventoryComplete) {
- const runtimeServers = initialServers.filter(
- server => !initialExternalEgressFailures.has(this.externalEgressRetryKey(server))
- )
- console.log('[K8s] Running initial full reconciliation...')
- await this.reconciler.fullReconcile(runtimeServers)
+ if (!this.mcpServerCacheSynced) {
+ this.scheduleMcpServerCacheRecovery()
}
try {
- const initialContexts = await listAllContexts()
- for (const context of initialContexts) {
- this.contexts.set(context.name, context)
- }
- console.log('[K8s] Running initial NetworkPolicy reconciliation...')
- await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
- serverInventoryComplete,
- ensureDefaults: false,
- })
+ const initialContextSnapshot = await listContextSnapshot()
+ await this.restartContextWatch(initialContextSnapshot)
} catch (error) {
+ this.contextCacheSynced = false
console.error(
- '[K8s] Skipping initial NetworkPolicy reconciliation because context discovery failed:',
+ '[K8s] Initial Context inventory is unavailable; HCC remains unready while in-process recovery continues:',
error
)
}
+ if (!this.contextCacheSynced) {
+ this.scheduleContextCacheRecovery()
+ }
+
+ // Host templates resolve Context SharedFileSystem references from this
+ // cache. Inventory is therefore a fixed startup boundary, not part of the
+ // unbounded SFS reconciliation fleet: letting the first Host pass run
+ // without it writes mount-less templates and immediately rolls the entire
+ // Host fleet a second time once discovery catches up.
+ const initialSharedFileSystemInventory = this.startSharedFileSystemInventoryAndWatch()
+ this.initialHostFleetSfsInventory = initialSharedFileSystemInventory
// ── CommunicationChannel snapshot + watch — MUST complete before Host fullReconcile ──
// A stateless Host may suspend only after this controller has a complete
@@ -1761,54 +2448,30 @@ export class McpServerWatcher implements McpServerProvider {
}
console.log(
- `[K8s] Running initial Host reconciliation... (ccCacheSynced=${this.ccCacheSynced})`
+ `[K8s] Starting initial Host background convergence... (ccCacheSynced=${this.ccCacheSynced})`
)
const initialLifecycleGeneration =
this.ccLifecycleGeneration === 0
? this.beginCommunicationChannelLifecycleTransition()
: this.ccLifecycleGeneration
- await this.requestHostFleetReconcile(
- 'initial Host reconciliation',
- initialLifecycleGeneration,
- 'full'
- )
-
- // ── SharedFileSystem initial load + reconciliation ──
try {
- const initialSfses = await listAllSharedFileSystems()
- for (const sfs of initialSfses) {
- this.sharedFileSystems.set(sfs.name, sfs)
- }
- console.log('[K8s] Running initial SharedFileSystem reconciliation...')
- await this.sharedFileSystemReconciler.fullReconcile(initialSfses)
- } catch (error) {
- console.error(
- '[K8s] Skipping initial SharedFileSystem reconciliation because discovery failed:',
- error
+ await this.recoverHostInventoryAndWatch(
+ 'initial Host reconciliation',
+ initialLifecycleGeneration,
+ 'cold-start'
)
- }
-
- // ── GlobalFileSystem (gfs) initial load + reconciliation ──
- // gfs is a cluster singleton (DISTINCT from SharedFileSystem). A failed
- // discovery (e.g. the CRD not installed yet) is skipped, not fatal — the
- // next HCC restart / resync retries. Live updates land via the watch
- // (follow-up); the deploy applies the singleton before HCC reconciles it.
- try {
- const initialGfses = await listAllGlobalFileSystems()
- console.log('[K8s] Running initial GlobalFileSystem reconciliation...')
- await this.gfsReconciler.fullReconcile(initialGfses)
} catch (error) {
- console.error(
- '[K8s] Skipping initial GlobalFileSystem reconciliation because discovery failed:',
+ // Host inventory is an essential readiness authority, but a transient
+ // LIST/WATCH failure must not kill the retry that recovery just armed.
+ // Continue startup with hostCacheSynced=false: the HTTP server stays live,
+ // dynamic readiness remains 503, and recovery promotes readiness only
+ // after a fresh LIST is paired with its continuing WATCH.
+ console.warn(
+ '[K8s] Initial Host inventory is unavailable; HCC remains unready while in-process recovery continues:',
error
)
}
- await this.startMcpServerWatch()
- await this.startContextWatch()
- await this.startSharedFileSystemWatch()
- await this.startGlobalFileSystemWatch()
-
const resyncSec = config.hostResyncIntervalSec
if (resyncSec > 0) {
this.resyncTimer = setInterval(() => {
@@ -1846,35 +2509,375 @@ export class McpServerWatcher implements McpServerProvider {
}
const externalEgressResyncSec = config.externalEgressResyncIntervalSec
- if (externalEgressResyncSec > 0) {
- // Self-rescheduling (not a fixed setInterval) so the delay can advance to
- // <= observed TTL/2 (H2, issue #299) and rescheduling AFTER each pass
- // completes guarantees no overlapping resyncs.
- const scheduleNext = (): void => {
- if (this.stopped) return
- const delayMs = externalEgressResyncDelayMs(
- externalEgressResyncSec,
- config.externalEgressRefreshFloorSec,
- this.netPolReconciler.externalEgressRefreshMinTtlMs
- )
- this.externalEgressResyncTimer = setTimeout(() => {
- void this.runExternalEgressResync().finally(() => scheduleNext())
- }, delayMs)
- this.externalEgressResyncTimer.unref?.()
- }
- // Idempotency guard (audit F5): never arm a second self-rescheduling chain
- // if one is already live (start() runs once today, but a future restart
- // path must not double the cadence).
- if (!this.externalEgressResyncTimer) scheduleNext()
- console.log(
- `[K8s] External egress periodic DNS resync enabled (every ${externalEgressResyncSec}s, advancing to <= TTL/2)`
+ // #205 delegates external-egress periodic resync to the convergence
+ // coordinator. Its runResyncCore drives reconcileExternalEgress, so the
+ // #299 sliding-window accumulation still converges on DNS rotation between
+ // McpServer events. The coordinator owns cadence with a self-rescheduling
+ // timer that advances to <= observed TTL/2 (H2, issue #299), bounded by the
+ // configured floor.
+ this.externalEgressCoordinator.startPeriodicResync(
+ externalEgressResyncSec,
+ config.externalEgressRefreshFloorSec
+ )
+
+ // Full convergence is observable and retains each lane's existing retry or
+ // periodic-resync contract, but it no longer extends provider.start().
+ // NetworkPolicy safety starts first and launches McpServer convergence from
+ // its certification callback, before additive policy convergence completes.
+ // This prevents the startup egress refresh from racing or undoing the safety
+ // pass while keeping readiness independent from both additive fleets.
+ void this.runInitialNetworkPolicyConvergence()
+ void initialSharedFileSystemInventory.then(inventoryComplete => {
+ if (inventoryComplete && !this.stopped) {
+ void this.runInitialSharedFileSystemConvergence()
+ }
+ })
+ void this.startGlobalFileSystemBackgroundLane()
+ }
+
+ private async startSharedFileSystemInventoryAndWatch(): Promise<boolean> {
+ let inventoryComplete = false
+ try {
+ const initialSfses = await listAllSharedFileSystems()
+ inventoryComplete = true
+ for (const sfs of initialSfses) this.sharedFileSystems.set(sfs.name, sfs)
+ } catch (error) {
+ console.error(
+ '[K8s] Skipping initial SharedFileSystem background convergence because discovery failed:',
+ error
)
- } else {
- console.warn(
- '[K8s] External egress periodic DNS resync DISABLED (HCC_EXTERNAL_EGRESS_RESYNC_SEC=0); ' +
- 'exact-host FQDN egress will NOT converge on DNS rotation between McpServer events — ' +
- 'accumulated IPs can go stale and block workloads (issue #299). Enable it unless you have a specific reason.'
+ }
+ // The cold-start Host fleet pass gates on this promise, so it must settle on
+ // the inventory alone. `k8s.Watch` carries no transport deadline — the
+ // client builds `AbortSignal.any([controller, timeout])` and then overwrites
+ // it with the bare controller — so awaiting the watch start here would
+ // re-strand the fleet behind an already-certified readiness, which is the
+ // exact failure bounding the LIST was meant to close.
+ void this.startSharedFileSystemWatch().catch(error => {
+ console.error('[K8s] SharedFileSystem background watch failed to start:', error)
+ this.scheduleSharedFileSystemWatchRestart(5000)
+ })
+ if (inventoryComplete && !this.stopped) {
+ return true
+ }
+ return false
+ }
+
+ private async startGlobalFileSystemBackgroundLane(): Promise<void> {
+ let inventoryComplete = false
+ try {
+ const initialGfses = await listAllGlobalFileSystems()
+ inventoryComplete = true
+ for (const gfs of initialGfses) this.globalFileSystems.set(gfs.name, gfs)
+ } catch (error) {
+ console.error(
+ '[K8s] Skipping initial GlobalFileSystem background convergence because discovery failed:',
+ error
+ )
+ }
+ try {
+ await this.startGlobalFileSystemWatch()
+ } catch (error) {
+ console.error('[K8s] GlobalFileSystem background watch failed to start:', error)
+ this.scheduleGlobalFileSystemWatchRestart(5000)
+ }
+ if (inventoryComplete && !this.stopped) {
+ void this.runInitialGlobalFileSystemConvergence()
+ }
+ }
+
+ private runInitialMcpServerConvergence(): Promise<void> {
+ return this.runInitialConvergence('McpServer')
+ }
+
+ private currentNetworkPolicySafetyCertificate(): NetworkPolicySafetyCertificate | null {
+ if (
+ this.stopped ||
+ !this.contextCacheSynced ||
+ !this.mcpServerCacheSynced ||
+ !this.netPolReconciler.hasCertifiedSafetyInventory() ||
+ this.networkPolicyRevocationContextGeneration !== this.contextWatchGeneration ||
+ this.networkPolicyRevocationServerGeneration !== this.mcpWatchGeneration ||
+ this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||
+ this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision
+ ) {
+ return null
+ }
+ return {
+ contextGeneration: this.contextWatchGeneration,
+ serverGeneration: this.mcpWatchGeneration,
+ contextRevision: this.contextDesiredRevision,
+ serverRevision: this.mcpServerDesiredRevision,
+ }
+ }
+
+ private recordNetworkPolicySafetyCertificate(
+ certificate: NetworkPolicySafetyCertificate
+ ): boolean {
+ if (
+ this.stopped ||
+ !this.contextCacheSynced ||
+ !this.mcpServerCacheSynced ||
+ this.contextWatchGeneration !== certificate.contextGeneration ||
+ this.mcpWatchGeneration !== certificate.serverGeneration ||
+ this.contextDesiredRevision !== certificate.contextRevision ||
+ this.mcpServerDesiredRevision !== certificate.serverRevision
+ ) {
+ return false
+ }
+ this.networkPolicyRevocationContextGeneration = certificate.contextGeneration
+ this.networkPolicyRevocationServerGeneration = certificate.serverGeneration
+ this.networkPolicyRevocationContextRevision = certificate.contextRevision
+ this.networkPolicyRevocationServerRevision = certificate.serverRevision
+ return true
+ }
+
+ private isNetworkPolicySafetyCertificateCurrent(
+ certificate: NetworkPolicySafetyCertificate
+ ): boolean {
+ const current = this.currentNetworkPolicySafetyCertificate()
+ return (
+ current !== null &&
+ current.contextGeneration === certificate.contextGeneration &&
+ current.serverGeneration === certificate.serverGeneration &&
+ current.contextRevision === certificate.contextRevision &&
+ current.serverRevision === certificate.serverRevision
+ )
+ }
+
+ private async runInitialMcpServerConvergenceCore(): Promise<void> {
+ const safetyCertificate = this.currentNetworkPolicySafetyCertificate()
+ if (safetyCertificate === null) return
+ const inventoryGeneration = this.mcpWatchGeneration
+ const inventoryAuthoritative = () =>
+ this.hasMcpServerInventoryAuthority(inventoryGeneration) &&
+ this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)
+ try {
+ const initialServers = [...this.servers.values()]
+ const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(
+ initialServers,
+ inventoryAuthoritative
+ )
+ if (this.stopped) return
+ console.log('[K8s] Running initial McpServer background reconciliation...')
+ await this.reconciler.fullReconcile(initialServers, {
+ runEffect: async (serverName, work) => {
+ const selected = this.servers.get(serverName)
+ const laneOwner = selected ?? {
+ name: serverName,
+ namespace: config.namespace,
+ }
+ if (!(await initialExternalEgressGates.waitFor(laneOwner))) return
+ if (!inventoryAuthoritative()) return
+ await this.enqueueMcpServerReconciliation(laneOwner, async () => {
+ if (!inventoryAuthoritative()) return
+ await work()
+ })
+ },
+ })
+ if (!inventoryAuthoritative()) return
+ initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)
+ this.clearInitialConvergenceRetry('McpServer')
+ } catch (error) {
+ console.error('[K8s] Initial McpServer background reconciliation failed:', error)
+ this.scheduleInitialConvergenceRetry('McpServer')
+ }
+ }
+
+ private runInitialNetworkPolicyConvergence(): Promise<void> {
+ return this.runInitialConvergence('NetworkPolicy')
+ }
+
+ private runInitialConvergence(lane: InitialConvergenceLane): Promise<void> {
+ const active = this.initialConvergenceRuns.get(lane)
+ if (active) {
+ active.trailingRequested = true
+ return active.promise
+ }
+
+ const run: ActiveInitialConvergenceRun = {
+ trailingRequested: false,
+ promise: Promise.resolve(),
+ }
+ this.initialConvergenceRuns.set(lane, run)
+ run.promise = Promise.resolve()
+ .then(async () => {
+ do {
+ run.trailingRequested = false
+ if (lane === 'McpServer') {
+ await this.runInitialMcpServerConvergenceCore()
+ } else {
+ await this.runInitialNetworkPolicyConvergenceCore()
+ }
+ } while (!this.stopped && run.trailingRequested)
+ // Retire the active slot synchronously before this promise settles.
+ // A request arriving in the following settlement microtask must start
+ // a new run instead of attaching to a pass whose loop already exited.
+ if (this.initialConvergenceRuns.get(lane) === run) {
+ this.initialConvergenceRuns.delete(lane)
+ }
+ })
+ .finally(() => {
+ // Errors bypass the normal retirement above.
+ if (this.initialConvergenceRuns.get(lane) === run) {
+ this.initialConvergenceRuns.delete(lane)
+ }
+ })
+ return run.promise
+ }
+
+ private async runInitialNetworkPolicyConvergenceCore(): Promise<void> {
+ // A NetworkPolicy full pass combines authoritative Context/McpServer allow
+ // revocation with additive Context policy effects. External egress
+ // creation and DNS refresh have one separate owner: the startup/resync
+ // coordinator. Running safety from a partial inventory would be neither a
+ // full convergence nor safe evidence of success. Recovery of either
+ // missing LIST -> WATCH pair schedules a fresh current-cache pass.
+ if (!this.contextCacheSynced || !this.mcpServerCacheSynced) return
+ let authoritativeRevocationCompleted = false
+ try {
+ const initialContexts = [...this.contexts.values()]
+ const initialServers = [...this.servers.values()]
+ const contextInventoryGeneration = this.contextWatchGeneration
+ const serverInventoryGeneration = this.mcpWatchGeneration
+ const safetyCertificate: NetworkPolicySafetyCertificate = {
+ contextGeneration: contextInventoryGeneration,
+ serverGeneration: serverInventoryGeneration,
+ contextRevision: this.contextDesiredRevision,
+ serverRevision: this.mcpServerDesiredRevision,
+ }
+ const serverInventoryComplete = this.mcpServerCacheSynced
+ const contextInventoryAuthoritative = () =>
+ this.hasContextInventoryAuthority(contextInventoryGeneration)
+ const serverInventoryAuthoritative = () =>
+ this.hasMcpServerInventoryAuthority(serverInventoryGeneration)
+ console.log('[K8s] Running initial NetworkPolicy background reconciliation...')
+ await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
+ serverInventoryComplete,
+ ensureDefaults: false,
+ contextInventoryAuthoritative,
+ serverInventoryAuthoritative,
+ runContextEffect: (contextId, work) =>
+ this.enqueueContextReconciliation(contextId, async () => {
+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return
+ await work()
+ }),
+ runServerEffect: (serverName, work) => {
+ const selected = this.servers.get(serverName)
+ return this.enqueueMcpServerReconciliation(
+ selected ?? { name: serverName, namespace: config.namespace },
+ async () => {
+ if (!serverInventoryAuthoritative()) return
+ await work()
+ }
+ )
+ },
+ resolveCurrentContext: name => this.contexts.get(name),
+ resolveCurrentContextById: contextId =>
+ [...this.contexts.values()].find(context => context.spec.contextId === contextId),
+ resolveCurrentServer: name => this.servers.get(name),
+ contextDesiredRevision: () => this.contextDesiredRevision,
+ serverDesiredRevision: () => this.mcpServerDesiredRevision,
+ // B3 piece-2: queue an additive recreation for every server whose
+ // external-egress allow the authoritative pass revoked — fired at the
+ // deletion, so an abort/throw cannot strand a revoked-but-desired allow.
+ // The coordinator maps against the current cache (absent → DELETED, no
+ // recreate) and rebuilds from the current spec with fresh DNS.
+ onExternalEgressRevoked: server =>
+ this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),
+ onAuthoritativeRevocationComplete: () => {
+ if (!this.recordNetworkPolicySafetyCertificate(safetyCertificate)) return
+ authoritativeRevocationCompleted = true
+ // Start runtime convergence at the safety boundary, before this full
+ // pass enters additive Context work. Startup egress gates may now
+ // preserve only policies certified by this exact authoritative
+ // inventory generation.
+ void this.runInitialMcpServerConvergence()
+ },
+ })
+ if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return
+ initialConvergenceLastSuccessTimestampSeconds.set(
+ { lane: 'NetworkPolicy' },
+ Date.now() / 1000
)
+ this.clearInitialConvergenceRetry('NetworkPolicy')
+ } catch (error) {
+ console.error(
+ authoritativeRevocationCompleted
+ ? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:'
+ : '[K8s] Initial NetworkPolicy background reconciliation failed:',
+ error
+ )
+ this.scheduleInitialConvergenceRetry('NetworkPolicy')
+ }
+ }
+
+ private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {
+ const timer = this.initialConvergenceRetryTimers.get(lane)
+ if (timer) {
+ clearTimeout(timer)
+ this.initialConvergenceRetryTimers.delete(lane)
+ }
+ this.initialConvergenceRetryAttempts.delete(lane)
+ }
+
+ private scheduleInitialConvergenceRetry(lane: InitialConvergenceLane): void {
+ if (this.stopped || this.initialConvergenceRetryTimers.has(lane)) return
+
+ const attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1
+ const delayMs =
+ INITIAL_CONVERGENCE_RETRY_DELAYS_MS[
+ Math.min(attempt - 1, INITIAL_CONVERGENCE_RETRY_DELAYS_MS.length - 1)
+ ]
+ this.initialConvergenceRetryAttempts.set(lane, attempt)
+ initialConvergenceRetriesTotal.inc({ lane })
+ console.warn(
+ `[K8s] Scheduling initial ${lane} background convergence retry ${attempt} in ${delayMs}ms`
+ )
+
+ const timer = setTimeout(() => {
+ this.initialConvergenceRetryTimers.delete(lane)
+ if (this.stopped) return
+ if (lane === 'McpServer') {
+ void this.runInitialMcpServerConvergence()
+ return
+ }
+ void this.runInitialNetworkPolicyConvergence()
+ }, delayMs)
+ this.initialConvergenceRetryTimers.set(lane, timer)
+ }
+
+ private runInitialSharedFileSystemConvergence(): Promise<void> {
+ const names = [...this.sharedFileSystems.keys()]
+ return this.enqueueSharedFileSystemFleetReconciliation(names, () =>
+ this.runInitialSharedFileSystemConvergenceCore()
+ )
+ }
+
+ private async runInitialSharedFileSystemConvergenceCore(): Promise<void> {
+ try {
+ const initialSfses = [...this.sharedFileSystems.values()]
+ console.log('[K8s] Running initial SharedFileSystem background reconciliation...')
+ await this.sharedFileSystemReconciler.fullReconcile(initialSfses)
+ } catch (error) {
+ console.error('[K8s] Initial SharedFileSystem background reconciliation failed:', error)
+ }
+ }
+
+ private runInitialGlobalFileSystemConvergence(): Promise<void> {
+ const names = [...this.globalFileSystems.keys()]
+ return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>
+ this.runInitialGlobalFileSystemConvergenceCore()
+ )
+ }
+
+ private async runInitialGlobalFileSystemConvergenceCore(): Promise<void> {
+ try {
+ const initialGfses = [...this.globalFileSystems.values()]
+ console.log('[K8s] Running initial GlobalFileSystem background reconciliation...')
+ await this.gfsReconciler.fullReconcile(initialGfses)
+ } catch (error) {
+ console.error('[K8s] Initial GlobalFileSystem background reconciliation failed:', error)
}
}
@@ -1902,42 +2905,58 @@ export class McpServerWatcher implements McpServerProvider {
} else if (type === 'DELETED') {
this.sharedFileSystems.delete(sfs.name)
}
+ this.sharedFileSystemCacheRevision += 1
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- await this.sharedFileSystemReconciler.reconcile(sfs)
- } else if (type === 'DELETED') {
- await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)
- }
- } catch (error) {
- console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)
- }
+ await this.enqueueSharedFileSystemReconciliation(sfs.name, async () => {
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.sharedFileSystemReconciler.reconcile(sfs)
+ } else if (type === 'DELETED') {
+ await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)
+ }
+ } catch (error) {
+ console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)
+ }
- // Re-reconcile any Host whose Context references this SharedFileSystem.
- // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on
- // the next mcp-host pod template diff so the pod stops failing to mount.
- try {
- await this.reconcileHostsReferencingSfs(sfs.name)
- } catch (error) {
- console.error(
- `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,
- error
- )
- }
+ // Re-reconcile any Host whose Context references this SharedFileSystem.
+ // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on
+ // the next mcp-host pod template diff so the pod stops failing to mount.
+ try {
+ await this.reconcileHostsReferencingSfs(sfs.name)
+ } catch (error) {
+ console.error(
+ `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,
+ error
+ )
+ }
+ })
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
+ this.sfsWatchRequest = null
if (err) {
console.error('[K8s] SharedFileSystem watch error:', err)
}
console.log('[K8s] SharedFileSystem watch ended, restarting...')
- setTimeout(() => this.startSharedFileSystemWatch(), err ? 5000 : 1000)
+ this.scheduleSharedFileSystemWatchRestart(err ? 5000 : 1000)
}
this.sfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
+ private scheduleSharedFileSystemWatchRestart(delayMs: number): void {
+ if (this.stopped || this.sfsWatchRestartTimer) return
+ this.sfsWatchRestartTimer = setTimeout(() => {
+ this.sfsWatchRestartTimer = null
+ if (this.stopped) return
+ void this.startSharedFileSystemWatch().catch(error => {
+ console.error('[K8s] SharedFileSystem background watch restart failed:', error)
+ this.scheduleSharedFileSystemWatchRestart(5000)
+ })
+ }, delayMs)
+ }
+
/**
* Watch the GlobalFileSystem singleton (gfs). ADDED/MODIFIED → reconcile the
* drive stack; DELETED → tear it down (respecting retainOnDelete). DISTINCT
@@ -1951,37 +2970,63 @@ export class McpServerWatcher implements McpServerProvider {
const watchCallback = async (
type: string,
- apiObj: { metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }
+ apiObj: {
+ metadata: { name: string; namespace?: string }
+ spec: GlobalFileSystemSpec
+ status?: GlobalFileSystemCRD['status']
+ }
) => {
const gfs: GlobalFileSystemCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || namespace,
spec: apiObj.spec,
+ status: apiObj.status,
}
console.log(`[K8s] GlobalFileSystem watch event: ${type} for ${gfs.name}`)
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- await this.gfsReconciler.reconcile(gfs)
- } else if (type === 'DELETED') {
- await this.gfsReconciler.reconcileDelete(gfs)
- }
- } catch (error) {
- console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ this.globalFileSystems.set(gfs.name, gfs)
+ } else if (type === 'DELETED') {
+ this.globalFileSystems.delete(gfs.name)
}
+ this.globalFileSystemCacheRevision += 1
+ await this.enqueueGlobalFileSystemReconciliation(gfs.name, async () => {
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.gfsReconciler.reconcile(gfs)
+ } else if (type === 'DELETED') {
+ await this.gfsReconciler.reconcileDelete(gfs)
+ }
+ } catch (error) {
+ console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)
+ }
+ })
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
+ this.gfsWatchRequest = null
if (err) {
console.error('[K8s] GlobalFileSystem watch error:', err)
}
console.log('[K8s] GlobalFileSystem watch ended, restarting...')
- setTimeout(() => this.startGlobalFileSystemWatch(), err ? 5000 : 1000)
+ this.scheduleGlobalFileSystemWatchRestart(err ? 5000 : 1000)
}
this.gfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
+ private scheduleGlobalFileSystemWatchRestart(delayMs: number): void {
+ if (this.stopped || this.gfsWatchRestartTimer) return
+ this.gfsWatchRestartTimer = setTimeout(() => {
+ this.gfsWatchRestartTimer = null
+ if (this.stopped) return
+ void this.startGlobalFileSystemWatch().catch(error => {
+ console.error('[K8s] GlobalFileSystem background watch restart failed:', error)
+ this.scheduleGlobalFileSystemWatchRestart(5000)
+ })
+ }, delayMs)
+ }
+
/**
* Extracted so unit tests can invoke the handler without spinning up a real
* K8s watch. Production calls this from `startCommunicationChannelWatch` and
@@ -2036,6 +3081,14 @@ export class McpServerWatcher implements McpServerProvider {
if (previousHostRef && previousHostRef !== cc.spec.hostRef) {
affectedHostRefs.add(previousHostRef)
}
+ if (!this.isHostInventoryAuthoritative()) {
+ const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()
+ this.admitHostDependentEffects(
+ 'CommunicationChannel event Host convergence',
+ lifecycleGeneration
+ )
+ return
+ }
let needsFleetRetry = false
for (const hostRef of affectedHostRefs) {
if (!(await this.reconcileHostsReferencingCC(hostRef))) {
@@ -2197,8 +3250,41 @@ export class McpServerWatcher implements McpServerProvider {
private async runGfsResync(): Promise<void> {
if (this.stopped) return
try {
+ const cacheRevisionAtListStart = this.globalFileSystemCacheRevision
const gfses = await listAllGlobalFileSystems()
- await this.gfsReconciler.fullReconcile(gfses)
+ const names = [
+ ...this.globalFileSystems.keys(),
+ ...gfses.map(globalFileSystem => globalFileSystem.name),
+ ]
+ return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>
+ this.runGfsResyncCore(gfses, cacheRevisionAtListStart)
+ )
+ } catch (err) {
+ console.error(
+ `[K8s] GlobalFileSystem periodic resync failed: ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ )
+ }
+ }
+
+ private async runGfsResyncCore(
+ listedGfses?: GlobalFileSystemCRD[],
+ listedAtCacheRevision?: number
+ ): Promise<void> {
+ if (this.stopped) return
+ try {
+ const cacheRevisionAtListStart = listedAtCacheRevision ?? this.globalFileSystemCacheRevision
+ const gfses = listedGfses ?? (await listAllGlobalFileSystems())
+ let reconcileInventory: GlobalFileSystemCRD[]
+ if (cacheRevisionAtListStart === this.globalFileSystemCacheRevision) {
+ this.globalFileSystems.clear()
+ for (const gfs of gfses) this.globalFileSystems.set(gfs.name, gfs)
+ reconcileInventory = gfses
+ } else {
+ reconcileInventory = [...this.globalFileSystems.values()]
+ }
+ await this.gfsReconciler.fullReconcile(reconcileInventory)
} catch (err) {
console.error(
`[K8s] GlobalFileSystem periodic resync failed: ${
@@ -2211,9 +3297,36 @@ export class McpServerWatcher implements McpServerProvider {
private async runSfsResync(): Promise<void> {
if (this.stopped) return
try {
+ const cacheRevisionAtListStart = this.sharedFileSystemCacheRevision
const sfses = await listAllSharedFileSystems()
- this.sharedFileSystems.clear()
- for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)
+ const names = [
+ ...this.sharedFileSystems.keys(),
+ ...sfses.map(sharedFileSystem => sharedFileSystem.name),
+ ]
+ return this.enqueueSharedFileSystemFleetReconciliation(names, () =>
+ this.runSfsResyncCore(sfses, cacheRevisionAtListStart)
+ )
+ } catch (error) {
+ console.error('[K8s] Periodic SharedFileSystem resync failed:', error)
+ }
+ }
+
+ private async runSfsResyncCore(
+ listedSfses?: SharedFileSystemCRD[],
+ listedAtCacheRevision?: number
+ ): Promise<void> {
+ if (this.stopped) return
+ try {
+ const cacheRevisionAtListStart = listedAtCacheRevision ?? this.sharedFileSystemCacheRevision
+ const sfses = listedSfses ?? (await listAllSharedFileSystems())
+ let reconcileInventory: SharedFileSystemCRD[]
+ if (cacheRevisionAtListStart === this.sharedFileSystemCacheRevision) {
+ this.sharedFileSystems.clear()
+ for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)
+ reconcileInventory = sfses
+ } else {
+ reconcileInventory = [...this.sharedFileSystems.values()]
+ }
// Capture each SFS's mountability (PVC Bound) BEFORE reconciling so we can
// detect a flip. resolveContextMounts() injects the RO mount + podAffinity
@@ -2222,11 +3335,11 @@ export class McpServerWatcher implements McpServerProvider {
// on phase==='Ready': a transient wfc readiness dip while the PVC stays
// Bound must NOT re-roll consumers — see resolveContextMounts.)
const wasMountable = new Map<string, boolean>()
- for (const sfs of sfses) {
+ for (const sfs of reconcileInventory) {
wasMountable.set(sfs.name, this.sharedFileSystemReconciler.isMountable(sfs))
}
- await this.sharedFileSystemReconciler.fullReconcile(sfses)
+ await this.sharedFileSystemReconciler.fullReconcile(reconcileInventory)
// #592 gap fix: heal the consuming mcp-host Deployments on a mountability
// flip — inject the mount + affinity once the PVC binds, or drop it if the
@@ -2236,7 +3349,7 @@ export class McpServerWatcher implements McpServerProvider {
// an already-Bound SFS picked up on HCC restart still re-injects its mount
// within one resync interval instead of waiting for the (much longer) Host
// resync.
- for (const sfs of sfses) {
+ for (const sfs of reconcileInventory) {
const nowMountable = this.sharedFileSystemReconciler.isMountable(sfs)
if (nowMountable !== (wasMountable.get(sfs.name) ?? false)) {
await this.reconcileHostsReferencingSfs(sfs.name)
@@ -2247,86 +3360,14 @@ export class McpServerWatcher implements McpServerProvider {
}
}
- private async runExternalEgressResync(): Promise<void> {
- if (this.stopped) return
- const servers = [...this.servers.values()]
- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
- .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))
-
- if (servers.length === 0) return
-
- console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)
- let index = 0
- const workers = Array.from(
- { length: Math.min(EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY, servers.length) },
- async () => {
- while (!this.stopped) {
- const server = servers[index++]
- if (!server) return
- const key = this.externalEgressRetryKey(server)
- if (this.externalEgressInFlight.has(key)) {
- console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)
- continue
- }
- if (this.externalEgressRetryTimers.has(key)) {
- console.log(`[K8s] Skipping external egress resync for ${key}; retry already scheduled`)
- continue
- }
- const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)
- if (jitterMs > 0) await delay(jitterMs)
- try {
- await this.runExternalEgressOnce('MODIFIED', server)
- } catch (error) {
- console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)
- this.scheduleExternalEgressRetry('MODIFIED', server)
- }
- }
- }
- )
- await Promise.all(workers)
- }
-
- private async reconcileInitialExternalEgressBeforeRuntime(
- servers: McpServerCRD[]
- ): Promise<Set<string>> {
- const failures = new Set<string>()
- const serversWithExternalEgress = servers
- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
- .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))
-
- if (serversWithExternalEgress.length === 0) return failures
-
- console.log(
- `[K8s] Reconciling external egress before startup runtime reconciliation for ` +
- `${serversWithExternalEgress.length} McpServer(s)`
- )
-
- for (const server of serversWithExternalEgress) {
- const key = this.externalEgressRetryKey(server)
- try {
- await this.runExternalEgressOnce('MODIFIED', server)
- } catch (error) {
- failures.add(key)
- console.error(
- `[K8s] Initial external egress reconciliation failed for ${key}; ` +
- 'runtime reconciliation will stay blocked until retry succeeds:',
- error
- )
- this.scheduleExternalEgressRetry('MODIFIED', server)
- }
- }
-
- return failures
- }
-
/**
* Start watching McpServer CRDs.
*/
- private getMcpServerWatchCallback(): (
- type: string,
- apiObj: McpServerWatchObject
- ) => Promise<void> {
+ private getMcpServerWatchCallback(
+ watchGeneration = this.mcpWatchGeneration
+ ): (type: string, apiObj: McpServerWatchObject) => Promise<void> {
return async (type, apiObj) => {
+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
const server: McpServerCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.namespace,
@@ -2340,227 +3381,324 @@ export class McpServerWatcher implements McpServerProvider {
console.log(`[K8s] McpServer watch event: ${type} for ${server.name}`)
+ const previous = this.servers.get(server.name)
+
// Update cache
+ let desiredStateChanged = false
if (type === 'ADDED' || type === 'MODIFIED') {
+ desiredStateChanged =
+ previous === undefined || !sameMcpServerDesiredRevision(previous, server)
this.servers.set(server.name, server)
} else if (type === 'DELETED') {
- this.servers.delete(server.name)
+ if (!previous?.uid || !server.uid || previous.uid === server.uid) {
+ desiredStateChanged = this.servers.delete(server.name)
+ }
+ }
+ if (desiredStateChanged) {
+ this.mcpServerDesiredRevision += 1
+ void this.runInitialNetworkPolicyConvergence()
}
- // External egress is part of the workload's pre-start contract. Reconcile
- // it before HCC creates or updates any managed runtime Deployment so a
- // stdio MCP with egressBindings cannot start before ExternalEgressReady.
- if (type === 'ADDED' || type === 'MODIFIED') {
- try {
- await this.runExternalEgressOnce(type, server)
- } catch (error) {
- console.error(
- `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
- error
- )
- this.scheduleExternalEgressRetry(type, server)
- this.changeCallback?.()
- return
- }
+ // HCC writes McpServer status during reconciliation. Those writes emit
+ // MODIFIED events but do not change the desired runtime or policy state.
+ // Suppress that self-induced work while still publishing the fresh status
+ // through discovery. Any UID/spec/annotation/label change remains live.
+ if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {
+ this.changeCallback?.()
+ return
}
- // Trigger deployment reconciliation
+ await this.enqueueMcpServerReconciliation(server, () =>
+ this.reconcileMcpServerWatchEvent(
+ type as ExternalEgressWatchEventType,
+ server,
+ watchGeneration
+ )
+ )
+ }
+ }
+
+ private async reconcileMcpServerWatchEvent(
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ watchGeneration: number,
+ retry?: ExternalEgressRetryHandle
+ ): Promise<void> {
+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
+ if (retry && !retry.isCurrent()) return
+ if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
+ this.scheduleExternalEgressRetry(type, server)
+ return
+ }
+ const current = this.servers.get(server.name)
+ if (type === 'DELETED') {
+ // A same-name recreation may already be accepted while this callback
+ // waits for its per-server effect. The newer identity owns the key.
+ if (current) return
+ } else {
+ if (!current || current.namespace !== server.namespace) return
+ server = current
+ }
+ const deleteAllowed = () =>
+ this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)
+ const contextInventoryGeneration = this.contextWatchGeneration
+ const contextPolicyInventoryIsCurrent = (): boolean =>
+ this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
+ this.hasContextInventoryAuthority(contextInventoryGeneration)
+
+ // External egress is part of the workload's pre-start contract. Reconcile
+ // it before HCC creates or updates any managed runtime Deployment so a
+ // stdio MCP with egressBindings cannot start before ExternalEgressReady.
+ let retryCompletion: ExternalEgressRetryHandle | undefined
+ if (type === 'ADDED' || type === 'MODIFIED') {
try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- await this.reconciler.reconcile(server)
- } else if (type === 'DELETED') {
- await this.reconciler.reconcileDelete(server.name, server.namespace)
- }
+ retryCompletion = await this.runExternalEgressOnce(type, server, {
+ retry,
+ isCurrent: () =>
+ this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
+ (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),
+ })
} catch (error) {
- console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)
- if (
- (type === 'ADDED' || type === 'MODIFIED') &&
- (server.spec.egressBindings?.length ?? 0) > 0
- ) {
- this.scheduleExternalEgressRetry(type, server)
- }
+ console.error(
+ `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
+ error
+ )
+ this.scheduleExternalEgressRetry(type, server)
+ this.changeCallback?.()
+ return
}
-
- // Trigger binding policy reconciliation (L3 ingress/egress)
+ } else {
+ if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) return
try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']
- if (bindingsJson) {
- const bindings: BindingDef[] = JSON.parse(bindingsJson)
- const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
- const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name
- await this.bindingReconciler.reconcileBindings(
- recipeName,
- bindings,
- mcpWorkloadName,
- server.name
- )
- }
- // Re-reconcile any cached Context that references this server.
- // Fixes race condition where Context MODIFIED arrives before
- // the McpServer ADDED event populates the cache.
- for (const ctx of this.contexts.values()) {
- if (ctx.spec.mcpServers?.includes(server.name)) {
- console.log(
- `[K8s] Re-reconciling context "${ctx.name}" after McpServer "${server.name}" cached`
- )
- await this.netPolReconciler.reconcileContext(ctx)
- }
- }
- } else if (type === 'DELETED') {
+ retryCompletion = await this.runExternalEgressOnce(type, server, {
+ deleteAllowed,
+ retry,
+ isCurrent: () =>
+ !this.stopped &&
+ watchGeneration === this.mcpWatchGeneration &&
+ (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),
+ })
+ } catch (error) {
+ console.error(
+ `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
+ error
+ )
+ this.scheduleExternalEgressRetry(type, server)
+ this.changeCallback?.()
+ return
+ }
+ }
+
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
+ this.scheduleExternalEgressRetry(type, server)
+ return
+ }
+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
+ server = this.servers.get(server.name)!
+ } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {
+ return
+ }
+
+ // Trigger deployment reconciliation.
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ const currentServer = server
+ await this.reconciler.reconcile(currentServer, {
+ isCurrent: () => this.mcpServerWatchEffectIsCurrent(currentServer, watchGeneration),
+ })
+ } else if (type === 'DELETED') {
+ await this.reconciler.reconcileDelete(server.name, server.namespace)
+ }
+ } catch (error) {
+ console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)
+ this.scheduleExternalEgressRetry(type, server)
+ this.changeCallback?.()
+ return
+ }
+
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
+ server = this.servers.get(server.name)!
+ } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {
+ return
+ }
+
+ // Trigger binding policy reconciliation (L3 ingress/egress).
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']
+ if (bindingsJson) {
+ const bindings: BindingDef[] = JSON.parse(bindingsJson)
const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
- await this.bindingReconciler.cleanupBindings(recipeName)
- await this.runExternalEgressOnce(type, server)
+ const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name
+ const bindingServer = server
+ await this.bindingReconciler.reconcileBindings(
+ recipeName,
+ bindings,
+ mcpWorkloadName,
+ bindingServer.name,
+ {
+ isCurrent: () => this.mcpServerWatchEffectIsCurrent(bindingServer, watchGeneration),
+ }
+ )
+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
+ server = this.servers.get(server.name)!
}
- } catch (error) {
- console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)
- if (type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED') {
- this.scheduleExternalEgressRetry(type, server)
+ // Re-reconcile cached Contexts after the server cache is populated.
+ for (const selectedContext of this.contexts.values()) {
+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
+ if (!selectedContext.spec.mcpServers?.includes(server.name)) continue
+ const selectedContextId = selectedContext.spec.contextId
+ console.log(
+ `[K8s] Re-reconciling context "${selectedContext.name}" after McpServer "${server.name}" cached`
+ )
+ await this.enqueueContextReconciliation(selectedContextId, async () => {
+ if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
+ const currentContext = this.contexts.get(selectedContext.name)
+ if (
+ currentContext?.spec.contextId !== selectedContextId ||
+ !currentContext.spec.mcpServers?.includes(server.name)
+ ) {
+ return
+ }
+ await this.netPolReconciler.reconcileContext(currentContext, {
+ isCurrent: () =>
+ contextPolicyInventoryIsCurrent() &&
+ this.contexts.get(currentContext.name) === currentContext,
+ })
+ })
}
+ } else if (type === 'DELETED') {
+ const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
+ await this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })
}
-
- // Notify listeners
+ } catch (error) {
+ console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)
+ this.scheduleExternalEgressRetry(type, server)
this.changeCallback?.()
+ return
}
+
+ retryCompletion?.complete()
+ this.changeCallback?.()
}
- private async startMcpServerWatch(): Promise<void> {
+ private async startMcpServerWatch(resourceVersion: string): Promise<number> {
+ this.requireInventoryResourceVersion('McpServer', resourceVersion)
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_MCPSERVERS}`
console.log(`[K8s] Starting McpServer watch`)
+ const watchGeneration = ++this.mcpWatchGeneration
+ if (this.mcpWatchRequest) {
+ this.mcpWatchRequest.abort()
+ this.mcpWatchRequest = null
+ }
+ let watchEnded = false
- const watchCallback = this.getMcpServerWatchCallback()
+ const watchCallback = this.getMcpServerWatchCallback(watchGeneration)
const doneCallback = (err: Error | null) => {
- if (this.stopped) return
+ if (watchEnded) return
+ watchEnded = true
+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
+ if (!this.retireMcpServerWatch(watchGeneration)) return
if (err) {
console.error('[K8s] McpServer watch error:', err)
}
- console.log('[K8s] McpServer watch ended, restarting...')
- setTimeout(() => this.startMcpServerWatch(), err ? 5000 : 1000)
+ console.log('[K8s] McpServer watch ended; recovering authoritative inventory...')
+ this.scheduleMcpServerCacheRecovery()
}
- this.mcpWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
+ const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {
+ request.abort()
+ return watchGeneration
+ }
+ this.mcpWatchRequest = request
+ return watchGeneration
}
- private externalEgressRetryKey(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {
- return `${server.namespace}/${server.name}`
+ private mcpServerWatchEffectIsCurrent(expected: McpServerCRD, watchGeneration: number): boolean {
+ if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return false
+ const current = this.servers.get(expected.name)
+ return current !== undefined && sameMcpServerDesiredRevision(expected, current)
}
- private async runExternalEgressOnce(
- type: string,
+ private runExternalEgressOnce(
+ type: ExternalEgressWatchEventType,
server: McpServerCRD,
- options: { clearRetryOnSuccess?: boolean } = {}
- ): Promise<void> {
- const clearRetryOnSuccess = options.clearRetryOnSuccess ?? true
- const key = this.externalEgressRetryKey(server)
- const existing = this.externalEgressInFlight.get(key)
- if (existing) {
- console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)
- await existing
- }
-
- const run = (async () => {
- if (type === 'DELETED') {
- await this.netPolReconciler.cleanupExternalEgress(server.name, server.namespace)
- } else {
- await this.netPolReconciler.reconcileExternalEgress(server)
- }
- })()
- this.externalEgressInFlight.set(key, run)
- try {
- await run
- if (clearRetryOnSuccess) {
- this.clearExternalEgressRetry(server)
- }
- } finally {
- if (this.externalEgressInFlight.get(key) === run) {
- this.externalEgressInFlight.delete(key)
- }
- }
- }
-
- private clearExternalEgressRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {
- const key = this.externalEgressRetryKey(server)
- const timer = this.externalEgressRetryTimers.get(key)
- if (timer) {
- clearTimeout(timer)
- this.externalEgressRetryTimers.delete(key)
- }
- this.externalEgressRetryAttempts.delete(key)
+ options: {
+ deleteAllowed?: () => Promise<boolean>
+ retry?: ExternalEgressRetryHandle
+ isCurrent?: () => boolean
+ } = {}
+ ): Promise<ExternalEgressRetryHandle | undefined> {
+ return this.externalEgressCoordinator.reconcile(type, server, options)
}
private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {
- if (type !== 'ADDED' && type !== 'MODIFIED' && type !== 'DELETED') return
- if (this.stopped) return
-
- const key = this.externalEgressRetryKey(server)
- if (this.externalEgressRetryTimers.has(key)) return
-
- const attempt = (this.externalEgressRetryAttempts.get(key) ?? 0) + 1
- const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
- if (!delayMs) {
- console.error(
- `[K8s] External egress retry exhausted for McpServer "${server.name}" in namespace "${server.namespace}"`
- )
- return
- }
-
- this.externalEgressRetryAttempts.set(key, attempt)
- console.warn(
- `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length} ` +
- `for McpServer "${server.name}" in ${delayMs}ms`
- )
-
- const timer = setTimeout(() => {
- this.externalEgressRetryTimers.delete(key)
- void this.retryExternalEgress(type, server)
- }, delayMs)
- this.externalEgressRetryTimers.set(key, timer)
+ this.externalEgressCoordinator.scheduleRetry(type, server)
}
- private async retryExternalEgress(type: string, server: McpServerCRD): Promise<void> {
- if (this.stopped) return
- try {
- if (type === 'DELETED') {
- await this.runExternalEgressOnce(type, server)
- } else {
- const current = this.servers.get(server.name)
- if (!current || current.namespace !== server.namespace) {
- this.clearExternalEgressRetry(server)
- return
- }
- await this.runExternalEgressOnce(type, current, { clearRetryOnSuccess: false })
- try {
- await this.reconciler.reconcile(current)
- this.clearExternalEgressRetry(current)
- } catch (error) {
- console.error(
- `[K8s] Runtime reconciliation after external egress retry failed for ${server.name}:`,
- error
- )
- this.scheduleExternalEgressRetry(type, current)
- }
+ private async performExternalEgressMutation(
+ type: ExternalEgressWatchEventType,
+ server: McpServerCRD,
+ options: {
+ deleteAllowed?: () => Promise<boolean>
+ isCurrent: () => boolean
+ }
+ ): Promise<void> {
+ if (type === 'DELETED') {
+ const deleteAllowed = async (): Promise<boolean> => {
+ if (!options.isCurrent()) return false
+ if (options.deleteAllowed && !(await options.deleteAllowed())) return false
+ return options.isCurrent()
}
- } catch (error) {
- console.error(`[K8s] External egress retry failed for ${server.name}:`, error)
- this.scheduleExternalEgressRetry(type, server)
+ await this.netPolReconciler.cleanupExternalEgress(
+ server.name,
+ server.namespace,
+ undefined,
+ deleteAllowed
+ )
+ return
}
+ await this.netPolReconciler.reconcileExternalEgress(server, {
+ isCurrent: options.isCurrent,
+ })
}
/**
* Start watching Context CRDs for NetworkPolicy reconciliation.
*/
- private async startContextWatch(): Promise<void> {
+ private async startContextWatch(resourceVersion: string): Promise<number> {
+ this.requireInventoryResourceVersion('Context', resourceVersion)
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_CONTEXTS}`
console.log(`[K8s] Starting Context watch`)
+ const watchGeneration = ++this.contextWatchGeneration
+ if (this.ctxWatchRequest) {
+ this.ctxWatchRequest.abort()
+ this.ctxWatchRequest = null
+ }
+ let watchEnded = false
const watchCallback = async (
type: string,
- apiObj: { metadata: { name: string; namespace?: string }; spec: ContextSpec }
+ apiObj: {
+ metadata: {
+ name: string
+ namespace?: string
+ uid?: string
+ generation?: number
+ }
+ spec: ContextSpec
+ }
) => {
+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
const context: ContextCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.namespace,
+ uid: apiObj.metadata.uid,
+ generation: apiObj.metadata.generation,
spec: apiObj.spec,
}
@@ -2569,58 +3707,168 @@ export class McpServerWatcher implements McpServerProvider {
// Capture the previous spec before mutating the cache so we can
// re-reconcile any SFS that *was* referenced but is no longer.
const previous = this.contexts.get(context.name)
+ // A scoped Context delta may replace only a prior, fully authoritative
+ // safety certificate. Capturing it before this event changes desired
+ // state proves there was no outstanding Context or McpServer revocation
+ // work that the scoped reconciliation could accidentally certify.
+ const previousSafetyCertificate = this.currentNetworkPolicySafetyCertificate()
// Cache the context for cross-resource re-reconciliation
+ let desiredStateChanged = false
+ let deltaSafetyCertificate: NetworkPolicySafetyCertificate | undefined
if (type === 'ADDED' || type === 'MODIFIED') {
+ desiredStateChanged =
+ previous === undefined || !sameContextDesiredRevision(previous, context)
this.contexts.set(context.name, context)
} else if (type === 'DELETED') {
- this.contexts.delete(context.name)
+ desiredStateChanged = this.contexts.delete(context.name)
+ }
+ if (desiredStateChanged) {
+ this.contextDesiredRevision += 1
+ // A same-identity Context MODIFIED event can complete its own stale
+ // allow revocation below. Once that exact delta is current, it is safe
+ // to restore readiness without waiting for an older pass's additive
+ // fleet. Deletes and identity changes remain fail-closed until the
+ // authoritative full pass certifies their broader cleanup.
+ if (
+ type === 'MODIFIED' &&
+ previous?.uid === context.uid &&
+ previous?.spec.contextId === context.spec.contextId &&
+ previousSafetyCertificate !== null
+ ) {
+ deltaSafetyCertificate = {
+ contextGeneration: previousSafetyCertificate.contextGeneration,
+ serverGeneration: previousSafetyCertificate.serverGeneration,
+ contextRevision: this.contextDesiredRevision,
+ serverRevision: previousSafetyCertificate.serverRevision,
+ }
+ }
+ void this.runInitialNetworkPolicyConvergence()
}
// Re-reconcile every SFS this Context referenced before or after the
// change so SharedFileSystem.status.mountedByContexts stays in sync
// without waiting for an SFS-level event.
const nextForSfs = type === 'DELETED' ? undefined : context
- void this.reconcileSharedFileSystemsReferencedByContext(previous, nextForSfs)
+ const contextInventoryAuthoritative = () => this.hasContextInventoryAuthority(watchGeneration)
+ const serverInventoryGeneration = this.mcpWatchGeneration
+ void this.reconcileSharedFileSystemsReferencedByContext(
+ previous,
+ nextForSfs,
+ contextInventoryAuthoritative
+ )
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- await this.netPolReconciler.reconcileContext(context)
- } else if (type === 'DELETED') {
- await this.netPolReconciler.reconcileDeleteContext(context.spec.contextId)
+ const previousContextId = previous?.spec.contextId
+ const effectContextIds =
+ previousContextId && previousContextId !== context.spec.contextId
+ ? [previousContextId, context.spec.contextId]
+ : [context.spec.contextId]
+ await this.enqueueContextIdentityReconciliation(effectContextIds, async () => {
+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
+
+ try {
+ if (previousContextId && previousContextId !== context.spec.contextId) {
+ const oldIdDeleteAllowed = () =>
+ this.contextIdAbsentForDelete(previousContextId, watchGeneration)
+ if (await oldIdDeleteAllowed()) {
+ await this.netPolReconciler.reconcileDeleteContext(
+ previousContextId,
+ oldIdDeleteAllowed
+ )
+ }
+ }
+
+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
+ const current = this.contexts.get(context.name)
+ if (type === 'DELETED') {
+ if (current) return
+ } else if (current?.spec.contextId !== context.spec.contextId) {
+ return
+ }
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ const selectedContext = current!
+ // A scoped delta may only replace the safety certificate when its
+ // own stale-allow revocation ran to completion — `reconcileContext`
+ // aborts mid-pass whenever its authority fence breaks — and when
+ // the last authoritative pass certified its namespace-wide
+ // inventory, which a label-scoped delta cannot vouch for.
+ // `deltaSafetyCertificate` is only ever built for MODIFIED, so this
+ // is the one branch where it can be recorded.
+ const scopedRevocationCompleted = await this.netPolReconciler.reconcileContext(
+ selectedContext,
+ {
+ isCurrent: () =>
+ contextInventoryAuthoritative() &&
+ this.hasMcpServerInventoryAuthority(serverInventoryGeneration) &&
+ this.contexts.get(selectedContext.name) === selectedContext,
+ // Only opt in when there is a certificate to withhold. Without
+ // one the result below is discarded, so a reported lost fence
+ // would vanish: no throw to reach the retry, and the recovery
+ // in the desired-state branch does not run for an event that
+ // did not move the revision.
+ honorsLostFence: deltaSafetyCertificate !== undefined,
+ }
+ )
+ if (deltaSafetyCertificate && scopedRevocationCompleted) {
+ if (this.netPolReconciler.hasCertifiedSafetyInventory()) {
+ this.recordNetworkPolicySafetyCertificate(deltaSafetyCertificate)
+ } else {
+ console.warn(
+ `[K8s] Not certifying the scoped delta for context "${context.spec.contextId}": the last authoritative safety pass ended without certifying its namespace-wide inventory`
+ )
+ }
+ }
+ } else if (type === 'DELETED') {
+ const deleteAllowed = () =>
+ this.contextAbsentForDelete(context.name, context.namespace, watchGeneration)
+ if (!(await deleteAllowed())) return
+ await this.netPolReconciler.reconcileDeleteContext(
+ context.spec.contextId,
+ deleteAllowed
+ )
+ }
+ } catch (error) {
+ console.error(
+ `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,
+ error
+ )
+ void this.runInitialNetworkPolicyConvergence()
+ return
}
- } catch (error) {
- console.error(
- `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,
- error
- )
- }
- // Re-reconcile any Host that points at this Context so that changes to
- // spec.sharedFileSystems[] propagate into the mcp-host pod template.
- try {
- for (const host of this.hosts.values()) {
- if (host.spec.contextRef !== context.name) continue
- await this.hostReconciler.reconcile(host)
+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
+ // Re-reconcile any Host that points at this Context so that changes to
+ // spec.sharedFileSystems[] propagate into the mcp-host pod template.
+ try {
+ await this.reconcileHostsReferencingContext(context.name, contextInventoryAuthoritative)
+ } catch (error) {
+ console.error(
+ `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,
+ error
+ )
}
- } catch (error) {
- console.error(
- `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,
- error
- )
- }
+ })
}
const doneCallback = (err: Error | null) => {
- if (this.stopped) return
+ if (watchEnded) return
+ watchEnded = true
+ if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
+ if (!this.retireContextWatch(watchGeneration)) return
if (err) {
console.error('[K8s] Context watch error:', err)
}
- console.log('[K8s] Context watch ended, restarting...')
- setTimeout(() => this.startContextWatch(), err ? 5000 : 1000)
+ console.log('[K8s] Context watch ended; recovering authoritative inventory...')
+ this.scheduleContextCacheRecovery()
}
- this.ctxWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
+ const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
+ if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {
+ request.abort()
+ return watchGeneration
+ }
+ this.ctxWatchRequest = request
+ return watchGeneration
}
/**
@@ -2722,25 +3970,39 @@ export class McpServerWatcher implements McpServerProvider {
*/
async stop(): Promise<void> {
this.stopped = true
+ this.externalEgressCoordinator.stop()
+ this.retireMcpServerWatch()
+ this.retireContextWatch()
this.ccCacheSynced = false
this.hostCacheSynced = false
this.hostWatchGeneration += 1
this.ccWatchGeneration += 1
- this.resolveHostFleetShutdown()
- this.hostFleetReconcilePending?.resolve()
- this.hostFleetReconcilePending = null
+ this.hostFleetScheduler.stop()
if (this.ccCacheRecoveryTimer) {
clearTimeout(this.ccCacheRecoveryTimer)
this.ccCacheRecoveryTimer = null
}
- if (this.ccFleetRetryTimer) {
- clearTimeout(this.ccFleetRetryTimer)
- this.ccFleetRetryTimer = null
- }
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
+ if (this.sfsWatchRestartTimer) {
+ clearTimeout(this.sfsWatchRestartTimer)
+ this.sfsWatchRestartTimer = null
+ }
+ if (this.gfsWatchRestartTimer) {
+ clearTimeout(this.gfsWatchRestartTimer)
+ this.gfsWatchRestartTimer = null
+ }
+ this.hostCacheRecoveryIntent = null
+ if (this.mcpServerCacheRecoveryTimer) {
+ clearTimeout(this.mcpServerCacheRecoveryTimer)
+ this.mcpServerCacheRecoveryTimer = null
+ }
+ if (this.contextCacheRecoveryTimer) {
+ clearTimeout(this.contextCacheRecoveryTimer)
+ this.contextCacheRecoveryTimer = null
+ }
this.clearAllHostWatchRetries()
if (this.resyncTimer) {
clearInterval(this.resyncTimer)
@@ -2754,16 +4016,11 @@ export class McpServerWatcher implements McpServerProvider {
clearInterval(this.gfsResyncTimer)
this.gfsResyncTimer = null
}
- if (this.externalEgressResyncTimer) {
- clearTimeout(this.externalEgressResyncTimer)
- this.externalEgressResyncTimer = null
- }
- for (const timer of this.externalEgressRetryTimers.values()) {
+ for (const timer of this.initialConvergenceRetryTimers.values()) {
clearTimeout(timer)
}
- this.externalEgressRetryTimers.clear()
- this.externalEgressRetryAttempts.clear()
- this.externalEgressInFlight.clear()
+ this.initialConvergenceRetryTimers.clear()
+ this.initialConvergenceRetryAttempts.clear()
if (this.mcpWatchRequest) {
console.log('[K8s] Stopping McpServer watch')
this.mcpWatchRequest.abort()
diff --git a/host-context-controller/src/main.ts b/host-context-controller/src/main.ts
index 0e1502aa5..565c537d3 100644
--- a/host-context-controller/src/main.ts
+++ b/host-context-controller/src/main.ts
@@ -18,6 +18,7 @@ import {
createMcpServerProvider,
getKubeConfig,
} from './k8sClient'
+import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
import { ContextMapperServer } from './server'
import { StatelessLifecycleTracker } from './statelessLifecycleTracker'
import {
@@ -113,6 +114,16 @@ async function main(): Promise<void> {
const hasDesktopFn = hostReconciler
? (hostRef: string) => hostReconciler.hasDesktop(hostRef)
: undefined
+ // Dev (no watcher) gets an explicit always-authoritative gate: no inventory
+ // to certify there, and the server's fail-closed default would otherwise pin
+ // /ready at 503 forever (R3-B1). See resolveProviderAuthoritativeFn.
+ const providerAuthoritativeFn = resolveProviderAuthoritativeFn(watcher)
+ // Desktop status gates on Host inventory authority alone (not full readiness).
+ // Dev (no watcher) mirrors the provider gate above: there is no Host inventory
+ // to certify, so authority is unconditional (permissive) — otherwise the
+ // server's fail-closed default would pin /api/v1/desktop/* at 503 forever in
+ // dev (R2-M1). See resolveHostAuthoritativeFn.
+ const hostAuthoritativeFn = resolveHostAuthoritativeFn(watcher)
// Stateless heartbeat consumption — mcp-host pods authenticate their
// heartbeats toward control-api's /mcp-host facade (control-api is the
@@ -145,16 +156,23 @@ async function main(): Promise<void> {
`target=${config.controlApiBaseUrl})`
)
}
- server = new ContextMapperServer(provider, config.port, hostReconciler, hasDesktopFn)
+ server = new ContextMapperServer(
+ provider,
+ config.port,
+ hostReconciler,
+ hasDesktopFn,
+ providerAuthoritativeFn,
+ hostAuthoritativeFn
+ )
await server.start()
// One-shot legacy sweep: delete the static `clerum-channel-reader`
// Deployment if it still exists in the channels namespace. MUST run
- // BEFORE provider.start(): provider.start() invokes fullReconcile on
- // the initial Host list, which creates per-Host channel-reader Deployments
- // that immediately try to long-poll Telegram. If the static is still
- // alive at that moment, both pods compete on the same bot's getUpdates
- // and Telegram 409s one of them. Sweeping first guarantees zero overlap.
+ // BEFORE provider.start(): provider startup establishes the Host inventory
+ // watch and schedules its initial fleet convergence, which creates per-Host
+ // channel-reader Deployments in the background. If the static is still alive
+ // when that work begins, both pods compete on the same bot's getUpdates and
+ // Telegram 409s one of them. Sweeping first guarantees zero overlap.
// Idempotent; 404 (already gone) is the steady state. See issue #273
// for the empirical reproduction.
if (!config.devMode && hostReconciler) {
@@ -199,12 +217,11 @@ async function main(): Promise<void> {
// reconcileChannelReaderRevision is self-contained (does not throw), so
// the .catch here is defensive — it catches any unexpected runtime error
// in the SecretInformer's own callback path.
- if (!config.devMode && hostReconciler) {
+ if (!config.devMode && watcher) {
const kc = getKubeConfig()
if (kc) {
- const reconciler = hostReconciler
channelSecretInformer = new SecretInformer(kc, config.channelsNamespace, evt => {
- reconciler.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {
+ watcher.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {
console.error('[Main] reconcileChannelReaderRevision failed:', err)
})
})
diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.ts
new file mode 100644
index 000000000..19f75caf3
--- /dev/null
+++ b/host-context-controller/src/mcpServerSafety.test.ts
@@ -0,0 +1,267 @@
+import { describe, expect, it, vi } from 'vitest'
+import {
+ confirmAuthoritativeMcpServerAbsence,
+ isMcpServerStatusOnlyUpdate,
+ sameMcpServerDesiredRevision,
+} from './mcpServerSafety'
+import type { McpServerCRD } from './types'
+
+function makeServer(overrides: Partial<McpServerCRD> = {}): McpServerCRD {
+ return {
+ name: 'web-search',
+ namespace: 'mcp-server',
+ uid: 'web-search-uid',
+ generation: 7,
+ annotations: { 'clerum.io/recipe': 'research' },
+ labels: { 'clerum.io/workload': 'web-search' },
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ ...overrides,
+ }
+}
+
+describe('sameMcpServerDesiredRevision', () => {
+ it('matches equal desired state while ignoring status and one-sided identity metadata', () => {
+ const expected = makeServer({
+ status: { conditions: [{ type: 'Ready', status: 'False' }] },
+ })
+ const current = makeServer({
+ uid: undefined,
+ generation: undefined,
+ status: { conditions: [{ type: 'Ready', status: 'True' }] },
+ })
+
+ expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)
+ expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)
+ })
+
+ it('rejects a different resource key', () => {
+ const expected = makeServer()
+
+ expect(sameMcpServerDesiredRevision(expected, makeServer({ name: 'doc-generator' }))).toBe(
+ false
+ )
+ expect(sameMcpServerDesiredRevision(expected, makeServer({ namespace: 'other' }))).toBe(false)
+ })
+
+ it('rejects differing UID or generation only when both sides provide the field', () => {
+ const expected = makeServer()
+
+ expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: 'replacement-uid' }))).toBe(
+ false
+ )
+ expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: 8 }))).toBe(false)
+ expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: undefined }))).toBe(true)
+ expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: undefined }))).toBe(true)
+ })
+
+ it('compares desired object fields by semantic content, not Kubernetes map key order', () => {
+ const expected = makeServer({
+ annotations: { first: '1', second: '2' },
+ labels: { first: '1', second: '2' },
+ })
+
+ expect(
+ sameMcpServerDesiredRevision(
+ expected,
+ makeServer({
+ spec: {
+ ...expected.spec,
+ image: 'clerum/web-search:next',
+ },
+ labels: expected.labels,
+ })
+ )
+ ).toBe(false)
+ expect(
+ sameMcpServerDesiredRevision(
+ expected,
+ makeServer({
+ annotations: { second: '2', first: '1' },
+ labels: expected.labels,
+ })
+ )
+ ).toBe(true)
+ expect(
+ sameMcpServerDesiredRevision(
+ expected,
+ makeServer({
+ annotations: { 'clerum.io/recipe': 'other' },
+ labels: expected.labels,
+ })
+ )
+ ).toBe(false)
+ expect(
+ sameMcpServerDesiredRevision(
+ expected,
+ makeServer({
+ labels: { second: '2', first: '1' },
+ annotations: expected.annotations,
+ })
+ )
+ ).toBe(true)
+ })
+
+ it('ignores the controller-owned network-ready handshake annotation', () => {
+ const expected = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })
+ const current = makeServer({
+ annotations: {
+ 'clerum.io/pre-deploy': 'true',
+ 'clerum.io/network-ready': 'true',
+ },
+ })
+
+ expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)
+ expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)
+ })
+})
+
+describe('isMcpServerStatusOnlyUpdate', () => {
+ it('accepts only an exact identity and generation with unchanged desired state', () => {
+ const previous = makeServer({
+ status: { conditions: [{ type: 'Ready', status: 'False' }] },
+ })
+ const current = makeServer({
+ status: { conditions: [{ type: 'Ready', status: 'True' }] },
+ })
+
+ expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)
+ })
+
+ it('does not suppress events when UID or generation equality is only one-sided', () => {
+ const previous = makeServer()
+
+ expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ uid: undefined }))).toBe(false)
+ expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ generation: undefined }))).toBe(false)
+ })
+
+ it('does not suppress a desired-state change', () => {
+ const previous = makeServer()
+ const current = makeServer({
+ labels: { 'clerum.io/workload': 'replacement' },
+ })
+
+ expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(false)
+ })
+
+ it('treats the controller-owned network-ready handshake as a status-only update', () => {
+ const previous = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })
+ const current = makeServer({
+ annotations: {
+ 'clerum.io/pre-deploy': 'true',
+ 'clerum.io/network-ready': 'true',
+ },
+ })
+
+ expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)
+ })
+})
+
+describe('confirmAuthoritativeMcpServerAbsence', () => {
+ it('stops before the API read when inventory is not authoritative', async () => {
+ const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)
+ const readCurrent = vi.fn()
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => false,
+ resolveCurrent,
+ readCurrent,
+ })
+ ).resolves.toBe(false)
+ expect(resolveCurrent).not.toHaveBeenCalled()
+ expect(readCurrent).not.toHaveBeenCalled()
+ })
+
+ it('stops before the API read when the cache already contains the current server', async () => {
+ const readCurrent = vi.fn()
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => true,
+ resolveCurrent: () => makeServer(),
+ readCurrent,
+ })
+ ).resolves.toBe(false)
+ expect(readCurrent).not.toHaveBeenCalled()
+ })
+
+ it('rejects absence when the API read returns a current object', async () => {
+ const readCurrent = vi.fn().mockResolvedValue({
+ metadata: { name: 'web-search', namespace: 'mcp-server' },
+ })
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => true,
+ resolveCurrent: () => undefined,
+ readCurrent,
+ })
+ ).resolves.toBe(false)
+ expect(readCurrent).toHaveBeenCalledOnce()
+ })
+
+ it('propagates a non-404 API read failure', async () => {
+ const apiError = Object.assign(new Error('unavailable'), { code: 503 })
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => true,
+ resolveCurrent: () => undefined,
+ readCurrent: vi.fn().mockRejectedValue(apiError),
+ })
+ ).rejects.toBe(apiError)
+ })
+
+ it('rejects absence when inventory authority is lost after a 404', async () => {
+ const inventoryAuthoritative = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
+ const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative,
+ resolveCurrent,
+ readCurrent: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ })
+ ).resolves.toBe(false)
+ expect(resolveCurrent).toHaveBeenCalledOnce()
+ })
+
+ it('rejects absence when the server is recreated in cache after a 404', async () => {
+ const resolveCurrent = vi
+ .fn<() => McpServerCRD | undefined>()
+ .mockReturnValueOnce(undefined)
+ .mockReturnValueOnce(makeServer())
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => true,
+ resolveCurrent,
+ readCurrent: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ })
+ ).resolves.toBe(false)
+ expect(resolveCurrent).toHaveBeenCalledTimes(2)
+ })
+
+ it('confirms absence after a 404 when authority and cache absence remain stable', async () => {
+ const inventoryAuthoritative = vi.fn(() => true)
+ const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)
+
+ await expect(
+ confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative,
+ resolveCurrent,
+ readCurrent: vi.fn().mockRejectedValue({ response: { statusCode: 404 } }),
+ })
+ ).resolves.toBe(true)
+ expect(inventoryAuthoritative).toHaveBeenCalledTimes(2)
+ expect(resolveCurrent).toHaveBeenCalledTimes(2)
+ })
+})
diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.ts
new file mode 100644
index 000000000..7f8d8438d
--- /dev/null
+++ b/host-context-controller/src/mcpServerSafety.ts
@@ -0,0 +1,91 @@
+import type { McpServerCRD } from './types'
+import { getErrorCode } from './utils'
+
+const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'
+
+/**
+ * This acknowledgement is written by HCC after it has applied a server's
+ * NetworkPolicies so WRC can admit its workload. It is not user intent and it
+ * cannot change any NetworkPolicy decision. Treating that write as desired
+ * state makes HCC invalidate its own authoritative revocation pass under a
+ * large startup fleet, coupling /ready to additive runtime convergence again.
+ */
+function desiredAnnotations(annotations: Record<string, string> | undefined): Record<string, string> {
+ if (!annotations?.[NETWORK_READY_ANNOTATION]) return annotations ?? {}
+ const { [NETWORK_READY_ANNOTATION]: _networkReady, ...remaining } = annotations
+ return remaining
+}
+
+/**
+ * Kubernetes represents objects such as labels and annotations as maps, whose
+ * wire key order is not meaningful or stable across LIST and WATCH payloads.
+ * Preserve array order (it is semantic) while sorting object keys so a
+ * status-only event cannot masquerade as a desired-state revision.
+ */
+function stableJson(value: unknown): string {
+ if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`
+ if (value && typeof value === 'object') {
+ const record = value as Record<string, unknown>
+ return `{${Object.keys(record)
+ .sort()
+ .map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`)
+ .join(',')}}`
+ }
+ return JSON.stringify(value) ?? 'undefined'
+}
+
+export function sameMcpServerDesiredRevision(
+ expected: McpServerCRD,
+ current: McpServerCRD
+): boolean {
+ if (expected.name !== current.name || expected.namespace !== current.namespace) return false
+ if (expected.uid && current.uid && expected.uid !== current.uid) return false
+ if (
+ expected.generation !== undefined &&
+ current.generation !== undefined &&
+ expected.generation !== current.generation
+ ) {
+ return false
+ }
+
+ return (
+ stableJson(expected.spec) === stableJson(current.spec) &&
+ stableJson(desiredAnnotations(expected.annotations)) ===
+ stableJson(desiredAnnotations(current.annotations)) &&
+ stableJson(expected.labels ?? {}) === stableJson(current.labels ?? {})
+ )
+}
+
+export function isMcpServerStatusOnlyUpdate(
+ previous: McpServerCRD,
+ current: McpServerCRD
+): boolean {
+ return (
+ previous.uid === current.uid &&
+ previous.generation === current.generation &&
+ sameMcpServerDesiredRevision(previous, current)
+ )
+}
+
+export interface AuthoritativeMcpServerAbsenceCheck {
+ inventoryAuthoritative: () => boolean
+ resolveCurrent: () => McpServerCRD | undefined
+ readCurrent: () => Promise<unknown>
+}
+
+export async function confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative,
+ resolveCurrent,
+ readCurrent,
+}: AuthoritativeMcpServerAbsenceCheck): Promise<boolean> {
+ if (!inventoryAuthoritative() || resolveCurrent()) return false
+
+ try {
+ await readCurrent()
+ return false
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404) throw error
+ }
+
+ return inventoryAuthoritative() && !resolveCurrent()
+}
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index 75b8cc5f0..59465c3b3 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -79,6 +79,30 @@ export const networkPoliciesTotal = gauge({
labelNames: ['layer'] as const,
})
+export const initialConvergenceRetriesTotal = counter({
+ name: 'clerum_hcc_initial_convergence_retries_total',
+ help: 'Initial background convergence retries scheduled after a failed fleet pass.',
+ labelNames: ['lane'] as const,
+})
+
+export const initialConvergenceLastSuccessTimestampSeconds = gauge({
+ name: 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
+ help: 'Unix timestamp of the last successful initial background convergence pass.',
+ labelNames: ['lane'] as const,
+})
+
+export const networkPolicySafetyPassDurationSeconds = histogram({
+ name: 'clerum_hcc_networkpolicy_safety_pass_duration_seconds',
+ help: 'Seconds until an authoritative NetworkPolicy safety pass has revoked stale allows.',
+ labelNames: ['outcome'] as const,
+})
+
+export const networkPolicySafetyPassPoliciesTotal = counter({
+ name: 'clerum_hcc_networkpolicy_safety_pass_policies_total',
+ help: 'NetworkPolicies listed and revoked by authoritative HCC safety passes.',
+ labelNames: ['operation'] as const,
+})
+
export const contextReconciliationsTotal = counter({
name: 'clerum_hcc_context_reconciliations_total',
help: 'Total Context CRD reconciliations',
@@ -201,3 +225,13 @@ export const hostDeleteCleanupTotal = counter({
help: 'Host deletion cleanup by outcome (queued/confirmed/completed/retried/superseded).',
labelNames: ['outcome'] as const,
})
+
+// External-egress DNS retry saturation (#205 audit R3-M4 / R2-L1). A binding
+// whose DNS never resolves is retried forever at the capped (maximum) backoff;
+// without this signal the still-denied binding converges nowhere yet stays
+// invisible. Tracks the COUNT of servers pinned at the cap — never a per-server
+// label — so cardinality stays flat regardless of fleet size.
+export const externalEgressRetriesAtCap = gauge({
+ name: 'clerum_hcc_external_egress_retries_at_cap',
+ help: 'McpServers whose external-egress DNS retry is pinned at the capped (maximum) backoff, i.e. repeatedly failing to converge.',
+})
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 8f3d3f3e1..047ccd88e 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -4,9 +4,37 @@ import type { RecordWithTtl } from 'node:dns'
import * as dns from 'node:dns/promises'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
-import { NetworkPolicyReconciler, PUBLIC_EGRESS_EXCEPT_CIDRS } from './networkPolicyReconciler'
+import { confirmAuthoritativeMcpServerAbsence } from './mcpServerSafety'
+import {
+ networkPolicySafetyPassDurationSeconds,
+ networkPolicySafetyPassPoliciesTotal,
+} from './metrics'
+import {
+ NetworkPolicyReconciler,
+ PUBLIC_EGRESS_EXCEPT_CIDRS,
+ sameContextDesiredRevision,
+} from './networkPolicyReconciler'
import { ContextCRD, McpServerCRD } from './types'
+function deferred<T = void>(): {
+ promise: Promise<T>
+ resolve: (value: T | PromiseLike<T>) => void
+} {
+ let resolve!: (value: T | PromiseLike<T>) => void
+ const promise = new Promise<T>(res => {
+ resolve = res
+ })
+ return { promise, resolve }
+}
+
+vi.mock('./mcpServerSafety', async () => {
+ const actual = await vi.importActual<typeof import('./mcpServerSafety')>('./mcpServerSafety')
+ return {
+ ...actual,
+ confirmAuthoritativeMcpServerAbsence: vi.fn(actual.confirmAuthoritativeMcpServerAbsence),
+ }
+})
+
// resolve4({ ttl: true }) returns RecordWithTtl[]; `rec` builds that shape and
// `resolve4Mock` sidesteps the string[]/RecordWithTtl[] overload inference.
const rec = (...ips: string[]): RecordWithTtl[] => ips.map(address => ({ address, ttl: 300 }))
@@ -22,6 +50,7 @@ vi.mock('./config', () => ({
port: 8081,
runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],
k8sApiCidrs: [],
+ externalEgressDnsResolveTimeoutMs: 5_000,
// #299 sliding-window knobs read by reconcileExternalEgress.
externalEgressOverlapSec: 300,
externalEgressMaxEntries: 128,
@@ -39,10 +68,23 @@ vi.mock('node:dns/promises', () => ({
]),
}))
+vi.mock('./metrics', () => ({
+ networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },
+ networkPolicySafetyPassPoliciesTotal: { inc: vi.fn() },
+}))
+
function makeMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ readNamespacedNetworkPolicy: vi.fn().mockImplementation(async ({ name }) => {
+ if (
+ name === 'allow-desktop-egress-rpc-proxy' ||
+ name === 'allow-rpc-proxy-to-managed-mcp-servers'
+ ) {
+ throw Object.assign(new Error('not found'), { code: 404 })
+ }
+ return { metadata: { resourceVersion: '1' } }
+ }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
@@ -51,7 +93,17 @@ function makeMockNetworkingApi() {
function makeMockCustomApi() {
return {
- getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({ status: {} }),
+ getNamespacedCustomObject: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
+ getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({
+ metadata: {
+ generation: 7,
+ resourceVersion: 'status-rv-default',
+ },
+ status: {},
+ }),
patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
}
}
@@ -70,7 +122,48 @@ function makeReconciler(
})
const cache = serverCache ?? new Map()
const reconciler = new NetworkPolicyReconciler(kc, cache)
- ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = mockApi
+ const listedPolicies = new Map<string, k8s.V1NetworkPolicy>()
+ const networkingApi = {
+ ...mockApi,
+ listNamespacedNetworkPolicy: async (args: { namespace: string }) => {
+ const response = await mockApi.listNamespacedNetworkPolicy(args)
+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {
+ const name = policy.metadata?.name ?? `unnamed-${index}`
+ const materialized = {
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ name,
+ namespace: policy.metadata?.namespace ?? args.namespace,
+ uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,
+ resourceVersion: policy.metadata?.resourceVersion ?? '1',
+ },
+ }
+ listedPolicies.set(`${args.namespace}/${name}`, materialized)
+ return materialized
+ })
+ return { ...response, items }
+ },
+ readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {
+ const response = await mockApi.readNamespacedNetworkPolicy(args)
+ const listed = listedPolicies.get(`${args.namespace}/${args.name}`)
+ return {
+ ...listed,
+ ...response,
+ metadata: {
+ ...listed?.metadata,
+ ...response.metadata,
+ name: response.metadata?.name ?? listed?.metadata?.name ?? args.name,
+ namespace: response.metadata?.namespace ?? listed?.metadata?.namespace ?? args.namespace,
+ uid:
+ response.metadata?.uid ?? listed?.metadata?.uid ?? `${args.namespace}:${args.name}:uid`,
+ resourceVersion:
+ response.metadata?.resourceVersion ?? listed?.metadata?.resourceVersion ?? '1',
+ },
+ }
+ },
+ }
+ ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi
;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi
return reconciler
}
@@ -81,6 +174,7 @@ describe('NetworkPolicyReconciler', () => {
let reconciler: NetworkPolicyReconciler
beforeEach(() => {
+ vi.useRealTimers()
mockApi = makeMockNetworkingApi()
mockCustomApi = makeMockCustomApi()
reconciler = makeReconciler(mockApi, undefined, mockCustomApi)
@@ -88,6 +182,102 @@ describe('NetworkPolicyReconciler', () => {
resolve4Mock.mockResolvedValue(rec('1.2.3.4', '5.6.7.8'))
})
+ it('compares Context desired revisions canonically and fences identity changes', () => {
+ const expected: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ uid: 'context-uid',
+ generation: 7,
+ spec: {
+ contextId: 'default',
+ mcpServers: ['alpha'],
+ sharedFileSystems: [{ name: 'workspace', mountPath: '/workspace' }],
+ },
+ }
+ const reordered: ContextCRD = {
+ namespace: 'mcp-server',
+ name: 'default',
+ generation: 7,
+ uid: 'context-uid',
+ spec: {
+ sharedFileSystems: [{ mountPath: '/workspace', name: 'workspace' }],
+ mcpServers: ['alpha'],
+ contextId: 'default',
+ },
+ }
+
+ expect(sameContextDesiredRevision(expected, reordered)).toBe(true)
+ expect(sameContextDesiredRevision(expected, { ...reordered, uid: 'replacement-uid' })).toBe(
+ false
+ )
+ expect(sameContextDesiredRevision(expected, { ...reordered, generation: 8 })).toBe(false)
+ })
+
+ it('records the authoritative safety-pass inventory, completed revocations, and duration', async () => {
+ mockApi.listNamespacedNetworkPolicy
+ .mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-context',
+ labels: { 'clerum.io/context': 'missing-context' },
+ },
+ },
+ ],
+ })
+ .mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-host-egress',
+ labels: { 'clerum.io/context': 'missing-context' },
+ },
+ },
+ ],
+ })
+ .mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-rpc-egress',
+ labels: { 'clerum.io/context': 'missing-context' },
+ },
+ },
+ ],
+ })
+ .mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-external-egress',
+ labels: { 'clerum.io/mcpserver': 'missing-server' },
+ },
+ },
+ ],
+ })
+ const complete = vi.fn()
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ onAuthoritativeRevocationComplete: complete,
+ })
+
+ expect(complete).toHaveBeenCalledOnce()
+ expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(
+ { operation: 'listed' },
+ 4
+ )
+ expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(
+ { operation: 'revoked' },
+ 4
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
+ { outcome: 'completed' },
+ expect.any(Number)
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
+ })
+
describe('public egress blocklist parity', () => {
it('keeps the HCC and WRC IPv4 exclusion lists in sync', () => {
const wrcFactory = readFileSync(
@@ -150,6 +340,50 @@ describe('NetworkPolicyReconciler', () => {
// ─── L1: Infrastructure Policies ───────────────────────────────────
describe('L1 — ensureInfrastructurePolicies', () => {
+ it.each([
+ ['deny-all-mcp-server', 'default-deny'],
+ ['allow-dns-egress-mcp-server', 'infrastructure'],
+ ['allow-hcc-api-egress-mcp-server', 'infrastructure'],
+ ['allow-k8s-api-egress-mcp-server', 'infrastructure'],
+ ])(
+ 'refuses to adopt a foreign policy that collides with %s',
+ async (targetName, policyType) => {
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(
+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ if (body.metadata?.name === targetName) {
+ throw Object.assign(new Error('already exists'), { code: 409 })
+ }
+ return {}
+ }
+ )
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (
+ name === 'allow-desktop-egress-rpc-proxy' ||
+ name === 'allow-rpc-proxy-to-managed-mcp-servers'
+ ) {
+ throw Object.assign(new Error('not found'), { code: 404 })
+ }
+ return {
+ metadata: {
+ name,
+ namespace,
+ uid: 'foreign-baseline-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': policyType,
+ },
+ },
+ }
+ })
+
+ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name: targetName })
+ )
+ }
+ )
+
it('creates DNS egress policies in every runtime namespace', async () => {
await reconciler.ensureDefaultPolicies()
@@ -308,34 +542,204 @@ describe('NetworkPolicyReconciler', () => {
})
it('removes legacy static policies now replaced by generated scoped policies', async () => {
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `${name}-uid`,
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ }))
+
await reconciler.ensureDefaultPolicies()
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'rpc-proxy',
name: 'allow-desktop-egress-rpc-proxy',
+ body: {
+ preconditions: {
+ uid: 'allow-desktop-egress-rpc-proxy-uid',
+ resourceVersion: '7',
+ },
+ },
})
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'mcp-server',
name: 'allow-rpc-proxy-to-managed-mcp-servers',
+ body: {
+ preconditions: {
+ uid: 'allow-rpc-proxy-to-managed-mcp-servers-uid',
+ resourceVersion: '7',
+ },
+ },
})
})
- it('fails closed when legacy static policy cleanup fails', async () => {
- const cleanupError = Object.assign(new Error('rbac denied'), { code: 403 })
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
- if (name === 'allow-rpc-proxy-to-managed-mcp-servers') {
- throw cleanupError
+ it.each([
+ ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],
+ ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],
+ ])('refuses to delete a foreign legacy policy %s/%s', async (targetNamespace, targetName) => {
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (name !== targetName) {
+ throw Object.assign(new Error('not found'), { code: 404 })
+ }
+ return {
+ metadata: {
+ name,
+ namespace,
+ uid: 'foreign-legacy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ },
+ },
}
- return {}
})
- await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(cleanupError)
+ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ namespace: targetNamespace, name: targetName })
+ )
})
+
+ it.each([
+ ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],
+ ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],
+ ])(
+ 'propagates an identity conflict deleting legacy policy %s/%s',
+ async (targetNamespace, targetName) => {
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (name !== targetName) {
+ throw Object.assign(new Error('not found'), { code: 404 })
+ }
+ return {
+ metadata: {
+ name,
+ namespace,
+ uid: 'legacy-policy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ }
+ })
+ const conflict = Object.assign(new Error('identity changed'), { code: 409 })
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
+ if (name === targetName && namespace === targetNamespace) throw conflict
+ return {}
+ })
+
+ await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(conflict)
+ }
+ )
})
// ─── L1: Allow API ─────────────────────────────────────────────────
describe('L1 — ensureAllowContextMapperApi', () => {
+ it('updates the existing HCC-owned allow-api policy without requiring Context owner labels', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(
+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ if (body.metadata?.name === 'allow-host-context-controller-api') {
+ throw Object.assign(new Error('already exists'), { code: 409 })
+ }
+ return {}
+ }
+ )
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: 'existing-default-policy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'allow-api',
+ },
+ },
+ }))
+
+ await expect(reconciler.ensureDefaultPolicies()).resolves.toBeUndefined()
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'allow-host-context-controller-api',
+ namespace: 'mcp-server',
+ })
+ )
+ })
+
+ it('refuses to adopt a foreign policy that collides with the allow-api policy name', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(
+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ if (body.metadata?.name === 'allow-host-context-controller-api') {
+ throw Object.assign(new Error('already exists'), { code: 409 })
+ }
+ return {}
+ }
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ metadata: {
+ name: 'allow-host-context-controller-api',
+ namespace: 'mcp-server',
+ uid: 'foreign-default-policy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'allow-api',
+ },
+ },
+ })
+
+ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('revalidates allow-api ownership after a replace conflict', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(
+ async ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ if (body.metadata?.name === 'allow-host-context-controller-api') {
+ throw Object.assign(new Error('already exists'), { code: 409 })
+ }
+ return {}
+ }
+ )
+ mockApi.readNamespacedNetworkPolicy
+ .mockResolvedValueOnce({
+ metadata: {
+ name: 'allow-host-context-controller-api',
+ namespace: 'mcp-server',
+ uid: 'owned-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'allow-api',
+ },
+ },
+ })
+ .mockResolvedValueOnce({
+ metadata: {
+ name: 'allow-host-context-controller-api',
+ namespace: 'mcp-server',
+ uid: 'foreign-uid',
+ resourceVersion: '8',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'allow-api',
+ },
+ },
+ })
+ mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('conflict'), { code: 409 })
+ )
+
+ await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ })
+
it('creates allow-api ingress policy using _from (K8s client convention)', async () => {
await reconciler.ensureDefaultPolicies()
@@ -355,6 +759,82 @@ describe('NetworkPolicyReconciler', () => {
// ─── L2: Context-Based Policies ────────────────────────────────────
describe('L2 — reconcileContext', () => {
+ it('refuses to adopt a foreign policy that collides with a live Context policy name', async () => {
+ const server: McpServerCRD = {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]))
+ const context: ContextCRD = {
+ name: 'dev',
+ namespace: 'mcp-server',
+ spec: { contextId: 'dev', mcpServers: [server.name] },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ctx-dev-mongo',
+ namespace: 'mcp-server',
+ uid: 'foreign-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'dev',
+ 'clerum.io/mcpserver': 'mongo',
+ },
+ },
+ })
+
+ await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses to replace an HCC Context policy owned by a different Context/server pair', async () => {
+ const server: McpServerCRD = {
+ name: 'b-c',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'a',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]))
+ const context: ContextCRD = {
+ name: 'a',
+ namespace: 'mcp-server',
+ spec: { contextId: 'a', mcpServers: ['b-c'] },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ctx-a-b-c',
+ namespace: 'mcp-server',
+ uid: 'other-hcc-owner-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'a-b',
+ 'clerum.io/mcpserver': 'c',
+ },
+ },
+ })
+
+ await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('creates context-allow ingress policy per server in mcp-server namespace', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
@@ -374,7 +854,7 @@ describe('NetworkPolicyReconciler', () => {
spec: { contextId: 'dev', mcpServers: ['mongo'] },
}
- await rec.reconcileContext(context)
+ await rec.reconcileContext(context, { isCurrent: () => true })
// First call: ingress in mcp-server namespace
const ingressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
@@ -416,109 +896,355 @@ describe('NetworkPolicyReconciler', () => {
)
})
- it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {
- const cache = new Map<string, McpServerCRD>()
- cache.set('mongo', {
+ it('retains the ingress policy it just replaced when the authoritative server port changes', async () => {
+ const oldServer: McpServerCRD = {
name: 'mongo',
namespace: 'mcp-server',
spec: {
contextRef: 'dev',
image: 'mongo:latest',
- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ transport: { type: 'streamableHttp', port: 3000 },
},
- })
-
+ }
+ const cache = new Map([[oldServer.name, oldServer]])
const rec = makeReconciler(mockApi, cache)
const context: ContextCRD = {
name: 'dev',
namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: ['mongo'] },
+ spec: { contextId: 'dev', mcpServers: [oldServer.name] },
}
- await rec.reconcileContext(context)
+ await rec.reconcileContext(context, { isCurrent: () => true })
+ const livePolicies = new Map(
+ mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {
+ const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body
+ return [`${policy.metadata?.namespace}/${policy.metadata?.name}`, policy]
+ })
+ )
- // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)
- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
- namespace: string
- body: {
- metadata: { name: string; namespace: string; labels: Record<string, string> }
- spec: {
- podSelector: { matchLabels: Record<string, string> }
- policyTypes: string[]
- egress: Array<{
- to: Array<{
- namespaceSelector: { matchLabels: Record<string, string> }
- podSelector: { matchLabels: Record<string, string> }
- }>
- ports: Array<{ port: number }>
- }>
- }
+ vi.clearAllMocks()
+ cache.set(oldServer.name, {
+ ...oldServer,
+ generation: 2,
+ spec: {
+ ...oldServer.spec,
+ transport: { type: 'streamableHttp', port: 4000 },
+ },
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: [...livePolicies.values()].filter(
+ policy => policy.metadata?.namespace === namespace
+ ),
+ })
+ )
+ mockApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {
+ livePolicies.set(`${namespace}/${body.metadata?.name}`, body)
+ return {}
+ })
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, name, body }) => {
+ livePolicies.set(`${namespace}/${name}`, body)
+ return {}
}
- }
- expect(egressCall.namespace).toBe('mcp-host')
- expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')
- expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')
- expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])
- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/context': 'dev',
+ )
+ // The delete mock MUST remove from livePolicies, or the port-4000
+ // assertion below survives a spurious deletion and the test passes with
+ // the bug reinjected (R3-H7).
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ namespace, name }) => {
+ livePolicies.delete(`${namespace}/${name}`)
+ return {}
})
- // Egress targets mcp-server namespace + specific server pod
- const egressRule = egressCall.body.spec.egress[0]
- expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(
- 'mcp-server'
+
+ await rec.reconcileContext(context, { isCurrent: () => true })
+
+ // Filter the real calls by name+namespace: the production delete always
+ // carries `preconditions`, so `not.toHaveBeenCalledWith({name,namespace})`
+ // never matched any call and was vacuously true (R3-H7).
+ const deletedCtxDevMongo = mockApi.deleteNamespacedNetworkPolicy.mock.calls.some(
+ call =>
+ (call[0] as { name?: string; namespace?: string })?.name === 'ctx-dev-mongo' &&
+ (call[0] as { name?: string; namespace?: string })?.namespace === 'mcp-server'
)
- expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')
- expect(egressRule.ports[0].port).toBe(3000)
+ expect(deletedCtxDevMongo).toBe(false)
+ const ingress = livePolicies.get('mcp-server/ctx-dev-mongo')
+ expect(ingress?.spec?.ingress?.[0]?.ports).toEqual([expect.objectContaining({ port: 4000 })])
})
- it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {
+ it('stops remaining policy mutations after its combined authority lease is retired', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
namespace: 'mcp-server',
spec: {
- contextRef: 'ctx-name',
+ contextRef: 'dev',
image: 'mongo:latest',
- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ transport: { type: 'streamableHttp', port: 3000 },
},
})
-
+ const firstCreateStarted = deferred()
+ const releaseFirstCreate = deferred()
+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ firstCreateStarted.resolve(undefined)
+ await releaseFirstCreate.promise
+ return {}
+ })
const rec = makeReconciler(mockApi, cache)
+ let contextGeneration = 4
+ let serverGeneration = 8
+ const capturedContextGeneration = contextGeneration
+ const capturedServerGeneration = serverGeneration
const context: ContextCRD = {
- name: 'ctx-name',
+ name: 'dev',
namespace: 'mcp-server',
- spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },
+ spec: { contextId: 'dev', mcpServers: ['mongo'] },
}
- await rec.reconcileContext(context)
-
- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
- body: {
- metadata: { labels: Record<string, string> }
- spec: { podSelector: { matchLabels: Record<string, string> } }
- }
- }
- expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')
- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/context': 'ctx-name',
+ const reconcile = rec.reconcileContext(context, {
+ isCurrent: () =>
+ contextGeneration === capturedContextGeneration &&
+ serverGeneration === capturedServerGeneration,
})
+ await firstCreateStarted.promise
+
+ contextGeneration = 6
+ serverGeneration = 10
+ releaseFirstCreate.resolve(undefined)
+ await reconcile
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
- it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {
+ it('does not mutate a Context policy from a mixed McpServer cache revision', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
namespace: 'mcp-server',
+ uid: 'mongo-uid',
+ generation: 1,
spec: {
contextRef: 'dev',
- image: 'mongo:latest',
- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ image: 'mongo:old',
+ transport: { type: 'streamableHttp', port: 3000 },
},
})
-
+ const firstCreateStarted = deferred()
+ const releaseFirstCreate = deferred()
+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ firstCreateStarted.resolve(undefined)
+ await releaseFirstCreate.promise
+ return {}
+ })
+ const rec = makeReconciler(mockApi, cache)
+ const context: ContextCRD = {
+ name: 'dev',
+ namespace: 'mcp-server',
+ spec: { contextId: 'dev', mcpServers: ['mongo'] },
+ }
+
+ const reconcile = rec.reconcileContext(context, { isCurrent: () => true })
+ await firstCreateStarted.promise
+ cache.set('mongo', {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ uid: 'mongo-uid',
+ generation: 2,
+ spec: {
+ contextRef: 'dev',
+ image: 'mongo:new',
+ transport: { type: 'streamableHttp', port: 4000 },
+ },
+ })
+ releaseFirstCreate.resolve(undefined)
+ await reconcile
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('continues a Context policy pass across a status-only McpServer replacement', async () => {
+ const cache = new Map<string, McpServerCRD>()
+ const selected: McpServerCRD = {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ uid: 'mongo-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'dev',
+ image: 'mongo:stable',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ status: { conditions: [{ type: 'Ready', status: 'False' }] },
+ }
+ cache.set(selected.name, selected)
+ const firstCreateStarted = deferred()
+ const releaseFirstCreate = deferred()
+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ firstCreateStarted.resolve(undefined)
+ await releaseFirstCreate.promise
+ return {}
+ })
+ const rec = makeReconciler(mockApi, cache)
+ const context: ContextCRD = {
+ name: 'dev',
+ namespace: 'mcp-server',
+ spec: { contextId: 'dev', mcpServers: ['mongo'] },
+ }
+
+ const reconcile = rec.reconcileContext(context, { isCurrent: () => true })
+ await firstCreateStarted.promise
+ cache.set(selected.name, {
+ ...selected,
+ status: { conditions: [{ type: 'Ready', status: 'True' }] },
+ })
+ releaseFirstCreate.resolve(undefined)
+ await reconcile
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)
+ })
+
+ it('does not adopt an McpServer that appeared after Context selection', async () => {
+ const cache = new Map<string, McpServerCRD>()
+ const inventoryListed = deferred()
+ const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()
+ mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ inventoryListed.resolve(undefined)
+ return releaseInventory.promise
+ })
+ const rec = makeReconciler(mockApi, cache)
+ const context: ContextCRD = {
+ name: 'dev',
+ namespace: 'mcp-server',
+ spec: { contextId: 'dev', mcpServers: ['late-server'] },
+ }
+
+ const reconcile = rec.reconcileContext(context, { isCurrent: () => true })
+ await inventoryListed.promise
+ cache.set('late-server', {
+ name: 'late-server',
+ namespace: 'mcp-server',
+ uid: 'late-server-uid',
+ generation: 1,
+ spec: {
+ contextRef: 'dev',
+ image: 'late-server:new',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ })
+ releaseInventory.resolve({ items: [] })
+ await reconcile
+
+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {
+ const cache = new Map<string, McpServerCRD>()
+ cache.set('mongo', {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ },
+ })
+
+ const rec = makeReconciler(mockApi, cache)
+ const context: ContextCRD = {
+ name: 'dev',
+ namespace: 'mcp-server',
+ spec: { contextId: 'dev', mcpServers: ['mongo'] },
+ }
+
+ await rec.reconcileContext(context, { isCurrent: () => true })
+
+ // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)
+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
+ namespace: string
+ body: {
+ metadata: { name: string; namespace: string; labels: Record<string, string> }
+ spec: {
+ podSelector: { matchLabels: Record<string, string> }
+ policyTypes: string[]
+ egress: Array<{
+ to: Array<{
+ namespaceSelector: { matchLabels: Record<string, string> }
+ podSelector: { matchLabels: Record<string, string> }
+ }>
+ ports: Array<{ port: number }>
+ }>
+ }
+ }
+ }
+ expect(egressCall.namespace).toBe('mcp-host')
+ expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')
+ expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')
+ expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])
+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/context': 'dev',
+ })
+ // Egress targets mcp-server namespace + specific server pod
+ const egressRule = egressCall.body.spec.egress[0]
+ expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(
+ 'mcp-server'
+ )
+ expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')
+ expect(egressRule.ports[0].port).toBe(3000)
+ })
+
+ it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {
+ const cache = new Map<string, McpServerCRD>()
+ cache.set('mongo', {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'ctx-name',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ },
+ })
+
+ const rec = makeReconciler(mockApi, cache)
+ const context: ContextCRD = {
+ name: 'ctx-name',
+ namespace: 'mcp-server',
+ spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },
+ }
+
+ await rec.reconcileContext(context, { isCurrent: () => true })
+
+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
+ body: {
+ metadata: { labels: Record<string, string> }
+ spec: { podSelector: { matchLabels: Record<string, string> } }
+ }
+ }
+ expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')
+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/context': 'ctx-name',
+ })
+ })
+
+ it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {
+ const cache = new Map<string, McpServerCRD>()
+ cache.set('mongo', {
+ name: 'mongo',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ },
+ })
+
const rec = makeReconciler(mockApi, cache)
const context: ContextCRD = {
name: 'dev',
@@ -526,7 +1252,7 @@ describe('NetworkPolicyReconciler', () => {
spec: { contextId: 'dev', mcpServers: ['mongo'] },
}
- await rec.reconcileContext(context)
+ await rec.reconcileContext(context, { isCurrent: () => true })
const rpcProxyEgressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[2][0] as {
namespace: string
@@ -566,6 +1292,118 @@ describe('NetworkPolicyReconciler', () => {
// ─── L3: External Egress ───────────────────────────────────────────
describe('L3 — reconcileExternalEgress', () => {
+ it('removes a deterministic safety policy created after its desired-state fence is lost', async () => {
+ const server: McpServerCRD = {
+ name: 'race-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'race:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ const createStarted = deferred()
+ const releaseCreate = deferred()
+ let current = true
+ mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ createStarted.resolve(undefined)
+ await releaseCreate.promise
+ return {
+ metadata: {
+ name: 'ext-egress-race-server-104-18-0-0-16-443',
+ namespace: server.namespace,
+ uid: 'created-race-policy',
+ resourceVersion: '7',
+ },
+ }
+ })
+
+ const safety = (reconciler as any).reconcileExternalEgressSafety(
+ server,
+ [{ metadata: { name: 'ext-egress-race-server-stale' } }],
+ () => current
+ ) as Promise<boolean>
+
+ await createStarted.promise
+ current = false
+ releaseCreate.resolve(undefined)
+
+ expect(await safety).toBe(false)
+ const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: created.metadata.name,
+ namespace: server.namespace,
+ body: { preconditions: { uid: 'created-race-policy', resourceVersion: '7' } },
+ })
+ )
+ })
+
+ it('refuses to adopt a foreign policy that collides with an external-egress name', async () => {
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ext-egress-openai-mcp-104-18-0-0-16-443',
+ namespace: 'mcp-server',
+ uid: 'foreign-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ })
+
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('refuses to replace an HCC external-egress policy owned by a different server', async () => {
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: 'ext-egress-openai-mcp-104-18-0-0-16-443',
+ namespace: 'mcp-server',
+ uid: 'other-hcc-owner-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': 'different-server',
+ },
+ },
+ })
+
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('creates egress policy for CIDR binding', async () => {
const server: McpServerCRD = {
name: 'openai-mcp',
@@ -578,7 +1416,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
@@ -601,7 +1439,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
@@ -639,35 +1477,69 @@ describe('NetworkPolicyReconciler', () => {
)
})
- it('fails closed when ExternalEgressReady status cannot be written', async () => {
- const statusError = new Error('status update denied')
- mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
+ it('does not mutate policy or status after its McpServer authority lease is retired', async () => {
+ const dnsStarted = deferred()
+ const releaseDns = deferred<string[]>()
+ vi.mocked(dns.resolve4).mockImplementationOnce(async () => {
+ dnsStarted.resolve(undefined)
+ return releaseDns.promise
+ })
+ let inventoryGeneration = 7
+ const capturedGeneration = inventoryGeneration
const server: McpServerCRD = {
- name: 'openai-mcp',
+ name: 'authority-race-mcp',
namespace: 'mcp-server',
- generation: 7,
+ uid: 'authority-race-uid',
+ generation: 3,
spec: {
contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ image: 'authority-race:test',
+ transport: { type: 'streamableHttp', port: 3000 },
egressBindings: [{ dns: 'api.openai.com', port: 443 }],
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
- 'status update denied'
- )
+ const reconcile = reconciler.reconcileExternalEgress(server, {
+ isCurrent: () => inventoryGeneration === capturedGeneration,
+ })
+ await dnsStarted.promise
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
+ // Retire and recover to a different generation. The recovered cache is
+ // authoritative again, but never for this stale pass.
+ inventoryGeneration = 9
+ releaseDns.resolve(['1.2.3.4'])
+ await reconcile
+
+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
- it('fails closed when ExternalEgressReady status cannot be read', async () => {
- const statusError = new Error('status read denied')
- mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
+ it('identity-fences the status patch and preserves unrelated conditions', async () => {
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: {
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-17',
+ },
+ status: {
+ conditions: [
+ {
+ type: 'RuntimeReady',
+ status: 'True',
+ reason: 'DeploymentAvailable',
+ message: 'Runtime deployment is available',
+ lastTransitionTime: '2026-07-29T00:00:00.000Z',
+ observedGeneration: 7,
+ },
+ ],
+ },
+ })
const server: McpServerCRD = {
name: 'openai-mcp',
namespace: 'mcp-server',
+ uid: 'openai-mcp-uid',
generation: 7,
spec: {
contextRef: 'dev',
@@ -677,14 +1549,201 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('status read denied')
-
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- })
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
- it('creates explicit public-web egress with private and special-use ranges excluded', async () => {
- const server: McpServerCRD = {
+ const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
+ expect(patch.slice(0, 3)).toEqual([
+ { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },
+ { op: 'test', path: '/metadata/generation', value: 7 },
+ { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },
+ ])
+ expect(patch).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ path: '/status/conditions',
+ value: expect.arrayContaining([
+ expect.objectContaining({
+ type: 'RuntimeReady',
+ reason: 'DeploymentAvailable',
+ }),
+ expect.objectContaining({
+ type: 'ExternalEgressReady',
+ status: 'True',
+ observedGeneration: 7,
+ }),
+ ]),
+ }),
+ ])
+ )
+ })
+
+ it('fences every identity field read from status when synthetic input omits them', async () => {
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: {
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-17',
+ },
+ status: {},
+ })
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+
+ const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
+ expect(patch.slice(0, 3)).toEqual([
+ { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },
+ { op: 'test', path: '/metadata/generation', value: 7 },
+ { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },
+ ])
+ })
+
+ it.each([
+ {
+ name: 'UID',
+ currentMetadata: {
+ uid: 'replacement-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-18',
+ },
+ },
+ {
+ name: 'generation',
+ currentMetadata: {
+ uid: 'openai-mcp-uid',
+ generation: 8,
+ resourceVersion: 'status-rv-18',
+ },
+ },
+ ])(
+ 'rejects a stale $name before patching external egress status',
+ async ({ currentMetadata }) => {
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: currentMetadata,
+ status: {},
+ })
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/stale McpServer/i)
+
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ }
+ )
+
+ it('fails closed when a same-generation status writer wins the resourceVersion race', async () => {
+ const conflict = Object.assign(new Error('JSON patch test failed'), { code: 409 })
+ mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: {
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ resourceVersion: 'status-rv-17',
+ },
+ status: {
+ conditions: [
+ {
+ type: 'RuntimeReady',
+ status: 'True',
+ reason: 'DeploymentAvailable',
+ lastTransitionTime: '2026-07-29T00:00:00.000Z',
+ },
+ ],
+ },
+ })
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict)
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ uid: 'openai-mcp-uid',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toBe(conflict)
+
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.arrayContaining([
+ {
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: 'status-rv-17',
+ },
+ ]),
+ })
+ )
+ })
+
+ it('fails closed when ExternalEgressReady status cannot be written', async () => {
+ const statusError = new Error('status update denied')
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ 'status update denied'
+ )
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
+ })
+
+ it('fails closed when ExternalEgressReady status cannot be read', async () => {
+ const statusError = new Error('status read denied')
+ mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
+ const server: McpServerCRD = {
+ name: 'openai-mcp',
+ namespace: 'mcp-server',
+ generation: 7,
+ spec: {
+ contextRef: 'dev',
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('status read denied')
+
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ })
+
+ it('creates explicit public-web egress with private and special-use ranges excluded', async () => {
+ const server: McpServerCRD = {
name: 'search-mcp',
namespace: 'mcp-server',
spec: {
@@ -695,7 +1754,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(dns.resolve4).not.toHaveBeenCalled()
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
@@ -737,7 +1796,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/public-web external egress bindings/
)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@ -755,7 +1814,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/public-web external egress bindings/
)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@ -793,7 +1852,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
@@ -839,7 +1898,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
@@ -864,7 +1923,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
const resolvedPatch = patch.find(
@@ -889,7 +1948,7 @@ describe('NetworkPolicyReconciler', () => {
}
// First reconcile bootstraps and writes the external-egress policy.
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -916,7 +1975,7 @@ describe('NetworkPolicyReconciler', () => {
// Second resync: same DNS answers → same IP set → must NOT rewrite the
// policy (a timestamp-only refresh is a no-op). This is the churn the audit
// caught: before the fix, RESOLVED_AT + renewed expiry forced a write here.
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -932,7 +1991,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ dns: 'api.example.com', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const written = (
mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
).body
@@ -964,7 +2023,7 @@ describe('NetworkPolicyReconciler', () => {
// Same DNS → set unchanged (changed=false) but window aging → renewalDue=true
// → the policy MUST be re-persisted (deleting `|| egressRenewalDue` breaks this,
// reintroducing the M1 overlap-grace loss).
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const wrote =
mockApi.createNamespacedNetworkPolicy.mock.calls.length +
mockApi.replaceNamespacedNetworkPolicy.mock.calls.length
@@ -982,7 +2041,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ dns: 'api.example.com', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const written = (
mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
).body
@@ -1035,7 +2094,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
// The reconcile must rewrite the policy WITHOUT the private IP.
const call =
mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??
@@ -1060,7 +2119,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ dns: 'api.example.com', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
// The min observed TTL (ms) must reflect the 15s answer — deleting the
// Math.min update leaves it Infinity and the resync degrades to the fixed
// interval (the #299 under-sampling failure mode).
@@ -1080,7 +2139,7 @@ describe('NetworkPolicyReconciler', () => {
}
// Round 1: DNS serves A only.
resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.3', ttl: 15 }])
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const written = (
mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
).body
@@ -1093,7 +2152,7 @@ describe('NetworkPolicyReconciler', () => {
// Round 2: DNS rotates to B. The written policy must be the UNION {A, B} —
// this only holds if previousAnnotations rehydration is actually wired in.
resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.4', ttl: 15 }])
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const call =
mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??
mockApi.createNamespacedNetworkPolicy.mock.calls[0]?.[0]
@@ -1115,7 +2174,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -1126,7 +2185,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -1146,7 +2205,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -1163,7 +2222,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -1179,7 +2238,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -1198,7 +2257,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
// The reconciler must rewrite the policy back to the declared CIDR.
const wrote =
mockApi.createNamespacedNetworkPolicy.mock.calls.length +
@@ -1219,7 +2278,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/External egress reconciliation failed/
)
}
@@ -1240,7 +2299,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/resolved disallowed address/
)
@@ -1275,7 +2334,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/private, internal, metadata/
)
@@ -1296,7 +2355,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/resolved disallowed address/
)
@@ -1322,13 +2381,15 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-openai-mcp-old-example-com-443',
- namespace: 'mcp-server',
- })
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-openai-mcp-old-example-com-443',
+ namespace: 'mcp-server',
+ })
+ )
})
it('fails closed when an orphaned external egress policy cannot be deleted', async () => {
@@ -1351,7 +2412,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('delete denied')
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('delete denied')
})
it('deletes stale policies and fails when DNS resolution for that binding fails', async () => {
@@ -1374,17 +2435,21 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/dns timeout/)
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/dns timeout/)
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-openai-mcp-api.openai.com-443',
- namespace: 'mcp-server',
- })
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-openai-mcp-old-example-com-443',
- namespace: 'mcp-server',
- })
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-openai-mcp-api.openai.com-443',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-openai-mcp-old-example-com-443',
+ namespace: 'mcp-server',
+ })
+ )
expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
@@ -1415,13 +2480,36 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
/must declare dns or cidr/
)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
+ it('bounds DNS resolution so a stuck resolver cannot block reconciliation indefinitely', async () => {
+ vi.useFakeTimers()
+ vi.mocked(dns.resolve4).mockImplementationOnce(() => new Promise(() => undefined))
+ const server: McpServerCRD = {
+ name: 'stuck-dns-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'stuck-dns:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ },
+ }
+
+ const reconcile = reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ const rejection = expect(reconcile).rejects.toThrow(/DNS resolution timed out after 5000ms/)
+ await vi.advanceTimersByTimeAsync(5_000)
+ await rejection
+
+ expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
it('skips if no egressBindings', async () => {
const server: McpServerCRD = {
name: 'basic-mcp',
@@ -1433,7 +2521,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server)
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -1454,6 +2542,14 @@ describe('NetworkPolicyReconciler', () => {
await expect((reconciler as any).listAllContextPolicies()).rejects.toThrow('api list denied')
})
+ it('propagates all-context mcp-host egress policy list failures', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))
+
+ await expect((reconciler as any).listAllContextEgressPolicies()).rejects.toThrow(
+ 'api list denied'
+ )
+ })
+
it('propagates all external-egress policy list failures', async () => {
mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))
@@ -1463,81 +2559,2496 @@ describe('NetworkPolicyReconciler', () => {
})
})
- describe('startup fullReconcile', () => {
- it('reconciles external egress policies for existing servers on startup', async () => {
+ describe('delete-context TOCTOU re-check', () => {
+ it('stops mid-inventory when the context is recreated between two deletes', async () => {
+ // deleteAllowed is a live GET, so its answer can change while the loop
+ // runs. Hoisting the check out of the loop would satisfy every existing
+ // test and still delete the recreated context's own policies.
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValueOnce({
+ items: [
+ { metadata: { name: 'ctx-allow-first' } },
+ { metadata: { name: 'ctx-allow-second' } },
+ ],
+ })
+ const deleteAllowed = vi
+ .fn<() => Promise<boolean>>()
+ .mockResolvedValueOnce(true)
+ .mockResolvedValue(false)
+
+ await reconciler.reconcileDeleteContext('dev', deleteAllowed)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'ctx-allow-first' })
+ )
+ expect(deleteAllowed).toHaveBeenCalledTimes(2)
+ })
+
+ it('re-checks before every mcp-host egress delete', async () => {
+ mockApi.listNamespacedNetworkPolicy
+ .mockResolvedValueOnce({ items: [] }) // context-allow
+ .mockResolvedValueOnce({
+ items: [
+ { metadata: { name: 'host-egress-first' } },
+ { metadata: { name: 'host-egress-second' } },
+ ],
+ })
+ const deleteAllowed = vi
+ .fn<() => Promise<boolean>>()
+ .mockResolvedValueOnce(true)
+ .mockResolvedValue(false)
+
+ await reconciler.reconcileDeleteContext('dev', deleteAllowed)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'host-egress-first' })
+ )
+ })
+
+ it('re-checks before every rpc-proxy egress delete', async () => {
+ mockApi.listNamespacedNetworkPolicy
+ .mockResolvedValueOnce({ items: [] }) // context-allow
+ .mockResolvedValueOnce({ items: [] }) // mcp-host egress
+ .mockResolvedValueOnce({
+ items: [
+ { metadata: { name: 'rpc-egress-first' } },
+ { metadata: { name: 'rpc-egress-second' } },
+ ],
+ })
+ const deleteAllowed = vi
+ .fn<() => Promise<boolean>>()
+ .mockResolvedValueOnce(true)
+ .mockResolvedValue(false)
+
+ await reconciler.reconcileDeleteContext('dev', deleteAllowed)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'rpc-egress-first' })
+ )
+ })
+ })
+
+ describe('L3 — DNS-derived retention by binding identity (B3)', () => {
+ // Live DNS policy built through the reconciler's OWN builder so its identity
+ // is exact-by-construction; each test drifts exactly one dimension.
+ function makeDnsFixture() {
const server: McpServerCRD = {
- name: 'airtable-mcp',
+ name: 'openai-mcp',
namespace: 'mcp-server',
spec: {
contextRef: 'dev',
- image: 'airtable:latest',
- transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },
- egressBindings: [{ dns: 'api.airtable.com', port: 443 }],
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.openai.com', port: 443 }],
},
}
+ const binding = server.spec.egressBindings![0]
+ const name = (reconciler as any).externalEgressPolicyName(server.name, binding) as string
+ const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [
+ '1.2.3.4/32',
+ ]) as k8s.V1NetworkPolicy
+ live.metadata!.uid = 'live-dns-uid'
+ live.metadata!.resourceVersion = '11'
+ return { server, binding, name, live }
+ }
+
+ async function runSafety(server: McpServerCRD, live: k8s.V1NetworkPolicy): Promise<boolean> {
+ return (reconciler as any).reconcileExternalEgressSafety(
+ server,
+ [live],
+ () => true
+ ) as Promise<boolean>
+ }
+
+ function expectRevoked(name: string) {
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name,
+ namespace: 'mcp-server',
+ body: { preconditions: { uid: 'live-dns-uid', resourceVersion: '11' } },
+ })
+ )
+ }
- await reconciler.fullReconcile([], [server])
+ it('(a) retains an unchanged DNS-derived allow — no revocation', async () => {
+ const { server, live } = makeDnsFixture()
+ await expect(runSafety(server, live)).resolves.toBe(true)
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
- const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
- (call: unknown[]) => {
- const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }
- return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'
- }
- )
- expect(externalPolicies).toHaveLength(1)
- const body = (
- externalPolicies[0][0] as {
- body: {
- metadata: { name: string }
- spec: { egress: Array<{ ports: Array<{ port: number }> }> }
- }
- }
- ).body
- expect(body.metadata.name).toContain('ext-egress-airtable-mcp')
- expect(body.spec.egress[0].ports[0].port).toBe(443)
+ it('(b) revokes on spec-level port drift under the same policy name', async () => {
+ const { server, name, live } = makeDnsFixture()
+ live.spec!.egress![0].ports = [{ port: 8443, protocol: 'TCP' }]
+ await expect(runSafety(server, live)).resolves.toBe(true)
+ expectRevoked(name)
})
- it('deletes orphaned external egress policies for servers no longer present', async () => {
+ it('(c) revokes on protocol drift TCP→UDP', async () => {
+ const { server, name, live } = makeDnsFixture()
+ live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]
+ await expect(runSafety(server, live)).resolves.toBe(true)
+ expectRevoked(name)
+ })
+
+ it('(d) revokes a foreign-owned policy even when the spec is identical', async () => {
+ const { server, name, live } = makeDnsFixture()
+ live.metadata!.labels!['clerum.io/managed-by'] = 'foreign-controller'
+ await expect(runSafety(server, live)).resolves.toBe(true)
+ expectRevoked(name)
+ })
+
+ it('(f) revokes a degenerate zero-cidr "allow" (deny in disguise)', async () => {
+ const { server, name, live } = makeDnsFixture()
+ live.spec!.egress = []
+ await expect(runSafety(server, live)).resolves.toBe(true)
+ expectRevoked(name)
+ })
+
+ it('(e) queues recreation AT the revocation even when the pass then aborts and throws', async () => {
+ const { server, binding, name } = makeDnsFixture()
+ // Identity-drifted (protocol UDP) → this DNS allow is revoked (deleted).
+ const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [
+ '1.2.3.4/32',
+ ]) as k8s.V1NetworkPolicy
+ live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]
+ live.metadata!.uid = 'live-dns-uid'
+ live.metadata!.resourceVersion = '11'
+
mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('context-allow')) return { items: [] }
- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-old-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'old-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: !labelSelector && namespace === 'mcp-server' ? [live] : [],
+ })
)
- await reconciler.fullReconcile([], [], { serverInventoryComplete: true })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-old-server-api-example-com-443',
- namespace: 'mcp-server',
+ let resolved: McpServerCRD | undefined = server
+ const spy = vi.fn()
+ // Flip authority false AT the deletion, so the pass aborts right after the
+ // revocation hook fires — reproducing the abort/throw window B3 must survive.
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ resolved = undefined
+ return {}
})
+
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
+
+ await expect(
+ rec.fullReconcile([], [server], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ resolveCurrentServer: () => resolved,
+ onExternalEgressRevoked: spy,
+ })
+ ).rejects.toThrow(/inventory changed during authoritative revocation/)
+
+ expect(spy).toHaveBeenCalledTimes(1)
+ expect(spy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: server.name, namespace: server.namespace })
+ )
})
+ })
- it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ describe('startup fullReconcile', () => {
+ it('discovers and revokes orphaned policies with one missing ownership marker in every safety lane', async () => {
+ const partiallyLabelledByNamespace: Record<string, k8s.V1NetworkPolicy[]> = {
+ 'mcp-server': [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/context': 'orphan-context',
+ 'clerum.io/mcpserver': 'old-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ },
+ {
+ metadata: {
+ name: 'ext-egress-orphan-server-old-example-com-443',
+ labels: {
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': 'orphan-server',
+ 'clerum.io/egress-class': 'exact-host',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ },
+ ],
+ 'mcp-host': [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server-egress',
+ labels: {
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'orphan-context',
+ 'clerum.io/mcpserver': 'old-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ },
+ ],
+ 'rpc-proxy': [
+ {
+ metadata: {
+ name: 'rpc-egress-orphan-context-old-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/context': 'orphan-context',
+ 'clerum.io/mcpserver': 'old-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ },
+ ],
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: labelSelector ? [] : (partiallyLabelledByNamespace[namespace ?? ''] ?? []),
+ })
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-orphan-context-old-server',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-orphan-context-old-server-egress',
+ namespace: 'mcp-host',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'rpc-egress-orphan-context-old-server',
+ namespace: 'rpc-proxy',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-orphan-server-old-example-com-443',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(4)
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it('repairs one-marker live widened policies in every lane before certification', async () => {
+ const server: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'live-context',
+ image: 'live:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443 }],
+ },
+ }
+ const context: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [server.name] },
+ }
+ const cache = new Map([[server.name, server]])
+ const rec = makeReconciler(mockApi, cache, mockCustomApi)
+
+ await rec.reconcileContext(context, { isCurrent: () => true })
+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })
+ const generated = mockApi.createNamespacedNetworkPolicy.mock.calls.map(
+ call => (call[0] as { body: k8s.V1NetworkPolicy }).body
+ )
+ const widened = generated.map((policy, index) => ({
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ labels: Object.fromEntries(
+ Object.entries(policy.metadata?.labels ?? {}).filter(
+ ([label]) =>
+ label !== (index % 2 === 0 ? 'clerum.io/managed-by' : 'clerum.io/policy-type')
+ )
+ ),
+ },
+ spec: {
+ ...policy.spec,
+ policyTypes: [...(policy.spec?.policyTypes ?? []), 'Ingress'],
+ },
+ }))
+
+ vi.clearAllMocks()
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: labelSelector
+ ? []
+ : widened.filter(policy => policy.metadata?.namespace === namespace),
+ })
+ )
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ const ordering: string[] = []
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ ordering.push(`replace:${name}`)
+ return {}
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))
+
+ await rec.fullReconcile([context], [server], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ resolveCurrentContext: () => context,
+ resolveCurrentServer: () => server,
+ onAuthoritativeRevocationComplete,
+ })
+
+ const certificationIndex = ordering.indexOf('certify')
+ expect(certificationIndex).toBeGreaterThanOrEqual(4)
+ for (const expectedName of [
+ 'ctx-live-context-live-server',
+ 'ctx-live-context-live-server-egress',
+ 'rpc-egress-live-context-live-server',
+ 'ext-egress-live-server-8-8-8-8-32-443',
+ ]) {
+ expect(ordering.indexOf(`replace:${expectedName}`)).toBeGreaterThanOrEqual(0)
+ expect(ordering.indexOf(`replace:${expectedName}`)).toBeLessThan(certificationIndex)
+ }
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it.each([
+ [
+ 'context ingress',
+ 'mcp-server',
+ {
+ metadata: { name: 'ctx-reserved-context-server', labels: {} },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ },
+ ],
+ [
+ 'host egress',
+ 'mcp-host',
+ {
+ metadata: { name: 'ctx-reserved-context-server-egress', labels: {} },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ },
+ ],
+ [
+ 'rpc egress',
+ 'rpc-proxy',
+ {
+ metadata: { name: 'rpc-egress-reserved-context-server', labels: {} },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ },
+ ],
+ [
+ 'external egress',
+ 'mcp-server',
+ {
+ metadata: { name: 'ext-egress-reserved-server-example-443', labels: {} },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ },
+ ],
+ ])(
+ 'fails closed without takeover for a both-markers-missing reserved %s name',
+ async (_lane, namespace, candidate) => {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({
+ namespace: listedNamespace,
+ labelSelector,
+ }: {
+ namespace?: string
+ labelSelector?: string
+ }) => ({
+ items: !labelSelector && listedNamespace === namespace ? [candidate] : [],
+ })
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ }
+ )
+
+ it('fails closed for an explicit foreign ownership conflict on an HCC-shaped policy', async () => {
+ const candidate: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ext-egress-live-server-api-example-com-443',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': 'live-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],
+ })
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('treats a non-reserved external policy missing its manager marker as ambiguous', async () => {
+ const candidate: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'custom-egress-policy',
+ labels: {
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': 'live-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],
+ })
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('lets the broad snapshot shadow a selected-owned object that became unrelated at the same UID', async () => {
+ const selected: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'legacy-custom-policy',
+ uid: 'stable-policy-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'orphan-context',
+ 'clerum.io/mcpserver': 'old-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ const broadRelabelled: k8s.V1NetworkPolicy = {
+ ...selected,
+ metadata: {
+ ...selected.metadata,
+ resourceVersion: '8',
+ labels: { 'tenant.example/managed-by': 'tenant-controller' },
+ },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace !== 'mcp-server') return { items: [] }
+ if (labelSelector?.includes('policy-type=context-allow')) return { items: [selected] }
+ if (!labelSelector) return { items: [broadRelabelled] }
+ return { items: [] }
+ }
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it('leaves unrelated namespace policies untouched and still certifies', async () => {
+ const unrelated: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'tenant-owned-policy',
+ labels: {
+ 'app.kubernetes.io/managed-by': 'tenant-operator',
+ 'tenant.example/purpose': 'custom-egress',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Egress'] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: !labelSelector && namespace === 'mcp-server' ? [unrelated] : [],
+ })
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('fails closed when the broad safety inventory LIST fails', async () => {
+ mockApi.listNamespacedNetworkPolicy
+ .mockResolvedValueOnce({ items: [] })
+ .mockResolvedValueOnce({ items: [] })
+ .mockResolvedValueOnce({ items: [] })
+ .mockResolvedValueOnce({ items: [] })
+ .mockRejectedValueOnce(new Error('broad inventory denied'))
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow('broad inventory denied')
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('withholds certification and mutation when Context authority is lost during the broad LIST', async () => {
+ let contextAuthoritative = true
+ const candidate: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-orphan-context-old-server',
+ labels: {
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'orphan-context',
+ 'clerum.io/mcpserver': 'old-server',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (!labelSelector && namespace === 'mcp-server') {
+ contextAuthoritative = false
+ return { items: [candidate] }
+ }
+ return { items: [] }
+ }
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => contextAuthoritative,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('does not reconcile a queued Context when its current contextId changed', async () => {
+ const queuedContext: ContextCRD = {
+ name: 'mutable-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'captured-context', mcpServers: [] },
+ }
+ const currentContext: ContextCRD = {
+ ...queuedContext,
+ spec: { contextId: 'replacement-context', mcpServers: [] },
+ }
+ const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)
+ const effectKeys: string[] = []
+
+ await reconciler.fullReconcile([queuedContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => false,
+ resolveCurrentContext: () => currentContext,
+ runContextEffect: async (contextId, work) => {
+ effectKeys.push(contextId)
+ await work()
+ },
+ })
+
+ expect(effectKeys).toEqual(['captured-context', 'captured-context'])
+ expect(reconcileContext).not.toHaveBeenCalled()
+ })
+
+ it('retires a full-pass Context lease when the selected object is replaced', async () => {
+ const selected: ContextCRD = {
+ name: 'full-pass-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'full-pass-context', description: 'old', mcpServers: [] },
+ }
+ const replacement: ContextCRD = {
+ ...selected,
+ spec: { ...selected.spec, description: 'new' },
+ }
+ let current = selected
+ let selectedLease: (() => boolean) | undefined
+ const mutationStarted = deferred()
+ const releaseMutation = deferred()
+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async (_context, options) => {
+ selectedLease = options?.isCurrent
+ mutationStarted.resolve(undefined)
+ await releaseMutation.promise
+ return true
+ })
+
+ const fullPass = reconciler.fullReconcile([selected], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ resolveCurrentContext: () => current,
+ })
+ await mutationStarted.promise
+ expect(selectedLease?.()).toBe(true)
+
+ current = replacement
+ expect(selectedLease?.()).toBe(false)
+ releaseMutation.resolve(undefined)
+ await fullPass
+ })
+
+ it('deletes orphaned Context policies in all three namespaces through the canonical contextId effect key', async () => {
+ const effectKeys: string[] = []
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server-egress',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'rpc-egress-orphan-context-old-server',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ runContextEffect: async (contextId, work) => {
+ effectKeys.push(contextId)
+ await work()
+ },
+ })
+
+ expect(effectKeys).toEqual(['orphan-context', 'orphan-context', 'orphan-context'])
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-orphan-context-old-server',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-orphan-context-old-server-egress',
+ namespace: 'mcp-host',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'rpc-egress-orphan-context-old-server',
+ namespace: 'rpc-proxy',
+ })
+ )
+ })
+
+ it('routes desired and orphan Context policy lanes through canonical contextId effect keys', async () => {
+ const desiredContext: ContextCRD = {
+ name: 'desired-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'desired-context', mcpServers: [] },
+ }
+ const effectKeys: string[] = []
+ vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'rpc-egress-orphan-context-old-server',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([desiredContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ runContextEffect: async (contextId, work) => {
+ effectKeys.push(contextId)
+ await work()
+ },
+ })
+
+ expect(effectKeys).toEqual([
+ 'orphan-context',
+ 'orphan-context',
+ 'desired-context',
+ 'desired-context',
+ ])
+ })
+
+ it('revokes every orphan allow lane before a slow additive Context effect', async () => {
+ const desiredContext: ContextCRD = {
+ name: 'desired-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'desired-context', mcpServers: [] },
+ }
+ const additiveStarted = deferred()
+ const releaseAdditive = deferred()
+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {
+ additiveStarted.resolve(undefined)
+ await releaseAdditive.promise
+ return true
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-orphan-context-old-server-egress',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'rpc-egress-orphan-context-old-server',
+ labels: { 'clerum.io/context': 'orphan-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-orphan-server-old.example',
+ labels: { 'clerum.io/mcpserver': 'orphan-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ const fullPass = reconciler.fullReconcile([desiredContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ })
+
+ await additiveStarted.promise
+ try {
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-orphan-context-old-server',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-orphan-context-old-server-egress',
+ namespace: 'mcp-host',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'rpc-egress-orphan-context-old-server',
+ namespace: 'rpc-proxy',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-orphan-server-old.example',
+ namespace: 'mcp-server',
+ })
+ )
+ } finally {
+ releaseAdditive.resolve(undefined)
+ await fullPass
+ }
+ })
+
+ it('revokes every stale allow lane for a live Context before signaling authoritative revocation', async () => {
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [] },
+ }
+ const additiveStarted = deferred()
+ const releaseAdditive = deferred()
+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {
+ additiveStarted.resolve(undefined)
+ await releaseAdditive.promise
+ return true
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-live-context-removed-server',
+ labels: { 'clerum.io/context': 'live-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-live-context-removed-server-egress',
+ labels: { 'clerum.io/context': 'live-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'rpc-egress-live-context-removed-server',
+ labels: { 'clerum.io/context': 'live-context' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ const fullPass = reconciler.fullReconcile([liveContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ await additiveStarted.promise
+ try {
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-live-context-removed-server',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ctx-live-context-removed-server-egress',
+ namespace: 'mcp-host',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'rpc-egress-live-context-removed-server',
+ namespace: 'rpc-proxy',
+ })
+ )
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ } finally {
+ releaseAdditive.resolve(undefined)
+ await fullPass
+ }
+ })
+
+ it('revokes stale external egress for a live McpServer before readiness certification', async () => {
+ const liveServer: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [],
+ },
+ }
+ const ordering: string[] = []
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-live-server-old.example-443',
+ labels: { 'clerum.io/mcpserver': 'live-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('delete')
+ return {}
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ ordering.push('certify')
+ })
+
+ await reconciler.fullReconcile([], [liveServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-live-server-old.example-443',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it('does not let one post-certification Context failure starve later additive owners', async () => {
+ const contexts: ContextCRD[] = ['aaa-poisoned', 'bbb-healthy'].map(contextId => ({
+ name: `${contextId}-resource`,
+ namespace: 'mcp-server',
+ spec: { contextId, mcpServers: [] },
+ }))
+ const ordering: string[] = []
+ const additiveFailure = new Error('poisoned additive Context')
+ vi.spyOn(reconciler, 'reconcileContext').mockImplementation(async context => {
+ ordering.push(`context:${context.spec.contextId}`)
+ if (context.spec.contextId === 'aaa-poisoned') throw additiveFailure
+ return true
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ ordering.push('certify')
+ })
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+
+ const error = await reconciler
+ .fullReconcile(contexts, [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ .then(
+ () => undefined,
+ failure => failure
+ )
+
+ expect(error).toBeInstanceOf(AggregateError)
+ expect((error as AggregateError).errors).toEqual([additiveFailure])
+ expect(ordering).toEqual(['certify', 'context:aaa-poisoned', 'context:bbb-healthy'])
+ expect(errorLog).toHaveBeenCalledWith(
+ '[NetPol] Additive Context reconciliation failed for "aaa-poisoned":',
+ additiveFailure
+ )
+ })
+
+ it('replaces same-name Context allow policies in every lane before certification', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ transport: { type: 'streamableHttp', port: 4000 },
+ },
+ }
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [oldServer.name] },
+ }
+ const serverCache = new Map([[oldServer.name, oldServer]])
+ const rec = makeReconciler(mockApi, serverCache, mockCustomApi)
+
+ await rec.reconcileContext(liveContext, { isCurrent: () => true })
+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(
+ call => (call[0] as { body: k8s.V1NetworkPolicy }).body
+ )
+ expect(oldPolicies).toHaveLength(3)
+
+ vi.clearAllMocks()
+ serverCache.set(currentServer.name, currentServer)
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (labelSelector?.includes('external-egress')) return { items: [] }
+ return {
+ items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),
+ }
+ }
+ )
+ vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)
+ vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)
+ const ordering: string[] = []
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => {
+ ordering.push(`replace:${namespace}`)
+ return {}
+ }
+ )
+ const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))
+
+ await rec.fullReconcile([liveContext], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(ordering).toEqual([
+ 'replace:mcp-server',
+ 'replace:mcp-host',
+ 'replace:rpc-proxy',
+ 'certify',
+ ])
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it.each([
+ ['context-ingress', 'mcp-server'],
+ ['context-host-egress', 'mcp-host'],
+ ['context-rpc-egress', 'rpc-proxy'],
+ ])(
+ 'refuses a same-name %s safety replacement owned by another server',
+ async (_lane, targetNamespace) => {
+ const oldServer: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ transport: { type: 'streamableHttp', port: 4000 },
+ },
+ }
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [oldServer.name] },
+ }
+ const serverCache = new Map([[oldServer.name, oldServer]])
+ const rec = makeReconciler(mockApi, serverCache, mockCustomApi)
+
+ await rec.reconcileContext(liveContext, { isCurrent: () => true })
+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {
+ const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body
+ if (policy.metadata?.namespace !== targetNamespace) return policy
+ return {
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ labels: {
+ ...policy.metadata?.labels,
+ 'clerum.io/mcpserver': 'different-server',
+ },
+ },
+ }
+ })
+
+ vi.clearAllMocks()
+ serverCache.set(currentServer.name, currentServer)
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (labelSelector?.includes('external-egress')) return { items: [] }
+ return {
+ items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),
+ }
+ }
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ rec.fullReconcile([liveContext], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/ownership/i)
+
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ namespace: targetNamespace })
+ )
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ }
+ )
+
+ it('fails the safety pass without certifying when a same-name Context replacement fails', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ transport: { type: 'streamableHttp', port: 4000 },
+ },
+ }
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [oldServer.name] },
+ }
+ const serverCache = new Map([[oldServer.name, oldServer]])
+ const rec = makeReconciler(mockApi, serverCache, mockCustomApi)
+
+ await rec.reconcileContext(liveContext, { isCurrent: () => true })
+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(
+ call => (call[0] as { body: k8s.V1NetworkPolicy }).body
+ )
+
+ vi.clearAllMocks()
+ serverCache.set(currentServer.name, currentServer)
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (labelSelector?.includes('external-egress')) return { items: [] }
+ return {
+ items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),
+ }
+ }
+ )
+ const replacementFailure = new Error('replace unavailable')
+ mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(replacementFailure)
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ rec.fullReconcile([liveContext], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toBe(replacementFailure)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
+ { outcome: 'failed' },
+ expect.any(Number)
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
+ })
+
+ it('retains authoritative same-name policies when only API object key order differs', async () => {
+ const server: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [server.name] },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
+ const reverseObjectKeys = (value: unknown): unknown => {
+ if (Array.isArray(value)) return value.map(reverseObjectKeys)
+ if (value === null || typeof value !== 'object') return value
+ return Object.fromEntries(
+ Object.entries(value as Record<string, unknown>)
+ .reverse()
+ .map(([key, entry]) => [key, reverseObjectKeys(entry)])
+ )
+ }
+
+ await rec.reconcileContext(liveContext, { isCurrent: () => true })
+ const existingPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {
+ const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body
+ return { ...policy, spec: reverseObjectKeys(policy.spec) as k8s.V1NetworkPolicySpec }
+ })
+ vi.clearAllMocks()
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (labelSelector?.includes('external-egress')) return { items: [] }
+ return {
+ items: existingPolicies.filter(policy => policy.metadata?.namespace === namespace),
+ }
+ }
+ )
+ vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)
+ vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)
+
+ await rec.fullReconcile([liveContext], [server], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('revokes same-name DNS egress before certifying and leaves refresh to its coordinator', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
+
+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
+ const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
+ expect(oldPolicy).toBeDefined()
+
+ vi.clearAllMocks()
+ const ordering: string[] = []
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: [oldPolicy!] } : { items: [] }
+ )
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('replace')
+ return {}
+ })
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('delete')
+ return {}
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ ordering.push('certify')
+ })
+
+ await rec.fullReconcile([], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])
+ expect(ordering).not.toContain('replace')
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it('revokes a divergent same-name DNS policy when its safe replacement cannot resolve', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'live-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'live:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
+
+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
+ const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
+ expect(oldPolicy).toBeDefined()
+
+ vi.clearAllMocks()
+ const ordering: string[] = []
+ vi.mocked(dns.resolve4).mockImplementation(async () => {
+ ordering.push('dns')
+ throw new Error('dns unavailable')
+ })
+ let currentPolicies = [oldPolicy!]
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('delete')
+ currentPolicies = []
+ return {}
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ ordering.push('certify')
+ })
+
+ await expect(
+ rec.fullReconcile([], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).resolves.toBeUndefined()
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: oldPolicy!.metadata!.name!,
+ namespace: oldServer.namespace,
+ })
+ )
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ expect(ordering).toEqual(['delete', 'certify'])
+ })
+
+ it('revokes a renamed DNS binding before certification and performs no DNS lookup on the safety path', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'renamed-dns-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'dns:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'old.example.com', port: 443 }],
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ egressBindings: [{ dns: 'new.example.com', port: 443 }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
+ const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
+
+ vi.clearAllMocks()
+ const ordering: string[] = []
+ let currentPolicies = [oldPolicy]
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ ordering.push('delete')
+ currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)
+ return {}
+ })
+ vi.mocked(dns.resolve4).mockImplementation(async () => {
+ ordering.push('dns')
+ return ['1.2.3.4']
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ ordering.push('certify')
+ })
+
+ await rec.fullReconcile([], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(ordering).toEqual(['delete', 'certify'])
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: oldPolicy.metadata!.name!,
+ namespace: oldServer.namespace,
+ body: {
+ preconditions: {
+ uid: `${oldServer.namespace}:${oldPolicy.metadata!.name!}:uid`,
+ resourceVersion: '1',
+ },
+ },
+ })
+ })
+
+ it('replaces deterministic drift and revokes renamed DNS before certification without resolving DNS', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'mixed-drift-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'mixed:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [
+ { cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' },
+ { dns: 'old.example.com', port: 443, protocol: 'TCP' },
+ ],
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ egressBindings: [
+ { cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' },
+ { dns: 'new.example.com', port: 443, protocol: 'TCP' },
+ ],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
+ const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .filter(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
+
+ vi.clearAllMocks()
+ const ordering: string[] = []
+ let currentPolicies = oldPolicies
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
+ )
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('replace')
+ return {}
+ })
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ ordering.push('delete')
+ currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)
+ return {}
+ })
+ vi.mocked(dns.resolve4).mockImplementation(async () => {
+ ordering.push('dns')
+ return ['1.2.3.4']
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ ordering.push('certify')
+ })
+
+ await rec.fullReconcile([], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(ordering).toEqual(['replace', 'delete', 'certify'])
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ })
+
+ it('aborts certification when an identity-drifted safety delete reports a recreate conflict', async () => {
+ const server: McpServerCRD = {
+ name: 'recreated-dns-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'dns:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443 }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })
+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
+ // B3: identity-drift the live policy (protocol TCP→UDP) so the safety pass
+ // still revokes it. A same-intent (identity-stable) DNS allow is now
+ // RETAINED (see the retention pins in the B3 describe block); the revoke /
+ // abort machinery this test guards fires only for a drifted policy.
+ existing.spec!.egress = existing.spec!.egress!.map(rule => ({
+ ...rule,
+ ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),
+ }))
+
+ vi.clearAllMocks()
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }
+ )
+ const conflict = Object.assign(new Error('delete precondition conflict'), { code: 409 })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(conflict)
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ rec.fullReconcile([], [server], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toBe(conflict)
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: existing.metadata!.name!,
+ namespace: server.namespace,
+ body: {
+ preconditions: {
+ uid: `${server.namespace}:${existing.metadata!.name!}:uid`,
+ resourceVersion: '1',
+ },
+ },
+ })
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ [
+ 'foreign relabel',
+ {
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': 'fresh-read-server',
+ },
+ },
+ ],
+ ['UID drift', { uid: 'replacement-uid' }],
+ ['resourceVersion drift', { resourceVersion: '2' }],
+ ])(
+ 'aborts a deterministic safety replace after fresh-read %s',
+ async (_case, metadataDrift) => {
+ const oldServer: McpServerCRD = {
+ name: 'fresh-read-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'fresh:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
+
+ vi.clearAllMocks()
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ metadata: {
+ ...metadataDrift,
+ },
+ })
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ rec.fullReconcile([], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/ownership/)
+
+ expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: existing.metadata!.name!,
+ namespace: oldServer.namespace,
+ })
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ }
+ )
+
+ it('uses the fresh identity to complete a deterministic safety replacement before certification', async () => {
+ const oldServer: McpServerCRD = {
+ name: 'fresh-success-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'fresh:test',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const currentServer: McpServerCRD = {
+ ...oldServer,
+ spec: {
+ ...oldServer.spec,
+ egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
+ await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
+
+ vi.clearAllMocks()
+ const ordering: string[] = []
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }
+ )
+ mockApi.readNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('read')
+ return { metadata: { resourceVersion: '1' } }
+ })
+ mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {
+ ordering.push('replace')
+ return {}
+ })
+ const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))
+
+ await rec.fullReconcile([], [currentServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(ordering.slice(0, 3)).toEqual(['read', 'replace', 'certify'])
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: existing.metadata!.name!,
+ namespace: oldServer.namespace,
+ body: expect.objectContaining({
+ metadata: expect.objectContaining({
+ name: existing.metadata!.name!,
+ namespace: oldServer.namespace,
+ resourceVersion: '1',
+ }),
+ spec: expect.objectContaining({
+ egress: expect.arrayContaining([
+ expect.objectContaining({
+ ports: [expect.objectContaining({ protocol: 'UDP' })],
+ }),
+ ]),
+ }),
+ }),
+ })
+ })
+
+ it('revokes an identity-drifted DNS policy before readiness and defers its refresh', async () => {
+ const server: McpServerCRD = {
+ name: 'stable-dns-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'stable:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })
+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
+ expect(existing).toBeDefined()
+ // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a
+ // same-intent DNS allow is now retained. The revoke-before-readiness +
+ // defer-refresh machinery this test guards still fires on the drifted one.
+ existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({
+ ...rule,
+ ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),
+ }))
+
+ vi.clearAllMocks()
+ let currentPolicies = [existing!]
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
+ currentPolicies = []
+ return {}
+ })
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await rec.fullReconcile([], [server], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: existing!.metadata!.name!,
+ namespace: server.namespace,
+ })
+ )
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ })
+
+ it('does not certify safety when an identity-drifted DNS policy cannot be revoked', async () => {
+ const server: McpServerCRD = {
+ name: 'undeletable-dns-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'stable:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
+ },
+ }
+ const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
+ await rec.reconcileExternalEgress(server, { isCurrent: () => true })
+ const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
+ .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
+ .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
+ expect(existing).toBeDefined()
+ // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a
+ // same-intent DNS allow is now retained. The no-certify-when-revoke-fails
+ // machinery this test guards still fires on the drifted one.
+ existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({
+ ...rule,
+ ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),
+ }))
+
+ vi.clearAllMocks()
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) =>
+ labelSelector?.includes('external-egress') ? { items: [existing!] } : { items: [] }
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('delete denied'))
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ rec.fullReconcile([], [server], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow('delete denied')
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: existing!.metadata!.name!,
+ namespace: server.namespace,
+ })
+ )
+ })
+
+ it('does not certify revocation when an earlier Context changes while a later owner is paused', async () => {
+ const contexts: ContextCRD[] = ['first', 'later'].map(contextId => ({
+ name: `${contextId}-resource`,
+ namespace: 'mcp-server',
+ spec: { contextId, mcpServers: [] },
+ }))
+ const currentContexts = new Map(contexts.map(context => [context.name, context]))
+ let contextDesiredRevision = 10
+ let laterPaused = false
+ const laterStarted = deferred()
+ const releaseLater = deferred()
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ const fullPass = reconciler.fullReconcile(contexts, [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ resolveCurrentContext: name => currentContexts.get(name),
+ runContextEffect: async (contextId, work) => {
+ if (contextId === 'later' && !laterPaused) {
+ laterPaused = true
+ laterStarted.resolve(undefined)
+ await releaseLater.promise
+ }
+ await work()
+ },
+ contextDesiredRevision: () => contextDesiredRevision,
+ onAuthoritativeRevocationComplete,
+ })
+
+ await laterStarted.promise
+ currentContexts.set('first-resource', {
+ ...contexts[0],
+ spec: { ...contexts[0].spec, mcpServers: ['replacement'] },
+ })
+ contextDesiredRevision += 1
+ releaseLater.resolve(undefined)
+
+ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('does not certify revocation when a new owner is added after the global inventory LISTs', async () => {
+ const anchorContext: ContextCRD = {
+ name: 'anchor-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'anchor', mcpServers: [] },
+ }
+ const currentContexts = new Map([[anchorContext.name, anchorContext]])
+ let contextDesiredRevision = 21
+ const safetyEffectStarted = deferred()
+ const releaseSafetyEffect = deferred()
+ let safetyEffectPaused = false
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ const fullPass = reconciler.fullReconcile([anchorContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ resolveCurrentContext: name => currentContexts.get(name),
+ runContextEffect: async (_contextId, work) => {
+ if (!safetyEffectPaused) {
+ safetyEffectPaused = true
+ safetyEffectStarted.resolve(undefined)
+ await releaseSafetyEffect.promise
+ }
+ await work()
+ },
+ contextDesiredRevision: () => contextDesiredRevision,
+ onAuthoritativeRevocationComplete,
+ })
+
+ await safetyEffectStarted.promise
+ currentContexts.set('late-resource', {
+ name: 'late-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'late', mcpServers: [] },
+ })
+ contextDesiredRevision += 1
+ releaseSafetyEffect.resolve(undefined)
+
+ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('does not certify revocation when a new McpServer owner is added after inventory LISTs', async () => {
+ const anchorServer: McpServerCRD = {
+ name: 'anchor-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'anchor:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const currentServers = new Map([[anchorServer.name, anchorServer]])
+ let serverDesiredRevision = 31
+ const safetyEffectStarted = deferred()
+ const releaseSafetyEffect = deferred()
+ let safetyEffectPaused = false
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ const fullPass = reconciler.fullReconcile([], [anchorServer], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ resolveCurrentServer: name => currentServers.get(name),
+ runServerEffect: async (_serverName, work) => {
+ if (!safetyEffectPaused) {
+ safetyEffectPaused = true
+ safetyEffectStarted.resolve(undefined)
+ await releaseSafetyEffect.promise
+ }
+ await work()
+ },
+ serverDesiredRevision: () => serverDesiredRevision,
+ onAuthoritativeRevocationComplete,
+ })
+
+ await safetyEffectStarted.promise
+ currentServers.set('late-server', {
+ name: 'late-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'late:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ })
+ serverDesiredRevision += 1
+ releaseSafetyEffect.resolve(undefined)
+
+ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('does not certify revocation when desired inventory changes while default policies are ensured', async () => {
+ const context: ContextCRD = {
+ name: 'default-window-context',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default-window', mcpServers: [] },
+ }
+ const server: McpServerCRD = {
+ name: 'default-window-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default-window',
+ image: 'default-window:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ let contextDesiredRevision = 41
+ let serverDesiredRevision = 51
+ const defaultsStarted = deferred()
+ const releaseDefaults = deferred()
+ const onAuthoritativeRevocationComplete = vi.fn()
+ vi.spyOn(reconciler, 'ensureDefaultPolicies').mockImplementation(async () => {
+ defaultsStarted.resolve(undefined)
+ await releaseDefaults.promise
+ })
+
+ const fullPass = reconciler.fullReconcile([context], [server], {
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ contextDesiredRevision: () => contextDesiredRevision,
+ serverDesiredRevision: () => serverDesiredRevision,
+ onAuthoritativeRevocationComplete,
+ })
+
+ await defaultsStarted.promise
+ contextDesiredRevision += 1
+ serverDesiredRevision += 1
+ releaseDefaults.resolve(undefined)
+
+ await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('does not signal authoritative revocation when a live-owner delete fails', async () => {
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-live-context-removed-server',
+ labels: { 'clerum.io/context': 'live-context' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(
+ new Error('policy delete unavailable')
+ )
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([liveContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow('policy delete unavailable')
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('does not delete or signal revocation after live Context authority is lost', async () => {
+ const liveContext: ContextCRD = {
+ name: 'live-context-resource',
+ namespace: 'mcp-server',
+ spec: { contextId: 'live-context', mcpServers: [] },
+ }
+ let authoritative = true
+ const inventoryListed = deferred()
+ const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()
+ mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {
+ inventoryListed.resolve(undefined)
+ return releaseInventory.promise
+ })
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ const fullPass = reconciler.fullReconcile([liveContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => authoritative,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ await inventoryListed.promise
+ authoritative = false
+ releaseInventory.resolve({
+ items: [
+ {
+ metadata: {
+ name: 'ctx-live-context-removed-server',
+ labels: { 'clerum.io/context': 'live-context' },
+ },
+ },
+ ],
+ })
+
+ await fullPass
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('uses selected lane LISTs plus one broad LIST per bounded namespace before additive convergence', async () => {
+ const contexts: ContextCRD[] = ['alpha', 'beta', 'gamma'].map(contextId => ({
+ name: `${contextId}-resource`,
+ namespace: 'mcp-server',
+ spec: { contextId, mcpServers: [] },
+ }))
+ const servers: McpServerCRD[] = ['one', 'two'].map(name => ({
+ name,
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: `${name}:latest`,
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }))
+ const additiveStarted = deferred()
+ const releaseAdditive = deferred()
+ vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {
+ additiveStarted.resolve(undefined)
+ await releaseAdditive.promise
+ return true
+ })
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+
+ const fullPass = reconciler.fullReconcile(contexts, servers, {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ })
+
+ await additiveStarted.promise
+ try {
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'mcp-server',
+ labelSelector: expect.stringContaining('policy-type=context-allow'),
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'mcp-host',
+ labelSelector: expect.stringContaining('policy-type=context-allow'),
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'rpc-proxy',
+ labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'mcp-server',
+ labelSelector: expect.stringContaining('policy-type=external-egress'),
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'mcp-server',
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'mcp-host',
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ namespace: 'rpc-proxy',
+ })
+ } finally {
+ releaseAdditive.resolve(undefined)
+ await fullPass
+ }
+ })
+
+ it('leaves startup external egress creation to its dedicated coordinator', async () => {
+ const server: McpServerCRD = {
+ name: 'airtable-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'airtable:latest',
+ transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.airtable.com', port: 443 }],
+ },
+ }
+
+ await reconciler.fullReconcile([], [server])
+
+ const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
+ (call: unknown[]) => {
+ const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }
+ return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'
+ }
+ )
+ expect(externalPolicies).toHaveLength(0)
+ expect(dns.resolve4).not.toHaveBeenCalled()
+ })
+
+ it('deletes orphaned external egress policies for servers no longer present', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) => {
+ if (labelSelector?.includes('context-allow')) return { items: [] }
+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-old-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'old-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], { serverInventoryComplete: true })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'ext-egress-old-server-api-example-com-443',
+ namespace: 'mcp-server',
+ })
+ )
+ })
+
+ it('fails closed by deleting HCC-managed policies with missing owner labels', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ const managedLabels = { 'clerum.io/managed-by': 'host-context-controller' }
+ if (labelSelector?.includes('policy-type=context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name:
+ namespace === 'mcp-host'
+ ? 'malformed-host-egress'
+ : 'malformed-context-ingress',
+ labels: managedLabels,
+ },
+ },
+ ],
+ }
+ }
+ if (labelSelector?.includes('rpc-proxy-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'malformed-rpc-proxy-egress',
+ labels: managedLabels,
+ },
+ },
+ ],
+ }
+ }
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'malformed-external-egress',
+ labels: managedLabels,
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], {
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ serverInventoryComplete: true,
+ })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'malformed-context-ingress',
+ namespace: 'mcp-server',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'malformed-host-egress',
+ namespace: 'mcp-host',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'malformed-rpc-proxy-egress',
+ namespace: 'rpc-proxy',
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ name: 'malformed-external-egress',
+ namespace: 'mcp-server',
+ })
+ )
+ })
+
+ it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) => {
+ if (labelSelector?.includes('context-allow')) return { items: [] }
+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-old-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'old-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], { serverInventoryComplete: false })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
+ name: 'ext-egress-old-server-api-example-com-443',
+ namespace: 'mcp-server',
+ })
+ })
+
+ it('finishes desired Context reconciliation but stops Context and rpc-proxy orphan cleanup when authority is lost after inventory listing', async () => {
+ const desiredContext: ContextCRD = {
+ name: 'current',
+ namespace: 'mcp-server',
+ spec: { contextId: 'current', mcpServers: [] },
+ }
+ const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)
+ const contextInventoryAuthoritative = vi
+ .fn()
+ .mockReturnValueOnce(true)
+ .mockReturnValueOnce(false)
+ .mockReturnValue(true)
+
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('policy-type=context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-deleted-context-old-server',
+ labels: { 'clerum.io/context': 'deleted-context' },
+ },
+ },
+ ],
+ }
+ }
+ if (labelSelector?.includes('external-egress')) return { items: [] }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([desiredContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative,
+ })
+
+ expect(reconcileContext).toHaveBeenCalledWith(desiredContext, {
+ isCurrent: expect.any(Function),
+ })
+ expect(contextInventoryAuthoritative).toHaveBeenCalled()
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
+ name: 'ctx-deleted-context-old-server',
+ namespace: 'mcp-server',
+ })
+ expect(mockApi.listNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: 'rpc-proxy',
+ labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),
+ })
+ )
+ })
+
+ it('stops external-egress orphan cleanup when authority is lost after inventory listing', async () => {
+ const desiredServer: McpServerCRD = {
+ name: 'current-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'current',
+ image: 'current:latest',
+ transport: { type: 'streamableHttp', url: 'http://current:3000', port: 3000 },
+ },
+ }
+ const serverInventoryAuthoritative = vi
+ .fn()
+ .mockReturnValueOnce(true)
+ .mockReturnValueOnce(false)
+ .mockReturnValue(true)
+
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('context-allow')) return { items: [] }
- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
if (labelSelector?.includes('external-egress')) {
return {
items: [
{
metadata: {
- name: 'ext-egress-old-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'old-server' },
+ name: 'ext-egress-deleted-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'deleted-server' },
},
},
],
@@ -1547,13 +5058,146 @@ describe('NetworkPolicyReconciler', () => {
}
)
- await reconciler.fullReconcile([], [], { serverInventoryComplete: false })
+ await reconciler.fullReconcile([], [desiredServer], {
+ ensureDefaults: false,
+ serverInventoryComplete: true,
+ serverInventoryAuthoritative,
+ })
+ expect(serverInventoryAuthoritative).toHaveBeenCalled()
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
- name: 'ext-egress-old-server-api-example-com-443',
+ name: 'ext-egress-deleted-server-api-example-com-443',
namespace: 'mcp-server',
})
})
+
+ it('does not delete any Context policy lane when a same-contextId CRD exists before its watch event arrives', async () => {
+ mockCustomApi.listNamespacedCustomObject.mockResolvedValue({
+ items: [{ metadata: { name: 'recreated-context' }, spec: { contextId: 'recreated' } }],
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-recreated-server',
+ labels: { 'clerum.io/context': 'recreated' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ctx-recreated-server-egress',
+ labels: { 'clerum.io/context': 'recreated' },
+ },
+ },
+ ],
+ }
+ }
+ if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'rpc-egress-recreated-server',
+ labels: { 'clerum.io/context': 'recreated' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ })
+
+ expect(mockCustomApi.listNamespacedCustomObject).toHaveBeenCalled()
+ expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: 'mcp-host',
+ labelSelector: expect.stringContaining('policy-type=context-allow'),
+ })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('does not delete external-egress policy when a same-name McpServer exists before ADDED arrives', async () => {
+ mockCustomApi.getNamespacedCustomObject.mockResolvedValue({
+ metadata: { name: 'recreated-server', uid: 'new-uid' },
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) => {
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-recreated-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'recreated-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ serverInventoryComplete: true,
+ serverInventoryAuthoritative: () => true,
+ })
+
+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledWith(
+ expect.objectContaining({ plural: 'mcpservers', name: 'recreated-server' })
+ )
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('delegates startup McpServer orphan safety to the canonical absence helper', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) => {
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-orphan-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'orphan-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
+
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ serverInventoryComplete: true,
+ serverInventoryAuthoritative: () => true,
+ })
+
+ expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledTimes(1)
+ expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledWith({
+ inventoryAuthoritative: expect.any(Function),
+ resolveCurrent: expect.any(Function),
+ readCurrent: expect.any(Function),
+ })
+ })
})
describe('L3 — cleanupExternalEgress', () => {
@@ -1564,7 +5208,16 @@ describe('NetworkPolicyReconciler', () => {
await reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: 'ext-egress-openai-mcp-api-443',
+ namespace: 'mcp-server',
+ body: {
+ preconditions: {
+ uid: 'mcp-server:ext-egress-openai-mcp-api-443:uid',
+ resourceVersion: '1',
+ },
+ },
+ })
})
it('fails closed when stale external egress policy deletion fails', async () => {
@@ -1590,5 +5243,467 @@ describe('NetworkPolicyReconciler', () => {
reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')
).resolves.toBeUndefined()
})
+
+ it('does not report a revocation when the policy was already absent', async () => {
+ const error = Object.assign(new Error('not found'), { code: 404 })
+ const onDeleted = vi.fn()
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [{ metadata: { name: 'ext-egress-openai-mcp-old-api-443' } }],
+ })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(error)
+
+ await reconciler.cleanupExternalEgress(
+ 'openai-mcp',
+ 'mcp-server',
+ undefined,
+ undefined,
+ onDeleted
+ )
+
+ expect(onDeleted).not.toHaveBeenCalled()
+ })
+ })
+
+ describe('scoped delta certification vs. a stuck safety inventory', () => {
+ const ambiguousReservedPolicy: k8s.V1NetworkPolicy = {
+ metadata: { name: 'ctx-reserved-context-server', labels: {} },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ const deltaContext: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default', mcpServers: [] },
+ }
+
+ function stickSafetyInventory(): void {
+ // Only the namespace-wide safety inventory sees the foreign policy. The
+ // label-scoped LISTs the delta lane uses stay clean, which is exactly why
+ // the delta cannot vouch for the namespace.
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items: !labelSelector && namespace === 'mcp-server' ? [ambiguousReservedPolicy] : [],
+ })
+ )
+ }
+
+ it('refuses to certify a scoped delta while the safety pass is stuck on its inventory', async () => {
+ stickSafetyInventory()
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+
+ // The delta's own revocation still completes — its label-scoped LISTs are
+ // clean. What it may not do is vouch for the namespace-wide inventory,
+ // which the readiness lane checks separately.
+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ })
+
+ it('certifies a scoped delta again once a later safety pass completes', async () => {
+ stickSafetyInventory()
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: vi.fn(),
+ })
+ ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
+
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ const onAuthoritativeRevocationComplete = vi.fn()
+ await reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
+ })
+
+ it('does not certify the safety inventory until an authoritative pass has run', async () => {
+ // Fail-closed default: before any authoritative safety pass certifies, the
+ // namespace-wide inventory is NOT certified, even though the scoped delta's
+ // own label-scoped revocation completes (returns true). A delta cannot
+ // vouch for the namespace-wide inventory it never enumerated.
+ expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ })
+ })
+
+ describe('lost delete fence during stale-allow revocation', () => {
+ const staleContext: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default', mcpServers: [] },
+ }
+ const stalePolicy: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-default-alpha',
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'default',
+ 'clerum.io/mcpserver': 'alpha',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ const lostFence = Object.assign(new Error('the UID in the precondition does not match'), {
+ code: 409,
+ })
+
+ function listOnlyTheStalePolicy(): void {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: namespace === 'mcp-server' ? [stalePolicy] : [],
+ })
+ )
+ }
+
+ it('refuses to certify a scoped delta instead of throwing the lost fence', async () => {
+ listOnlyTheStalePolicy()
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)
+
+ // 409 is what Kubernetes returns when the uid/resourceVersion
+ // preconditions no longer match: the object live under that name is not
+ // the one this pass classified. Nothing was revoked, so the delta must
+ // decline to certify — and it must not blow up the caller, which would
+ // only force a spurious full re-convergence.
+ // The scoped delta lane reads this boolean and withholds the certificate,
+ // so it is the one caller allowed to take the lost fence as an outcome
+ // rather than a throw.
+ expect(await reconciler.reconcileContext(staleContext, { honorsLostFence: true, isCurrent: () => true })).toBe(false)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'ctx-default-alpha', namespace: 'mcp-server' })
+ )
+ })
+
+ it('withdraws certification when an additive revocation loses the fence after certifying', async () => {
+ // The additive phase re-runs reconcileContext, which carries a second
+ // revocation lane over a fresh LIST. Losing the fence there happens after
+ // the pass already certified, and the outcome recorder is first-wins — so
+ // the duration sample stays 'completed', which is right, but the safety
+ // fence must still degrade. It is an assertion about live allows, and it
+ // may only ever move toward unsafe.
+ const additiveContext: ContextCRD = {
+ name: 'additive',
+ namespace: 'mcp-server',
+ spec: { contextId: 'additive', mcpServers: [] },
+ }
+ const additiveStalePolicy: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-additive-alpha',
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'additive',
+ 'clerum.io/mcpserver': 'alpha',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ // Self-synchronising: the stale policy only becomes visible once the pass
+ // has already certified, so the safety phase is clean by construction and
+ // only the additive re-LIST finds work.
+ let certified = false
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: certified && namespace === 'mcp-server' ? [additiveStalePolicy] : [],
+ })
+ )
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(lostFence)
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ certified = true
+ })
+
+ await expect(
+ reconciler.fullReconcile([additiveContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow()
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ })
+
+ it('marks the inventory uncertified the moment the authoritative pass loses the fence', async () => {
+ // The doom must be visible mid-pass, not only after the unwind. A lost
+ // fence is the one doom cause that does not bump a watch generation, so
+ // it is the only one where the whole certification machinery stays green
+ // while the pass is already condemned. A scoped delta interleaving in
+ // that window would certify readiness over a stale allow.
+ //
+ // Two contexts, each with its own stale policy: the first loses the fence
+ // and the second's delete gives an observation point while the pass is
+ // condemned but still unwinding.
+ const secondContext: ContextCRD = {
+ name: 'second',
+ namespace: 'mcp-server',
+ spec: { contextId: 'second', mcpServers: [] },
+ }
+ const secondStalePolicy: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ctx-second-beta',
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'context-allow',
+ 'clerum.io/context': 'second',
+ 'clerum.io/mcpserver': 'beta',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+ }
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: namespace === 'mcp-server' ? [stalePolicy, secondStalePolicy] : [],
+ })
+ )
+ let certifiedDuringUnwind: boolean | undefined
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
+ if (
+ certifiedDuringUnwind === undefined &&
+ mockApi.deleteNamespacedNetworkPolicy.mock.calls.length > 1
+ ) {
+ certifiedDuringUnwind = reconciler.hasCertifiedSafetyInventory()
+ return {}
+ }
+ throw lostFence
+ })
+
+ await expect(
+ reconciler.fullReconcile([staleContext, secondContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: vi.fn(),
+ })
+ ).rejects.toThrow()
+
+ expect(certifiedDuringUnwind).toBe(false)
+ })
+
+ it('fails the additive phase when it loses the delete fence', async () => {
+ listOnlyTheStalePolicy()
+ // Safety phase revokes cleanly and certifies; the additive phase then
+ // races another writer and loses the fence. The additive caller discards
+ // reconcileContext's boolean, so swallowing the 409 there would record
+ // the pass as a success, disarm the convergence retry, and leave a stale
+ // allow live with no NetworkPolicy resync to recover it.
+ mockApi.deleteNamespacedNetworkPolicy
+ .mockResolvedValueOnce({})
+ .mockRejectedValueOnce(lostFence)
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([staleContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow()
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ })
+
+ it('refuses to certify the safety pass instead of throwing the lost fence', async () => {
+ listOnlyTheStalePolicy()
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ await expect(
+ reconciler.fullReconcile([staleContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/Desired NetworkPolicy inventory changed/)
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ })
+
+ it('still fails loud when the delete fails for any other reason', async () => {
+ listOnlyTheStalePolicy()
+ const forbidden = Object.assign(new Error('forbidden'), { code: 403 })
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)
+
+ await expect(reconciler.reconcileContext(staleContext, { isCurrent: () => true })).rejects.toBe(forbidden)
+ })
+
+ it('still certifies when the stale policy was already gone', async () => {
+ listOnlyTheStalePolicy()
+ mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('not found'), { code: 404 })
+ )
+
+ expect(await reconciler.reconcileContext(staleContext, { isCurrent: () => true })).toBe(true)
+ })
+ })
+
+ describe('same-name conflict creating a safety external-egress policy', () => {
+ const server: McpServerCRD = {
+ name: 'renamed-server',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'default',
+ image: 'renamed:latest',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
+ },
+ }
+ const desiredName = 'ext-egress-renamed-server-104-18-0-0-16-443'
+ const stalePredecessor: k8s.V1NetworkPolicy = {
+ metadata: {
+ name: 'ext-egress-renamed-server-stale',
+ namespace: 'mcp-server',
+ uid: 'stale-uid',
+ resourceVersion: '3',
+ },
+ }
+
+ function reconcileExternalEgressSafety(): Promise<boolean> {
+ return (
+ reconciler as unknown as {
+ reconcileExternalEgressSafety: (
+ server: McpServerCRD,
+ existingPolicies: k8s.V1NetworkPolicy[],
+ isCurrent: () => boolean
+ ) => Promise<boolean>
+ }
+ ).reconcileExternalEgressSafety(server, [stalePredecessor], () => true)
+ }
+
+ it('converges an HCC-owned same-name policy instead of aborting the pass', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: desiredName,
+ namespace: server.namespace,
+ uid: 'own-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ })
+
+ // This is the only creation route that bypasses applyNetworkPolicy. A
+ // same-name object HCC already owns is an idempotent re-run, not a
+ // reason to fail the pass and hold readiness down.
+ expect(await reconcileExternalEgressSafety()).toBe(true)
+ expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
+ expect.objectContaining({ name: desiredName, namespace: server.namespace })
+ )
+ })
+
+ it('still fails closed when the same-name policy is foreign', async () => {
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
+ Object.assign(new Error('already exists'), { code: 409 })
+ )
+ mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
+ metadata: {
+ name: desiredName,
+ namespace: server.namespace,
+ uid: 'foreign-uid',
+ resourceVersion: '7',
+ labels: {
+ 'clerum.io/managed-by': 'foreign-controller',
+ 'clerum.io/policy-type': 'external-egress',
+ 'clerum.io/mcpserver': server.name,
+ },
+ },
+ })
+
+ await expect(reconcileExternalEgressSafety()).rejects.toThrow(/ownership/i)
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('still fails loud when the create fails for any other reason', async () => {
+ const forbidden = Object.assign(new Error('forbidden'), { code: 403 })
+ mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)
+
+ await expect(reconcileExternalEgressSafety()).rejects.toBe(forbidden)
+ })
+ })
+
+ describe('safety-pass duration on the uncertified return path', () => {
+ it('samples a failed outcome when the pass returns without certifying', async () => {
+ const onAuthoritativeRevocationComplete = vi.fn()
+
+ // Losing inventory authority ends the pass without an exception and
+ // without certifying. Operators alert on "the pass never completed", so
+ // this path has to produce a `failed` sample — absence is not a signal.
+ await expect(
+ reconciler.fullReconcile([], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => false,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).resolves.toBeUndefined()
+
+ expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
+ { outcome: 'failed' },
+ expect.any(Number)
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
+ })
+
+ it('keeps the completed sample when a certified pass fails an additive effect afterwards', async () => {
+ const context: ContextCRD = {
+ name: 'default',
+ namespace: 'mcp-server',
+ spec: { contextId: 'default', mcpServers: [] },
+ }
+ const additiveFailure = new Error('additive list unavailable')
+ let certified = false
+ const onAuthoritativeRevocationComplete = vi.fn(() => {
+ certified = true
+ })
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {
+ if (certified) throw additiveFailure
+ return { items: [] }
+ })
+
+ await expect(
+ reconciler.fullReconcile([context], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete,
+ })
+ ).rejects.toThrow(/additive Context NetworkPolicy reconciliations failed/)
+
+ expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
+ { outcome: 'completed' },
+ expect.any(Number)
+ )
+ expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
+ })
})
})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index 8778887cd..70eab2a1a 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -27,6 +27,14 @@ import {
POLICY_TYPE_LABEL,
} from './constants'
import { accumulateHostExactHostEgress } from './externalEgressAccumulator'
+import {
+ confirmAuthoritativeMcpServerAbsence,
+ sameMcpServerDesiredRevision,
+} from './mcpServerSafety'
+import {
+ networkPolicySafetyPassDurationSeconds,
+ networkPolicySafetyPassPoliciesTotal,
+} from './metrics'
import {
ContextCRD,
EgressBinding,
@@ -35,21 +43,181 @@ import {
McpServerCrdStatus,
McpServerResolvedEgressIP,
} from './types'
-import { applyNetworkPolicy, getErrorCode } from './utils'
+import { applyNetworkPolicy, getErrorCode, replaceWithConflictRetry } from './utils'
type JsonPatchOperation = {
- op: 'add' | 'replace' | 'remove'
+ op: 'add' | 'replace' | 'remove' | 'test'
path: string
value?: unknown
}
+type NetworkPolicyMutationOptions = {
+ // Required (not optional): reconcileContext and reconcileExternalEgress are the
+ // two functions that CREATE/RETAIN context-allow (L2) and external-egress ALLOW
+ // (L3) policies, so the authority fence must be supplied at compile time. A
+ // fail-open `?? (() => true)` default here would silently readmit the exact
+ // fail-open class hardened out of bindingPolicyReconciler (required isCurrent).
+ isCurrent: () => boolean
+ onRevoked?: () => void
+ /**
+ * Set only by a caller that reads the returned boolean and withholds
+ * readiness on false. It downgrades a lost delete fence from a throw to a
+ * reported outcome; a caller that discards the boolean must not set it, or a
+ * stale allow survives while the pass is recorded as a success.
+ */
+ honorsLostFence?: boolean
+}
+
+export function sameContextDesiredRevision(expected: ContextCRD, current: ContextCRD): boolean {
+ return (
+ expected.name === current.name &&
+ expected.namespace === current.namespace &&
+ expected.uid === current.uid &&
+ expected.generation === current.generation &&
+ JSON.stringify(canonicalizeNetworkPolicyValue(expected.spec)) ===
+ JSON.stringify(canonicalizeNetworkPolicyValue(current.spec))
+ )
+}
+
+function canonicalizeNetworkPolicyValue(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(canonicalizeNetworkPolicyValue)
+ if (value === null || typeof value !== 'object') return value
+ return Object.fromEntries(
+ Object.entries(value as Record<string, unknown>)
+ .filter(([, entry]) => entry !== undefined)
+ .sort(([left], [right]) => left.localeCompare(right))
+ .map(([key, entry]) => [key, canonicalizeNetworkPolicyValue(entry)])
+ )
+}
+
+function sameNetworkPolicySpec(
+ existing: k8s.V1NetworkPolicy,
+ desired: k8s.V1NetworkPolicy
+): boolean {
+ return (
+ JSON.stringify(canonicalizeNetworkPolicyValue(existing.spec)) ===
+ JSON.stringify(canonicalizeNetworkPolicyValue(desired.spec))
+ )
+}
+
+function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {
+ const labels = policy.metadata?.labels ?? {}
+ return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[POLICY_TYPE_LABEL] === policyType
+}
+
const GROUP = 'clerum.io'
const VERSION = 'v1alpha1'
const PLURAL_MCPSERVERS = 'mcpservers'
+const PLURAL_CONTEXTS = 'contexts'
const CONTEXT_LABEL = 'clerum.io/context'
const EGRESS_CLASS_LABEL = 'clerum.io/egress-class'
const RPC_PROXY_EGRESS_POLICY_TYPE = 'rpc-proxy-egress'
const RPC_PROXY_APP_LABEL = 'rpc-proxy'
+
+type SafetyInventoryLane =
+ | 'context-ingress'
+ | 'context-host-egress'
+ | 'context-rpc-egress'
+ | 'external-egress'
+
+type SafetyInventoryClassification = 'owned' | 'repairable' | 'ambiguous' | 'unrelated'
+
+function policyHasOwnerShape(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {
+ const labels = policy.metadata?.labels ?? {}
+ const hasServerOwner = Boolean(labels[MCPSERVER_LABEL])
+ if (lane === 'external-egress') return hasServerOwner
+ return Boolean(labels[CONTEXT_LABEL]) && hasServerOwner
+}
+
+function policyHasReservedName(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {
+ const name = policy.metadata?.name ?? ''
+ switch (lane) {
+ case 'context-ingress':
+ return name.startsWith('ctx-')
+ case 'context-host-egress':
+ return name.startsWith('ctx-') && name.endsWith('-egress')
+ case 'context-rpc-egress':
+ return name.startsWith('rpc-egress-')
+ case 'external-egress':
+ return name.startsWith('ext-egress-')
+ }
+}
+
+function policyNameBelongsToDifferentLane(
+ policy: k8s.V1NetworkPolicy,
+ lane: SafetyInventoryLane
+): boolean {
+ const name = policy.metadata?.name ?? ''
+ if (lane === 'context-ingress') return name.startsWith('ext-egress-')
+ if (lane === 'external-egress') return name.startsWith('ctx-')
+ return false
+}
+
+function expectedPolicyType(lane: SafetyInventoryLane): string {
+ return lane === 'context-rpc-egress'
+ ? RPC_PROXY_EGRESS_POLICY_TYPE
+ : lane === 'external-egress'
+ ? EXTERNAL_EGRESS_POLICY_TYPE
+ : 'context-allow'
+}
+
+/**
+ * Classify broad namespace inventory without taking ownership of unrelated
+ * policies. A single missing HCC marker is recoverable only when owner labels
+ * corroborate the lane. Reserved HCC names and conflicting ownership markers
+ * are never adopted implicitly.
+ */
+function classifySafetyInventoryPolicy(
+ policy: k8s.V1NetworkPolicy,
+ lane: SafetyInventoryLane
+): SafetyInventoryClassification {
+ const labels = policy.metadata?.labels ?? {}
+ const managedBy = labels[MANAGED_BY_LABEL]
+ const policyType = labels[POLICY_TYPE_LABEL]
+ const expectedType = expectedPolicyType(lane)
+ const managedMatches = managedBy === MANAGED_BY_VALUE
+ const typeMatches = policyType === expectedType
+ const managedMissing = managedBy === undefined
+ const typeMissing = policyType === undefined
+ const reservedName = policyHasReservedName(policy, lane)
+ const ownerShape = policyHasOwnerShape(policy, lane)
+
+ if (managedMatches && typeMatches) return 'owned'
+
+ const oneMarkerMissing = (managedMatches && typeMissing) || (managedMissing && typeMatches)
+ if (managedMissing && typeMatches && ownerShape) {
+ return reservedName ? 'repairable' : 'ambiguous'
+ }
+ if (managedMatches && typeMissing && ownerShape) {
+ if (reservedName) return 'repairable'
+ if (policyNameBelongsToDifferentLane(policy, lane)) return 'unrelated'
+ return 'ambiguous'
+ }
+
+ const knownOtherHccType =
+ policyType !== undefined &&
+ ['context-allow', RPC_PROXY_EGRESS_POLICY_TYPE, EXTERNAL_EGRESS_POLICY_TYPE].includes(
+ policyType
+ ) &&
+ !typeMatches
+ const hasRawConflictingMarker =
+ (managedBy !== undefined && !managedMatches) || (policyType !== undefined && !typeMatches)
+ const hasConflictingMarker =
+ (managedBy !== undefined && !managedMatches) ||
+ (policyType !== undefined && !typeMatches && !knownOtherHccType)
+ if (
+ reservedName &&
+ ((managedMissing && typeMissing) || hasRawConflictingMarker || oneMarkerMissing)
+ ) {
+ return 'ambiguous'
+ }
+ if (ownerShape && hasConflictingMarker) {
+ return 'ambiguous'
+ }
+
+ return 'unrelated'
+}
+
export const PUBLIC_EGRESS_EXCEPT_CIDRS = [
'0.0.0.0/8',
'10.0.0.0/8',
@@ -198,6 +366,45 @@ function k8sApiEgressPodSelector(namespace: string): k8s.V1LabelSelector {
return { matchLabels: { 'clerum.io/k8s-api-egress': 'true' } }
}
+export interface NetworkPolicyFullReconcileOptions {
+ serverInventoryComplete?: boolean
+ ensureDefaults?: boolean
+ /**
+ * Dynamic authority checks are intentionally callbacks: watch authority can
+ * be lost after the startup snapshot but before an orphan delete begins.
+ * Omitted callbacks preserve the historical authoritative-by-default API.
+ */
+ contextInventoryAuthoritative?: () => boolean
+ serverInventoryAuthoritative?: () => boolean
+ runContextEffect?: (contextId: string, work: () => Promise<void>) => Promise<void>
+ runServerEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
+ resolveCurrentContext?: (name: string) => ContextCRD | undefined
+ resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined
+ resolveCurrentServer?: (name: string) => McpServerCRD | undefined
+ /**
+ * Monotonic desired-state revisions fence ordinary watch events that arrive
+ * after an owner has completed its safety turn. Watch generations remain the
+ * separate authority fence for LIST -> WATCH recovery.
+ */
+ contextDesiredRevision?: () => number
+ serverDesiredRevision?: () => number
+ /**
+ * Called only after every authoritative orphan-allow revocation lane has
+ * completed. Additive Context/McpServer convergence intentionally happens
+ * afterwards and must not delay this safety boundary.
+ */
+ onAuthoritativeRevocationComplete?: () => void
+ /**
+ * Fired at the moment each external-egress allow owned by an McpServer is
+ * revoked (or replaced) during the authoritative safety pass — before any
+ * isCurrent abort or inventory-changed throw can unwind the pass. The owner
+ * uses it to queue an additive recreation for the affected server; queuing in
+ * a pass epilogue instead would reproduce exactly the abort gap this hook
+ * closes.
+ */
+ onExternalEgressRevoked?: (server: McpServerCRD) => void
+}
+
export class NetworkPolicyReconciler {
private networkingApi: k8s.NetworkingV1Api
private customApi: k8s.CustomObjectsApi
@@ -205,6 +412,22 @@ export class NetworkPolicyReconciler {
/** Reference to the MCP server cache so we can look up ports. */
private serverCache: Map<string, McpServerCRD>
+ /**
+ * Set whenever an authoritative safety pass ends without certifying, cleared
+ * as soon as one certifies again. Only that pass enumerates the bounded
+ * namespaces and classifies every object it finds; the scoped delta lane sees
+ * nothing but its own label selectors. A delta therefore cannot vouch for a
+ * namespace whose classified inventory is still stuck on an unresolved
+ * policy, and must not certify readiness on its behalf.
+ *
+ * Initialised fail-closed (true): hasCertifiedSafetyInventory() reports false
+ * until the first authoritative pass actually certifies, so certification
+ * cannot be implied before any pass has run. This does not depend on the
+ * readiness gate's generation-equality check for safety — it removes that
+ * coupling rather than relying on it.
+ */
+ private safetyPassLeftUncertified = true
+
// H2 (issue #299): smallest DNS TTL (ms) observed across external-egress
// resolutions. The resync loop advances to <= this/2 so a rotating low-TTL host
// is sampled every rotation. Ratchets down and never recovers until restart —
@@ -303,7 +526,7 @@ export class NetworkPolicyReconciler {
},
}
- await applyNetworkPolicy(this.networkingApi, name, ns, policy, '[NetPol]')
+ await this.applyFixedPolicy(name, ns, policy, 'default-deny')
}
}
@@ -377,7 +600,7 @@ export class NetworkPolicyReconciler {
egress,
},
}
- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')
+ await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)
}
/** Allow egress to host-context-controller API gateway in control-plane namespace. */
@@ -414,7 +637,7 @@ export class NetworkPolicyReconciler {
],
},
}
- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')
+ await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)
}
/**
@@ -459,7 +682,7 @@ export class NetworkPolicyReconciler {
],
},
}
- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')
+ await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)
}
/**
@@ -502,46 +725,26 @@ export class NetworkPolicyReconciler {
},
}
- await this.applyPolicy(name, policy)
+ await this.applyFixedPolicy(name, config.namespace, policy, 'allow-api')
}
// ─── Context-Based Policies ──────────────────────────────────────────
- /**
- * Reconcile NetworkPolicies for a Context CRD.
- * Creates one policy per allowed MCP server.
- */
- async reconcileContext(context: ContextCRD): Promise<void> {
+ private buildContextAllowPolicies(
+ context: ContextCRD,
+ server: McpServerCRD
+ ): {
+ ingress: k8s.V1NetworkPolicy
+ hostEgress: k8s.V1NetworkPolicy
+ rpcProxyEgress: k8s.V1NetworkPolicy
+ } {
const contextId = context.spec.contextId
- const allowedServers = context.spec.mcpServers || []
-
- console.log(
- `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`
- )
+ const serverName = server.name
+ const name = this.policyName(contextId, serverName)
+ const port = server.spec.transport.port || 3000
- // Build desired policy names
- const desiredNames = new Set(
- allowedServers.map(serverName => this.policyName(contextId, serverName))
- )
-
- // Get existing policies for this context
- const existingPolicies = await this.listPoliciesForContext(contextId)
-
- // Create or update policies for each allowed server
- for (const serverName of allowedServers) {
- const server = this.serverCache.get(serverName)
- if (!server) {
- console.warn(
- `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`
- )
- continue
- }
-
- const name = this.policyName(contextId, serverName)
- const port = server.spec.transport.port || 3000
-
- // L2 ingress: allow scoped mcp-host/rpc-proxy access to this MCP server.
- const ingressPolicy: k8s.V1NetworkPolicy = {
+ return {
+ ingress: {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
@@ -590,7 +793,6 @@ export class NetworkPolicyReconciler {
},
},
{
- // MCP Proxy ingress: allow proxy to reach MCP servers
namespaceSelector: {
matchLabels: {
'kubernetes.io/metadata.name': config.namespace,
@@ -607,18 +809,12 @@ export class NetworkPolicyReconciler {
},
],
},
- }
-
- await this.applyPolicy(name, ingressPolicy)
-
- // L2 egress counterpart: allow traffic FROM mcp-host pods TO this MCP server
- // Without this, L0 egress deny-all blocks agents from reaching MCP servers.
- const egressName = `${name}-egress`
- const egressPolicy: k8s.V1NetworkPolicy = {
+ },
+ hostEgress: {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
- name: egressName,
+ name: `${name}-egress`,
namespace: config.hostNamespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
@@ -655,78 +851,322 @@ export class NetworkPolicyReconciler {
},
],
},
+ },
+ rpcProxyEgress: this.buildRpcProxyEgressPolicy(contextId, serverName, port),
+ }
+ }
+
+ /**
+ * Reconcile NetworkPolicies for a Context CRD.
+ * Creates one policy per allowed MCP server.
+ *
+ * Returns whether this Context's stale-allow revocation ran to completion.
+ * Callers that certify readiness from a scoped delta MUST honour it: the pass
+ * aborts as soon as its authority fence breaks, and an aborted pass leaves
+ * stale allows live.
+ */
+ async reconcileContext(
+ context: ContextCRD,
+ options: NetworkPolicyMutationOptions
+ ): Promise<boolean> {
+ const callerIsCurrent = options.isCurrent
+ if (!callerIsCurrent()) return false
+ const contextId = context.spec.contextId
+ const allowedServers = context.spec.mcpServers || []
+ const selectedServers = new Map(
+ allowedServers.map(serverName => [serverName, this.serverCache.get(serverName)])
+ )
+ const isCurrent = (): boolean =>
+ callerIsCurrent() &&
+ allowedServers.every(serverName => {
+ const selected = selectedServers.get(serverName)
+ const current = this.serverCache.get(serverName)
+ if (!selected || !current) return selected === current
+ return sameMcpServerDesiredRevision(selected, current)
+ })
+ if (!isCurrent()) return false
+
+ console.log(
+ `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`
+ )
+
+ // Create or update policies for each allowed server.
+ for (const serverName of allowedServers) {
+ if (!isCurrent()) return false
+ const server = this.serverCache.get(serverName)
+ if (!server) {
+ console.warn(
+ `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`
+ )
+ continue
}
- await applyNetworkPolicy(
- this.networkingApi,
- egressName,
+ const desired = this.buildContextAllowPolicies(context, server)
+ const ingressName = desired.ingress.metadata!.name!
+ await this.applyContextIngressPolicy(ingressName, desired.ingress, isCurrent)
+ if (!isCurrent()) return false
+
+ const hostEgressName = desired.hostEgress.metadata!.name!
+ await this.applyOwnedPolicy(
+ hostEgressName,
config.hostNamespace,
- egressPolicy,
- '[NetPol]'
+ desired.hostEgress,
+ 'context-host-egress',
+ isCurrent
+ )
+ if (!isCurrent()) return false
+
+ const rpcProxyEgressName = desired.rpcProxyEgress.metadata!.name!
+ await this.applyOwnedPolicy(
+ rpcProxyEgressName,
+ config.rpcProxyNamespace,
+ desired.rpcProxyEgress,
+ 'context-rpc-egress',
+ isCurrent
)
+ }
+
+ // Re-LIST every lane after writes so a same-name policy that was just
+ // replaced is compared in its new form and cannot delete itself.
+ return await this.revokeStaleContextAllows(
+ context,
+ isCurrent,
+ undefined,
+ undefined,
+ options.honorsLostFence === true
+ )
+ }
- // L2 rpc-proxy egress: allow rpc-proxy pods to reach this MCP server
- await this.ensureRpcProxyEgress(contextId, serverName, port)
+ /**
+ * Whether the last authoritative safety pass certified its namespace-wide
+ * inventory. Only that pass enumerates the bounded namespaces and classifies
+ * every object it finds; a scoped delta sees nothing but its own label
+ * selectors, so it cannot vouch for a namespace whose classified inventory is
+ * still stuck. The readiness lane consults this before letting a delta
+ * replace the safety certificate.
+ */
+ hasCertifiedSafetyInventory(): boolean {
+ return !this.safetyPassLeftUncertified
+ }
+
+ private async revokeStaleContextAllows(
+ context: ContextCRD,
+ isCurrent: () => boolean,
+ listedPolicies?: {
+ context?: k8s.V1NetworkPolicy[]
+ hostEgress?: k8s.V1NetworkPolicy[]
+ rpcProxyEgress?: k8s.V1NetworkPolicy[]
+ },
+ onRevoked?: () => void,
+ callerHonorsLostFence = false
+ ): Promise<boolean> {
+ if (!isCurrent()) return false
+ const safetySnapshotProvided = listedPolicies !== undefined
+ // A delete whose uid/resourceVersion preconditions were rejected revoked
+ // nothing. Reporting it instead of throwing is only safe for a caller that
+ // reads the returned boolean: the authoritative snapshot lane aborts its
+ // pass on it, and the scoped delta lane declines to certify. Every other
+ // caller discards the boolean, so for them a swallowed 409 would leave a
+ // stale allow live while the pass is recorded as a success — they keep the
+ // loud default and let the failure reach the retry policy.
+ const lostFenceOutcome =
+ safetySnapshotProvided || callerHonorsLostFence ? ('report' as const) : ('throw' as const)
+ let lostDeleteFence = false
+ /**
+ * Revoke one orphaned policy in any lane. Returns false only when the
+ * caller must abort the pass outright, which is what the authoritative
+ * safety-snapshot lane does with a lost delete fence; the scoped lane
+ * records it and keeps going so the rest of its work still lands.
+ */
+ const revokeOrphanedPolicy = async (
+ namespace: string,
+ existing: k8s.V1NetworkPolicy
+ ): Promise<boolean> => {
+ const deleted = await this.deleteSafetyPolicySnapshot(
+ namespace,
+ existing,
+ isCurrent,
+ undefined,
+ lostFenceOutcome
+ )
+ if (deleted) return true
+ if (safetySnapshotProvided) {
+ // The authoritative pass is condemned from here: it will unwind and
+ // throw. Losing the fence is the only doom cause that does not bump a
+ // watch generation, so without this the certification machinery stays
+ // green for the whole unwind and a concurrent scoped delta can certify
+ // readiness over the allow this pass failed to revoke. Record it the
+ // moment it is known rather than when the pass finally reports.
+ this.safetyPassLeftUncertified = true
+ return false
+ }
+ lostDeleteFence = true
+ return true
+ }
+ const contextId = context.spec.contextId
+ const allowedServers = context.spec.mcpServers || []
+ const desiredIngress = new Map<string, k8s.V1NetworkPolicy>()
+ const desiredHostEgress = new Map<string, k8s.V1NetworkPolicy>()
+ const desiredRpcProxyEgress = new Map<string, k8s.V1NetworkPolicy>()
+ for (const serverName of allowedServers) {
+ const server = this.serverCache.get(serverName)
+ if (!server) continue
+ const desired = this.buildContextAllowPolicies(context, server)
+ desiredIngress.set(desired.ingress.metadata!.name!, desired.ingress)
+ desiredHostEgress.set(desired.hostEgress.metadata!.name!, desired.hostEgress)
+ desiredRpcProxyEgress.set(desired.rpcProxyEgress.metadata!.name!, desired.rpcProxyEgress)
}
- // Delete policies for servers no longer in the context
+ const existingPolicies =
+ listedPolicies?.context ?? (await this.listPoliciesForContext(contextId))
+ if (!isCurrent()) return false
for (const existing of existingPolicies) {
+ if (!isCurrent()) return false
const existingName = existing.metadata?.name || ''
- if (!desiredNames.has(existingName)) {
+ const desired = desiredIngress.get(existingName)
+ if (!desired) {
console.log(`[NetPol] Deleting orphaned policy "${existingName}"`)
- await this.deletePolicy(existingName)
+ if (!(await revokeOrphanedPolicy(config.namespace, existing))) return false
+ onRevoked?.()
+ } else if (
+ !sameNetworkPolicySpec(existing, desired) ||
+ !hasExpectedPolicyOwnership(existing, 'context-allow')
+ ) {
+ console.log(`[NetPol] Replacing stale same-name policy "${existingName}"`)
+ if (safetySnapshotProvided) {
+ if (
+ !(await this.replaceSafetyPolicySnapshot(
+ config.namespace,
+ existing,
+ desired,
+ 'context-ingress',
+ isCurrent
+ ))
+ ) {
+ return false
+ }
+ } else {
+ await this.replaceSafetyPolicySnapshot(
+ config.namespace,
+ existing,
+ desired,
+ 'context-ingress',
+ isCurrent
+ )
+ }
+ onRevoked?.()
}
}
- // Delete orphaned L2 egress counterparts in mcp-host namespace
- const existingEgressPolicies = await this.listEgressPoliciesForContext(contextId)
- const desiredEgressNames = new Set(
- allowedServers.map(serverName => `${this.policyName(contextId, serverName)}-egress`)
- )
+ const existingEgressPolicies =
+ listedPolicies?.hostEgress ?? (await this.listEgressPoliciesForContext(contextId))
+ if (!isCurrent()) return false
for (const existing of existingEgressPolicies) {
+ if (!isCurrent()) return false
const existingName = existing.metadata?.name || ''
- if (!desiredEgressNames.has(existingName)) {
+ const desired = desiredHostEgress.get(existingName)
+ if (!desired) {
console.log(`[NetPol] Deleting orphaned L2 egress policy "${existingName}"`)
- await this.deleteEgressPolicy(existingName)
+ if (!(await revokeOrphanedPolicy(config.hostNamespace, existing))) return false
+ onRevoked?.()
+ } else if (
+ !sameNetworkPolicySpec(existing, desired) ||
+ !hasExpectedPolicyOwnership(existing, 'context-allow')
+ ) {
+ console.log(`[NetPol] Replacing stale same-name L2 egress policy "${existingName}"`)
+ if (safetySnapshotProvided) {
+ if (
+ !(await this.replaceSafetyPolicySnapshot(
+ config.hostNamespace,
+ existing,
+ desired,
+ 'context-host-egress',
+ isCurrent
+ ))
+ ) {
+ return false
+ }
+ } else {
+ await this.replaceSafetyPolicySnapshot(
+ config.hostNamespace,
+ existing,
+ desired,
+ 'context-host-egress',
+ isCurrent
+ )
+ }
+ onRevoked?.()
}
}
- // Delete orphaned rpc-proxy egress counterparts. Keeping these after a
- // server is removed from a Context would preserve an old in-cluster route.
- const existingRpcProxyPolicies = await this.listRpcProxyEgressPoliciesForContext(contextId)
- const desiredRpcProxyNames = new Set(
- allowedServers.map(serverName => `rpc-egress-${contextId}-${serverName}`)
- )
+ const existingRpcProxyPolicies =
+ listedPolicies?.rpcProxyEgress ?? (await this.listRpcProxyEgressPoliciesForContext(contextId))
+ if (!isCurrent()) return false
for (const existing of existingRpcProxyPolicies) {
+ if (!isCurrent()) return false
const existingName = existing.metadata?.name || ''
- if (!desiredRpcProxyNames.has(existingName)) {
+ const desired = desiredRpcProxyEgress.get(existingName)
+ if (!desired) {
console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${existingName}"`)
- await this.deletePolicyInNamespace(config.rpcProxyNamespace, existingName)
+ if (!(await revokeOrphanedPolicy(config.rpcProxyNamespace, existing))) return false
+ onRevoked?.()
+ } else if (
+ !sameNetworkPolicySpec(existing, desired) ||
+ !hasExpectedPolicyOwnership(existing, RPC_PROXY_EGRESS_POLICY_TYPE)
+ ) {
+ console.log(`[NetPol] Replacing stale same-name rpc-proxy egress policy "${existingName}"`)
+ if (safetySnapshotProvided) {
+ if (
+ !(await this.replaceSafetyPolicySnapshot(
+ config.rpcProxyNamespace,
+ existing,
+ desired,
+ 'context-rpc-egress',
+ isCurrent
+ ))
+ ) {
+ return false
+ }
+ } else {
+ await this.replaceSafetyPolicySnapshot(
+ config.rpcProxyNamespace,
+ existing,
+ desired,
+ 'context-rpc-egress',
+ isCurrent
+ )
+ }
+ onRevoked?.()
}
}
+
+ return isCurrent() && !lostDeleteFence
}
/**
* Delete all NetworkPolicies for a deleted Context.
*/
- async reconcileDeleteContext(contextId: string): Promise<void> {
+ async reconcileDeleteContext(
+ contextId: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
console.log(`[NetPol] Context "${contextId}" deleted — removing all policies`)
const policies = await this.listPoliciesForContext(contextId)
for (const policy of policies) {
- const name = policy.metadata?.name || ''
- await this.deletePolicy(name)
+ if (deleteAllowed && !(await deleteAllowed())) return
+ await this.deleteSafetyPolicySnapshot(config.namespace, policy)
}
// Also delete L2 egress counterparts in mcp-host namespace
const egressPolicies = await this.listEgressPoliciesForContext(contextId)
for (const policy of egressPolicies) {
- const name = policy.metadata?.name || ''
- await this.deleteEgressPolicy(name)
+ if (deleteAllowed && !(await deleteAllowed())) return
+ await this.deleteSafetyPolicySnapshot(config.hostNamespace, policy)
}
// Also delete L2 rpc-proxy egress policies
- await this.deleteRpcProxyPoliciesForContext(contextId)
+ await this.deleteRpcProxyPoliciesForContext(contextId, deleteAllowed)
}
/**
@@ -735,80 +1175,742 @@ export class NetworkPolicyReconciler {
async fullReconcile(
contexts: ContextCRD[],
servers: McpServerCRD[] = [],
- options: { serverInventoryComplete?: boolean; ensureDefaults?: boolean } = {}
+ options: NetworkPolicyFullReconcileOptions = {}
+ ): Promise<void> {
+ const onAuthoritativeRevocationComplete = options.onAuthoritativeRevocationComplete
+ if (!onAuthoritativeRevocationComplete) {
+ await this.fullReconcileInternal(contexts, servers, options)
+ return
+ }
+
+ const safetyPassStartedAt = performance.now()
+ let safetyPassOutcomeRecorded = false
+ /**
+ * Settles both observable results of this pass: the duration sample and the
+ * cross-lane fence the scoped delta reads. First outcome wins for both — a
+ * pass that certified and only then failed an additive effect still revoked
+ * every stale allow its inventory implied.
+ */
+ const recordSafetyPassOutcome = (outcome: 'completed' | 'failed'): void => {
+ // The fence is an assertion about live allows, so it only ever moves
+ // toward unsafe. The additive phase re-runs reconcileContext, which
+ // carries a second revocation lane over a fresh LIST; losing the fence
+ // there happens after this pass certified, and first-outcome-wins would
+ // otherwise leave the gate green over an allow that was not revoked.
+ if (outcome === 'failed') this.safetyPassLeftUncertified = true
+ // The duration sample describes the pass's primary outcome, so it stays
+ // first-wins: a certified pass that later fails an additive effect did
+ // complete its authoritative revocation.
+ if (safetyPassOutcomeRecorded) return
+ safetyPassOutcomeRecorded = true
+ this.safetyPassLeftUncertified = outcome === 'failed'
+ networkPolicySafetyPassDurationSeconds.observe(
+ { outcome },
+ (performance.now() - safetyPassStartedAt) / 1_000
+ )
+ }
+
+ try {
+ await this.fullReconcileInternal(contexts, servers, {
+ ...options,
+ onAuthoritativeRevocationComplete: () => {
+ onAuthoritativeRevocationComplete()
+ recordSafetyPassOutcome('completed')
+ },
+ })
+ // The pass returned without certifying: no exception, no certificate.
+ // Operators alert on this exact condition, so it must produce a sample
+ // rather than the absence of one. Stated as a condition rather than
+ // leaning on first-outcome-wins to swallow it, because the fence below is
+ // deliberately not first-wins.
+ if (!safetyPassOutcomeRecorded) recordSafetyPassOutcome('failed')
+ } catch (error) {
+ recordSafetyPassOutcome('failed')
+ throw error
+ }
+ }
+
+ private async fullReconcileInternal(
+ contexts: ContextCRD[],
+ servers: McpServerCRD[] = [],
+ options: NetworkPolicyFullReconcileOptions = {}
): Promise<void> {
console.log(`[NetPol] Running full reconciliation for ${contexts.length} Context(s)`)
+ // This is intentionally scoped to callers that use the authoritative
+ // revocation callback. Ordinary additive reconciles must not be presented
+ // as readiness safety passes.
+ const measureSafetyPass = options.onAuthoritativeRevocationComplete !== undefined
+ let safetyPassListedPolicies = 0
+ let safetyPassRevokedPolicies = 0
+ const recordSafetyPassRevocation = (): void => {
+ if (measureSafetyPass) safetyPassRevokedPolicies += 1
+ }
+
+ // The revision fence starts at invocation, before any asynchronous
+ // default-policy setup can interleave a watch update with this snapshot.
+ const selectedContextDesiredRevision = options.contextDesiredRevision?.()
+ const selectedServerDesiredRevision = options.serverDesiredRevision?.()
+ const runContextEffect =
+ options.runContextEffect ?? ((_contextId: string, work: () => Promise<void>) => work())
+ const runServerEffect =
+ options.runServerEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
+ const contextInventoryIsCurrent = (): boolean =>
+ (options.contextInventoryAuthoritative?.() ?? true) &&
+ (options.serverInventoryAuthoritative?.() ?? true)
+
if (options.ensureDefaults !== false) {
await this.ensureDefaultPolicies()
}
- // Reconcile each context
- for (const context of contexts) {
- await this.reconcileContext(context)
+ // Authority is monotonic within this pass. Once a watch loses authority,
+ // this inventory snapshot cannot safely become authoritative again even if
+ // the callback flips back before the next delete.
+ let contextAuthorityLost = false
+ let contextAuthorityWarningLogged = false
+ const contextCleanupAuthoritative = (): boolean => {
+ if (
+ !contextAuthorityLost &&
+ options.contextInventoryAuthoritative &&
+ !options.contextInventoryAuthoritative()
+ ) {
+ contextAuthorityLost = true
+ }
+ if (contextAuthorityLost && !contextAuthorityWarningLogged) {
+ contextAuthorityWarningLogged = true
+ console.warn(
+ '[NetPol] Skipping remaining Context policy orphan cleanup because Context inventory authority was lost'
+ )
+ }
+ return !contextAuthorityLost
}
- // Reconcile external egress for all existing servers on startup so a
- // controller restart converges pre-existing McpServer CRDs too.
- for (const server of servers) {
- await this.reconcileExternalEgress(server)
+ let serverAuthorityLost = false
+ let serverAuthorityWarningLogged = false
+ const serverCleanupAuthoritative = (): boolean => {
+ if (
+ !serverAuthorityLost &&
+ options.serverInventoryAuthoritative &&
+ !options.serverInventoryAuthoritative()
+ ) {
+ serverAuthorityLost = true
+ }
+ if (serverAuthorityLost && !serverAuthorityWarningLogged) {
+ serverAuthorityWarningLogged = true
+ console.warn(
+ '[NetPol] Skipping remaining external egress orphan cleanup because McpServer inventory authority was lost'
+ )
+ }
+ return !serverAuthorityLost
}
- // Delete policies for contexts that no longer exist
const desiredContextIds = new Set(contexts.map(c => c.spec.contextId))
- const allContextPolicies = await this.listAllContextPolicies()
-
- for (const policy of allContextPolicies) {
- const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]
- if (contextId && !desiredContextIds.has(contextId)) {
- const name = policy.metadata?.name || ''
- console.log(
- `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`
- )
- await this.deletePolicy(name)
+ const cleanupOrphanedContextPolicies = async (
+ policies: k8s.V1NetworkPolicy[],
+ deletePolicy: (policy: k8s.V1NetworkPolicy) => Promise<boolean>,
+ describePolicy: (name: string, contextId: string) => string
+ ): Promise<void> => {
+ if (!contextCleanupAuthoritative()) return
+ for (const policy of policies) {
+ const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]
+ if (!contextId) {
+ if (!contextCleanupAuthoritative()) return
+ const name = policy.metadata?.name
+ if (!name) {
+ throw new Error('HCC-managed Context NetworkPolicy is missing its name and owner label')
+ }
+ console.warn(
+ `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${CONTEXT_LABEL}`
+ )
+ if (await deletePolicy(policy)) recordSafetyPassRevocation()
+ continue
+ }
+ if (!desiredContextIds.has(contextId)) {
+ const name = policy.metadata?.name || ''
+ await runContextEffect(contextId, async () => {
+ if (
+ !(await this.contextOrphanDeleteAllowed(
+ contextId,
+ contextCleanupAuthoritative,
+ options.resolveCurrentContextById
+ ))
+ ) {
+ return
+ }
+ console.log(describePolicy(name, contextId))
+ if (await deletePolicy(policy)) recordSafetyPassRevocation()
+ })
+ }
}
}
- // Also clean up orphaned rpc-proxy egress policies
- const allRpcProxyPolicies = await this.listAllRpcProxyEgressPolicies()
- for (const policy of allRpcProxyPolicies) {
- const ctxId = policy.metadata?.labels?.[CONTEXT_LABEL]
- if (ctxId && !desiredContextIds.has(ctxId)) {
- const pName = policy.metadata?.name || ''
- console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${pName}"`)
- await this.deletePolicyInNamespace(config.rpcProxyNamespace, pName)
- }
+ let allContextPolicies = contextCleanupAuthoritative()
+ ? await this.listAllContextPolicies()
+ : []
+ let allContextEgressPolicies = contextCleanupAuthoritative()
+ ? await this.listAllContextEgressPolicies()
+ : []
+ let allRpcProxyEgressPolicies = contextCleanupAuthoritative()
+ ? await this.listAllRpcProxyEgressPolicies()
+ : []
+ let allExternalPolicies: k8s.V1NetworkPolicy[] = []
+ if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {
+ allExternalPolicies = await this.listAllExternalEgressPolicies()
+ }
+
+ // Label selectors alone cannot prove safety: removing either ownership
+ // marker would hide an active allow from the selected LIST. Audit the
+ // bounded namespaces, then merge only owned or conservatively repairable
+ // candidates into their lane. Ambiguous reserved names fail the pass.
+ const needsMcpServerBroadInventory =
+ contextCleanupAuthoritative() ||
+ (options.serverInventoryComplete !== false && serverCleanupAuthoritative())
+ const broadMcpServerPolicies = needsMcpServerBroadInventory
+ ? await this.listNamespacePoliciesForSafety(config.namespace)
+ : []
+ const broadHostPolicies = contextCleanupAuthoritative()
+ ? await this.listNamespacePoliciesForSafety(config.hostNamespace)
+ : []
+ const broadRpcProxyPolicies = contextCleanupAuthoritative()
+ ? await this.listNamespacePoliciesForSafety(config.rpcProxyNamespace)
+ : []
+
+ if (contextCleanupAuthoritative()) {
+ allContextPolicies = this.mergeSafetyLaneInventory(
+ allContextPolicies,
+ broadMcpServerPolicies,
+ 'context-ingress'
+ )
+ allContextEgressPolicies = this.mergeSafetyLaneInventory(
+ allContextEgressPolicies,
+ broadHostPolicies,
+ 'context-host-egress'
+ )
+ allRpcProxyEgressPolicies = this.mergeSafetyLaneInventory(
+ allRpcProxyEgressPolicies,
+ broadRpcProxyPolicies,
+ 'context-rpc-egress'
+ )
+ }
+ if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {
+ allExternalPolicies = this.mergeSafetyLaneInventory(
+ allExternalPolicies,
+ broadMcpServerPolicies,
+ 'external-egress'
+ )
+ }
+ safetyPassListedPolicies +=
+ allContextPolicies.length +
+ allContextEgressPolicies.length +
+ allRpcProxyEgressPolicies.length +
+ allExternalPolicies.length
+
+ // Delete orphaned Context policies across every L2 lane. Each lane uses
+ // the canonical contextId effect key and rechecks authority plus live CRD
+ // presence immediately before its delete.
+ await cleanupOrphanedContextPolicies(
+ allContextPolicies,
+ policy =>
+ this.deleteSafetyPolicySnapshot(config.namespace, policy, contextCleanupAuthoritative),
+ (name, contextId) =>
+ `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`
+ )
+ await cleanupOrphanedContextPolicies(
+ allContextEgressPolicies,
+ policy =>
+ this.deleteSafetyPolicySnapshot(config.hostNamespace, policy, contextCleanupAuthoritative),
+ name => `[NetPol] Deleting orphaned L2 egress policy "${name}"`
+ )
+ await cleanupOrphanedContextPolicies(
+ allRpcProxyEgressPolicies,
+ policy =>
+ this.deleteSafetyPolicySnapshot(
+ config.rpcProxyNamespace,
+ policy,
+ contextCleanupAuthoritative
+ ),
+ name => `[NetPol] Deleting orphaned rpc-proxy egress policy "${name}"`
+ )
+
+ let liveDesiredRevisionChanged = false
+ for (const context of contexts) {
+ const contextId = context.spec.contextId
+ await runContextEffect(contextId, async () => {
+ const current = options.resolveCurrentContext
+ ? options.resolveCurrentContext(context.name)
+ : context
+ if (!current || current.spec.contextId !== contextId) {
+ liveDesiredRevisionChanged = true
+ return
+ }
+ const contextEffectIsCurrent = (): boolean => {
+ if (!contextCleanupAuthoritative() || !serverCleanupAuthoritative()) return false
+ if (!options.resolveCurrentContext) return true
+ const latest = options.resolveCurrentContext(current.name)
+ return latest !== undefined && sameContextDesiredRevision(current, latest)
+ }
+ const forContext = (policy: k8s.V1NetworkPolicy): boolean =>
+ policy.metadata?.labels?.[CONTEXT_LABEL] === contextId
+ const completed = await this.revokeStaleContextAllows(
+ current,
+ contextEffectIsCurrent,
+ {
+ context: allContextPolicies.filter(forContext),
+ hostEgress: allContextEgressPolicies.filter(forContext),
+ rpcProxyEgress: allRpcProxyEgressPolicies.filter(forContext),
+ },
+ recordSafetyPassRevocation
+ )
+ if (!completed && contextCleanupAuthoritative() && serverCleanupAuthoritative()) {
+ liveDesiredRevisionChanged = true
+ }
+ })
}
// Clean up external egress policies for servers that no longer exist.
- if (options.serverInventoryComplete !== false) {
+ if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {
const desiredServerNames = new Set(servers.map(s => s.name))
- const allExternalPolicies = await this.listAllExternalEgressPolicies()
for (const policy of allExternalPolicies) {
const serverName = policy.metadata?.labels?.[MCPSERVER_LABEL]
- if (serverName && !desiredServerNames.has(serverName)) {
+ if (!serverName) {
+ if (!serverCleanupAuthoritative()) break
+ const name = policy.metadata?.name
+ if (!name) {
+ throw new Error(
+ 'HCC-managed external egress NetworkPolicy is missing its name and owner label'
+ )
+ }
+ console.warn(
+ `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${MCPSERVER_LABEL}`
+ )
+ if (
+ await this.deleteSafetyPolicySnapshot(
+ config.namespace,
+ policy,
+ serverCleanupAuthoritative
+ )
+ ) {
+ recordSafetyPassRevocation()
+ }
+ continue
+ }
+ if (!desiredServerNames.has(serverName)) {
const name = policy.metadata?.name || ''
- console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)
- await this.deletePolicyInNamespace(config.namespace, name)
+ await runServerEffect(serverName, async () => {
+ if (
+ !(await confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: serverCleanupAuthoritative,
+ resolveCurrent: () => options.resolveCurrentServer?.(serverName),
+ readCurrent: () =>
+ this.customApi.getNamespacedCustomObject({
+ group: GROUP,
+ version: VERSION,
+ namespace: config.namespace,
+ plural: PLURAL_MCPSERVERS,
+ name: serverName,
+ }),
+ }))
+ ) {
+ return
+ }
+ console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)
+ if (
+ await this.deleteSafetyPolicySnapshot(
+ config.namespace,
+ policy,
+ serverCleanupAuthoritative
+ )
+ ) {
+ recordSafetyPassRevocation()
+ }
+ })
}
}
- } else {
+ } else if (options.serverInventoryComplete === false) {
console.warn(
'[NetPol] Skipping external egress orphan cleanup because server inventory is incomplete'
)
}
+ if (options.serverInventoryComplete !== false) {
+ for (const server of servers) {
+ await runServerEffect(server.name, async () => {
+ const current = options.resolveCurrentServer
+ ? options.resolveCurrentServer(server.name)
+ : server
+ if (!current) {
+ liveDesiredRevisionChanged = true
+ return
+ }
+ const serverEffectIsCurrent = (): boolean => {
+ if (!serverCleanupAuthoritative()) return false
+ if (!options.resolveCurrentServer) return true
+ const latest = options.resolveCurrentServer(current.name)
+ return latest !== undefined && sameMcpServerDesiredRevision(current, latest)
+ }
+ const completed = await this.reconcileExternalEgressSafety(
+ current,
+ allExternalPolicies.filter(
+ policy => policy.metadata?.labels?.[MCPSERVER_LABEL] === server.name
+ ),
+ serverEffectIsCurrent,
+ () => {
+ recordSafetyPassRevocation()
+ options.onExternalEgressRevoked?.(current)
+ }
+ )
+ if (!completed && serverCleanupAuthoritative()) {
+ liveDesiredRevisionChanged = true
+ }
+ })
+ }
+ }
+
+ const inventoryDesiredRevisionChanged =
+ (selectedContextDesiredRevision !== undefined &&
+ options.contextDesiredRevision?.() !== selectedContextDesiredRevision) ||
+ (selectedServerDesiredRevision !== undefined &&
+ options.serverDesiredRevision?.() !== selectedServerDesiredRevision)
+ if (
+ (liveDesiredRevisionChanged || inventoryDesiredRevisionChanged) &&
+ contextCleanupAuthoritative() &&
+ serverCleanupAuthoritative()
+ ) {
+ throw new Error('Desired NetworkPolicy inventory changed during authoritative revocation')
+ }
+
+ if (
+ options.serverInventoryComplete !== false &&
+ contextCleanupAuthoritative() &&
+ serverCleanupAuthoritative()
+ ) {
+ if (measureSafetyPass) {
+ networkPolicySafetyPassPoliciesTotal.inc({ operation: 'listed' }, safetyPassListedPolicies)
+ networkPolicySafetyPassPoliciesTotal.inc(
+ { operation: 'revoked' },
+ safetyPassRevokedPolicies
+ )
+ }
+ options.onAuthoritativeRevocationComplete?.()
+ }
+
+ // Additive Context convergence happens only after every stale allow policy
+ // implied by the authoritative inventory has been revoked. A slow or
+ // failing positive effect therefore cannot extend a stale-permission
+ // window after readiness becomes available. External egress creation and
+ // DNS refresh intentionally do not run here:
+ // ExternalEgressConvergenceCoordinator is their sole startup, retry, and
+ // periodic-resync owner.
+ const contextAdditiveFailures: unknown[] = []
+ for (const context of contexts) {
+ const contextId = context.spec.contextId
+ try {
+ await runContextEffect(contextId, async () => {
+ const current = options.resolveCurrentContext
+ ? options.resolveCurrentContext(context.name)
+ : context
+ if (!current || current.spec.contextId !== contextId) return
+ const contextEffectIsCurrent = (): boolean => {
+ if (!contextInventoryIsCurrent()) return false
+ if (!options.resolveCurrentContext) return true
+ const latest = options.resolveCurrentContext(current.name)
+ return latest !== undefined && sameContextDesiredRevision(current, latest)
+ }
+ await this.reconcileContext(current, { isCurrent: contextEffectIsCurrent })
+ })
+ } catch (error) {
+ contextAdditiveFailures.push(error)
+ console.error(`[NetPol] Additive Context reconciliation failed for "${contextId}":`, error)
+ }
+ }
+
+ if (contextAdditiveFailures.length > 0) {
+ throw new AggregateError(
+ contextAdditiveFailures,
+ 'One or more additive Context NetworkPolicy reconciliations failed'
+ )
+ }
+
console.log('[NetPol] Full reconciliation complete')
}
+ private async contextOrphanDeleteAllowed(
+ contextId: string,
+ inventoryAuthoritative: () => boolean,
+ resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined
+ ): Promise<boolean> {
+ if (!inventoryAuthoritative() || resolveCurrentContextById?.(contextId)) return false
+
+ const response = (await this.customApi.listNamespacedCustomObject({
+ group: GROUP,
+ version: VERSION,
+ namespace: config.namespace,
+ plural: PLURAL_CONTEXTS,
+ })) as { items?: Array<{ spec?: { contextId?: string } }> }
+ if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false
+
+ return inventoryAuthoritative() && !resolveCurrentContextById?.(contextId)
+ }
+
// ─── L3 External Egress ────────────────────────────────────────────
+ private buildPublicWebEgressPolicy(server: McpServerCRD, name: string): k8s.V1NetworkPolicy {
+ return {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name,
+ namespace: server.namespace,
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
+ [MCPSERVER_LABEL]: server.name,
+ [EGRESS_CLASS_LABEL]: 'public-web',
+ },
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { [MCPSERVER_LABEL]: server.name },
+ },
+ policyTypes: ['Egress'],
+ egress: [
+ {
+ to: [
+ {
+ ipBlock: {
+ cidr: '0.0.0.0/0',
+ except: PUBLIC_EGRESS_EXCEPT_CIDRS,
+ },
+ },
+ ],
+ ports: [
+ { port: 443, protocol: 'TCP' },
+ { port: 80, protocol: 'TCP' },
+ ],
+ },
+ ],
+ },
+ }
+ }
+
+ private buildExactHostEgressPolicy(
+ server: McpServerCRD,
+ name: string,
+ binding: EgressBinding,
+ cidrs: string[]
+ ): k8s.V1NetworkPolicy {
+ return {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name,
+ namespace: server.namespace,
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
+ [MCPSERVER_LABEL]: server.name,
+ [EGRESS_CLASS_LABEL]: 'exact-host',
+ },
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { [MCPSERVER_LABEL]: server.name },
+ },
+ policyTypes: ['Egress'],
+ egress: cidrs.map(cidr => ({
+ to: [{ ipBlock: { cidr } }],
+ ports: [{ port: binding.port!, protocol: binding.protocol ?? 'TCP' }],
+ })),
+ },
+ }
+ }
+
+ private async resolveExternalEgressDns(
+ hostname: string
+ ): Promise<{ address: string; ttl: number }[]> {
+ // Resolve with TTL (issue #299 H2 ratchet needs the smallest observed TTL)
+ // while keeping the #205 bounded-timeout race so a silent resolver cannot
+ // block safety convergence indefinitely.
+ let timeout: ReturnType<typeof setTimeout> | undefined
+ try {
+ return await Promise.race([
+ dns.resolve4(hostname, { ttl: true }),
+ new Promise<never>((_, reject) => {
+ timeout = setTimeout(
+ () =>
+ reject(
+ new Error(
+ `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`
+ )
+ ),
+ config.externalEgressDnsResolveTimeoutMs
+ )
+ }),
+ ])
+ } finally {
+ if (timeout !== undefined) clearTimeout(timeout)
+ }
+ }
+
+ private async reconcileExternalEgressSafety(
+ server: McpServerCRD,
+ existingPolicies: k8s.V1NetworkPolicy[],
+ isCurrent: () => boolean,
+ onRevoked?: () => void
+ ): Promise<boolean> {
+ if (!isCurrent()) return false
+ const desired = new Map<string, { binding: EgressBinding; policy?: k8s.V1NetworkPolicy }>()
+ for (const binding of server.spec.egressBindings ?? []) {
+ const name = this.externalEgressPolicyName(server.name, binding)
+ if (!name) continue
+ const egressClass = binding.egressClass ?? 'exact-host'
+ if (
+ egressClass === 'public-web' &&
+ !binding.dns &&
+ !binding.cidr &&
+ binding.port === undefined &&
+ binding.protocol === undefined
+ ) {
+ desired.set(name, { binding, policy: this.buildPublicWebEgressPolicy(server, name) })
+ } else if (
+ egressClass === 'exact-host' &&
+ binding.cidr &&
+ isAllowedExternalEgressCidr(binding.cidr) &&
+ binding.port !== undefined &&
+ Number.isInteger(binding.port) &&
+ binding.port >= 1 &&
+ binding.port <= 65535
+ ) {
+ desired.set(name, {
+ binding,
+ policy: this.buildExactHostEgressPolicy(server, name, binding, [binding.cidr]),
+ })
+ } else if (
+ egressClass === 'exact-host' &&
+ binding.dns &&
+ isPublicDnsHostname(binding.dns) &&
+ binding.port !== undefined &&
+ Number.isInteger(binding.port) &&
+ binding.port >= 1 &&
+ binding.port <= 65535
+ ) {
+ desired.set(name, { binding })
+ }
+ }
+
+ const stale: k8s.V1NetworkPolicy[] = []
+ const unprovableDnsPolicies: k8s.V1NetworkPolicy[] = []
+ const deterministicReplacements: Array<{
+ snapshot: k8s.V1NetworkPolicy
+ desired: k8s.V1NetworkPolicy
+ }> = []
+ for (const existing of existingPolicies) {
+ const name = existing.metadata?.name
+ const selected = name ? desired.get(name) : undefined
+ if (!selected) {
+ stale.push(existing)
+ continue
+ }
+ desired.delete(name!)
+ if (!selected.policy) {
+ // DNS binding: the /32s cannot be proven without a fresh lookup, but
+ // the binding IDENTITY (protocol, spec-level ports, selector,
+ // ownership) can. Retain the live allow when identity is unchanged;
+ // revoke only on an identity change (a port move in the CRD changes the
+ // policy NAME and is handled by the stale lane above). The comparison is
+ // "modulo cidr": the desired policy is rebuilt WITH the live cidrs, so
+ // the cidrs compare against themselves and everything else must match
+ // exactly. A policy with no ipBlock peers is a deny disguised as an
+ // allow and is never retained.
+ const existingCidrs = (existing.spec?.egress ?? [])
+ .flatMap(rule => rule.to ?? [])
+ .map(peer => peer.ipBlock?.cidr)
+ .filter((cidr): cidr is string => typeof cidr === 'string')
+ const identityUnchanged =
+ existingCidrs.length > 0 &&
+ hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE) &&
+ sameNetworkPolicySpec(
+ existing,
+ this.buildExactHostEgressPolicy(server, name!, selected.binding, existingCidrs)
+ )
+ if (!identityUnchanged) unprovableDnsPolicies.push(existing)
+ continue
+ }
+ const retainable =
+ sameNetworkPolicySpec(existing, selected.policy) &&
+ hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE)
+ if (!retainable) {
+ deterministicReplacements.push({ snapshot: existing, desired: selected.policy })
+ }
+ }
+
+ // Preserve traffic only with deterministic CIDR/public-web policies. A
+ // renamed deterministic binding is created before its stale predecessor is
+ // removed; a same-name conflict is converged only under the ownership
+ // guard, which still refuses to adopt a foreign live object.
+ if (stale.length > 0) {
+ for (const selected of desired.values()) {
+ if (!selected.policy) continue
+ if (
+ !(await this.createSafetyPolicy(
+ server.namespace,
+ selected.policy,
+ 'external-egress',
+ isCurrent
+ ))
+ ) {
+ return false
+ }
+ }
+ }
+ for (const replacement of deterministicReplacements) {
+ if (
+ !(await this.replaceSafetyPolicySnapshot(
+ server.namespace,
+ replacement.snapshot,
+ replacement.desired,
+ 'external-egress',
+ isCurrent
+ ))
+ ) {
+ return false
+ }
+ onRevoked?.()
+ }
+
+ // DNS-derived /32s are unprovable without a fresh lookup. Revoke them
+ // immediately; replacement belongs to the additive lane and must not put
+ // external DNS latency back on the global readiness path.
+ await this.cleanupExternalEgress(
+ server.name,
+ server.namespace,
+ unprovableDnsPolicies,
+ async () => isCurrent(),
+ onRevoked
+ )
+ if (!isCurrent()) return false
+
+ await this.cleanupExternalEgress(
+ server.name,
+ server.namespace,
+ stale,
+ async () => isCurrent(),
+ onRevoked
+ )
+ return isCurrent()
+ }
+
/**
* Reconcile external egress policies for an McpServer with egressBindings.
* For DNS bindings, resolves hostnames to IPs and creates ipBlock rules.
* For CIDR bindings, creates ipBlock rules directly.
*/
- async reconcileExternalEgress(server: McpServerCRD): Promise<void> {
+ async reconcileExternalEgress(
+ server: McpServerCRD,
+ options: NetworkPolicyMutationOptions
+ ): Promise<void> {
+ const isCurrent = options.isCurrent
+ if (!isCurrent()) return
let existingPolicies: k8s.V1NetworkPolicy[]
try {
existingPolicies = await this.listExternalEgressPoliciesForServer(
@@ -816,36 +1918,48 @@ export class NetworkPolicyReconciler {
server.namespace
)
} catch (error) {
+ if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
[],
'False',
'InventoryListFailed',
- `Failed to list existing external egress policies: ${this.errorMessage(error)}`
+ `Failed to list existing external egress policies: ${this.errorMessage(error)}`,
+ isCurrent
)
throw error
}
+ if (!isCurrent()) return
const bindings = server.spec.egressBindings
if (!bindings || bindings.length === 0) {
try {
- await this.cleanupExternalEgress(server.name, server.namespace, existingPolicies)
+ await this.cleanupExternalEgress(
+ server.name,
+ server.namespace,
+ existingPolicies,
+ async () => isCurrent()
+ )
} catch (error) {
+ if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
[],
'False',
'CleanupFailed',
- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`
+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,
+ isCurrent
)
throw error
}
+ if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
[],
'True',
'NoEgressBindings',
- 'No external egress bindings declared'
+ 'No external egress bindings declared',
+ isCurrent
)
return
}
@@ -857,9 +1971,11 @@ export class NetworkPolicyReconciler {
const desiredPolicyNames = new Set<string>()
const resolvedEgressIPs: McpServerResolvedEgressIP[] = []
const failures: string[] = []
- const resolvedAt = new Date().toISOString()
+ const resolvedAtMs = Date.now()
+ const resolvedAt = new Date(resolvedAtMs).toISOString()
for (const binding of bindings) {
+ if (!isCurrent()) return
const egressClass = binding.egressClass ?? 'exact-host'
if (egressClass !== 'exact-host' && egressClass !== 'public-web') {
failures.push(`egressClass "${String(binding.egressClass)}" is not supported`)
@@ -905,47 +2021,13 @@ export class NetworkPolicyReconciler {
desiredPolicyNames.add(name)
- const policy: k8s.V1NetworkPolicy = {
- apiVersion: 'networking.k8s.io/v1',
- kind: 'NetworkPolicy',
- metadata: {
- name,
- namespace: server.namespace,
- labels: {
- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
- [MCPSERVER_LABEL]: server.name,
- [EGRESS_CLASS_LABEL]: egressClass,
- },
- },
- spec: {
- podSelector: {
- matchLabels: { [MCPSERVER_LABEL]: server.name },
- },
- policyTypes: ['Egress'],
- egress: [
- {
- to: [
- {
- ipBlock: {
- cidr: '0.0.0.0/0',
- except: PUBLIC_EGRESS_EXCEPT_CIDRS,
- },
- },
- ],
- ports: [
- { port: 443, protocol: 'TCP' },
- { port: 80, protocol: 'TCP' },
- ],
- },
- ],
- },
- }
-
- // public-web is a static policy — write only when new/changed/drifted so
- // the TTL-accelerated resync does not rewrite it every tick (audit F2).
+ // public-web is a static policy. Preserve #205's ownership-fenced write
+ // (applyOwnedPolicy asserts ownership + honors the isCurrent generation
+ // fence) while keeping dev's F2 no-churn gate so the TTL-accelerated
+ // resync does not rewrite it every tick.
+ const policy = this.buildPublicWebEgressPolicy(server, name)
if (this.externalEgressWriteNeeded(existingPolicy, policy)) {
- await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')
+ await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
}
continue
}
@@ -993,7 +2075,8 @@ export class NetworkPolicyReconciler {
maxEntries: config.externalEgressMaxEntries,
}
try {
- const records = await dns.resolve4(binding.dns, { ttl: true })
+ const records = await this.resolveExternalEgressDns(binding.dns)
+ if (!isCurrent()) return
const uniqueIps = [...new Set(records.map(r => r.address))].sort()
if (uniqueIps.length === 0) {
failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)
@@ -1094,43 +2177,25 @@ export class NetworkPolicyReconciler {
desiredPolicyNames.add(name)
- const policy: k8s.V1NetworkPolicy = {
- apiVersion: 'networking.k8s.io/v1',
- kind: 'NetworkPolicy',
- metadata: {
- name,
- namespace: server.namespace,
- labels: {
- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
- [MCPSERVER_LABEL]: server.name,
- [EGRESS_CLASS_LABEL]: egressClass,
- },
- // Persist the sliding-window state so the next reconcile rehydrates the
- // accumulated set instead of collapsing to a single snapshot (#299).
- ...(stateAnnotations ? { annotations: stateAnnotations } : {}),
- },
- spec: {
- podSelector: {
- matchLabels: { [MCPSERVER_LABEL]: server.name },
- },
- policyTypes: ['Egress'],
- egress: cidrs.map(cidr => ({
- to: [{ ipBlock: { cidr } }],
- ports: [{ port: binding.port, protocol }],
- })),
- },
+ // Build the exact-host policy from #205's helper, then fold in the #299
+ // sliding-window state annotations so the next reconcile rehydrates the
+ // accumulated set instead of collapsing to a single snapshot.
+ const policy = this.buildExactHostEgressPolicy(server, name, binding, cidrs)
+ if (stateAnnotations) {
+ policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }
}
// Write only when the enforced rules changed / the policy is new / drifted
// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and
// must be re-persisted (audit M1). A pure timestamp refresh is a no-op.
+ // Route the write through applyOwnedPolicy so #205's ownership assertion
+ // and isCurrent generation fence still guard the mutation.
if (
this.externalEgressWriteNeeded(existingPolicy, policy) ||
egressRenewalDue ||
egressStateChanged
) {
- await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')
+ await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
} else {
console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)
}
@@ -1143,18 +2208,23 @@ export class NetworkPolicyReconciler {
existingPolicies.filter(policy => {
const name = policy.metadata?.name
return Boolean(name) && !desiredPolicyNames.has(name!)
- })
+ }),
+ async () => isCurrent(),
+ options.onRevoked
)
} catch (error) {
+ if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
resolvedEgressIPs,
'False',
'CleanupFailed',
- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`
+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,
+ isCurrent
)
throw error
}
+ if (!isCurrent()) return
if (failures.length > 0) {
const message = failures.join('; ')
@@ -1163,7 +2233,8 @@ export class NetworkPolicyReconciler {
resolvedEgressIPs,
'False',
'ExternalEgressRejected',
- message
+ message,
+ isCurrent
)
throw new Error(`External egress reconciliation failed for "${server.name}": ${message}`)
}
@@ -1175,7 +2246,8 @@ export class NetworkPolicyReconciler {
'Reconciled',
bindings.some(binding => binding.egressClass === 'public-web')
? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'
- : 'External egress policies reconciled'
+ : 'External egress policies reconciled',
+ isCurrent
)
}
@@ -1185,7 +2257,9 @@ export class NetworkPolicyReconciler {
async cleanupExternalEgress(
serverName: string,
namespace: string,
- policies?: k8s.V1NetworkPolicy[]
+ policies?: k8s.V1NetworkPolicy[],
+ deleteAllowed?: () => Promise<boolean>,
+ onDeleted?: () => void
): Promise<void> {
const policiesToDelete =
policies ?? (await this.listExternalEgressPoliciesForServer(serverName, namespace))
@@ -1194,13 +2268,19 @@ export class NetworkPolicyReconciler {
const name = policy.metadata?.name
if (!name) continue
try {
- await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
- console.log(`[NetPol] Deleted external egress policy "${name}"`)
+ if (deleteAllowed && !(await deleteAllowed())) return
+ await this.deleteSafetyPolicySnapshot(namespace, policy, undefined, () => {
+ console.log(`[NetPol] Deleted external egress policy "${name}"`)
+ onDeleted?.()
+ })
} catch (error: unknown) {
if (getErrorCode(error) !== 404) {
console.error(`[NetPol] Failed to delete external egress policy "${name}":`, error)
throw error
}
+ // A concurrent actor may already have removed the policy. The safety
+ // invariant still holds, but this pass did not revoke it and must not
+ // inflate the authoritative safety-pass `revoked` counter.
}
}
}
@@ -1282,10 +2362,19 @@ export class NetworkPolicyReconciler {
resolvedEgressIPs: McpServerResolvedEgressIP[],
status: 'True' | 'False' | 'Unknown',
reason: string,
- message: string
+ message: string,
+ isCurrent: () => boolean = () => true
): Promise<void> {
+ if (!isCurrent()) return
let currentStatus: McpServerCrdStatus = {}
let hasStatusObject = false
+ let currentMetadata:
+ | {
+ uid?: string
+ generation?: number
+ resourceVersion?: string
+ }
+ | undefined
try {
const current = (await this.customApi.getNamespacedCustomObjectStatus({
@@ -1294,7 +2383,15 @@ export class NetworkPolicyReconciler {
namespace: server.namespace,
plural: PLURAL_MCPSERVERS,
name: server.name,
- })) as { status?: McpServerCrdStatus }
+ })) as {
+ metadata?: {
+ uid?: string
+ generation?: number
+ resourceVersion?: string
+ }
+ status?: McpServerCrdStatus
+ }
+ currentMetadata = current.metadata
hasStatusObject = typeof current.status === 'object' && current.status !== null
currentStatus = current.status ?? {}
} catch (error) {
@@ -1310,6 +2407,20 @@ export class NetworkPolicyReconciler {
)
throw error
}
+ if (!isCurrent()) return
+
+ if (
+ (server.uid !== undefined && currentMetadata?.uid !== server.uid) ||
+ (server.generation !== undefined && currentMetadata?.generation !== server.generation)
+ ) {
+ throw new Error(
+ `Refusing to write external egress status for stale McpServer "${server.name}": ` +
+ `expected uid=${String(server.uid)} generation=${String(server.generation)}, ` +
+ `read uid=${String(currentMetadata?.uid)} generation=${String(
+ currentMetadata?.generation
+ )}`
+ )
+ }
const nextConditions = this.mergeStatusCondition(currentStatus.conditions ?? [], {
type: 'ExternalEgressReady',
@@ -1334,14 +2445,36 @@ export class NetworkPolicyReconciler {
return
}
- const statusPatch: JsonPatchOperation[] = hasStatusObject
+ const identityFence: JsonPatchOperation[] = []
+ const expectedUid = server.uid ?? currentMetadata?.uid
+ if (expectedUid !== undefined) {
+ identityFence.push({ op: 'test', path: '/metadata/uid', value: expectedUid })
+ }
+ const expectedGeneration = server.generation ?? currentMetadata?.generation
+ if (expectedGeneration !== undefined) {
+ identityFence.push({
+ op: 'test',
+ path: '/metadata/generation',
+ value: expectedGeneration,
+ })
+ }
+ if (currentMetadata?.resourceVersion !== undefined) {
+ identityFence.push({
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: currentMetadata.resourceVersion,
+ })
+ }
+ const statusMutation: JsonPatchOperation[] = hasStatusObject
? [
{ op: 'add', path: '/status/resolvedEgressIPs', value: stableResolvedEgressIPs },
{ op: 'add', path: '/status/conditions', value: nextConditions },
]
: [{ op: 'add', path: '/status', value: nextStatus }]
+ const statusPatch = [...identityFence, ...statusMutation]
try {
+ if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObjectStatus({
group: GROUP,
version: VERSION,
@@ -1394,6 +2527,61 @@ export class NetworkPolicyReconciler {
}
}
+ /** Broad, namespace-bounded inventory used only by the safety pass. */
+ private async listNamespacePoliciesForSafety(namespace: string): Promise<k8s.V1NetworkPolicy[]> {
+ try {
+ const response = await this.networkingApi.listNamespacedNetworkPolicy({ namespace })
+ return response.items || []
+ } catch (error) {
+ console.error(`[NetPol] Failed to list broad NetworkPolicy inventory in ${namespace}:`, error)
+ throw error
+ }
+ }
+
+ private mergeSafetyLaneInventory(
+ selected: k8s.V1NetworkPolicy[],
+ broad: k8s.V1NetworkPolicy[],
+ lane: SafetyInventoryLane
+ ): k8s.V1NetworkPolicy[] {
+ const result = new Map<string, k8s.V1NetworkPolicy>()
+ const keyFor = (policy: k8s.V1NetworkPolicy, index: number): string =>
+ policy.metadata?.uid ?? policy.metadata?.name ?? `unnamed-${index}`
+
+ selected.forEach((policy, index) => result.set(keyFor(policy, index), policy))
+ broad.forEach((policy, index) => {
+ const classification = classifySafetyInventoryPolicy(policy, lane)
+ if (classification === 'ambiguous') {
+ const name = policy.metadata?.name ?? '<unnamed>'
+ throw new Error(
+ `Ambiguous NetworkPolicy ownership for reserved HCC ${lane} policy "${name}"`
+ )
+ }
+ if (classification === 'owned' || classification === 'repairable') {
+ result.set(keyFor(policy, selected.length + index), policy)
+ } else {
+ // The broad LIST is later and authoritative for this object identity.
+ // Do not retain a selector snapshot after the same object was relabelled
+ // out of HCC ownership.
+ result.delete(keyFor(policy, selected.length + index))
+ }
+ })
+ return [...result.values()]
+ }
+
+ /** List all L2 context-allow egress policies in the mcp-host namespace. */
+ private async listAllContextEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {
+ try {
+ const response = await this.networkingApi.listNamespacedNetworkPolicy({
+ namespace: config.hostNamespace,
+ labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=context-allow`,
+ })
+ return response.items || []
+ } catch (error) {
+ console.error('[NetPol] Failed to list all Context egress policies:', error)
+ throw error
+ }
+ }
+
/** List all external-egress policies in the MCP server namespace. */
private async listAllExternalEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {
try {
@@ -1421,26 +2609,234 @@ export class NetworkPolicyReconciler {
}
/** Create or update a NetworkPolicy. */
- private async applyPolicy(name: string, policy: k8s.V1NetworkPolicy): Promise<void> {
- await applyNetworkPolicy(this.networkingApi, name, config.namespace, policy, '[NetPol]')
+ private async applyFixedPolicy(
+ name: string,
+ namespace: string,
+ policy: k8s.V1NetworkPolicy,
+ policyType: string
+ ): Promise<void> {
+ await applyNetworkPolicy(
+ this.networkingApi,
+ name,
+ namespace,
+ policy,
+ '[NetPol]',
+ undefined,
+ existing => {
+ if (!hasExpectedPolicyOwnership(existing, policyType)) {
+ throw new Error(
+ `NetworkPolicy "${name}" has conflicting ownership for the ${policyType} lane`
+ )
+ }
+ }
+ )
}
- /** Delete a NetworkPolicy in mcp-server namespace. */
- private async deletePolicy(name: string): Promise<void> {
+ /** Create or update a Context ingress NetworkPolicy. */
+ private async applyContextIngressPolicy(
+ name: string,
+ policy: k8s.V1NetworkPolicy,
+ isCurrent?: () => boolean
+ ): Promise<void> {
+ await this.applyOwnedPolicy(name, config.namespace, policy, 'context-ingress', isCurrent)
+ }
+
+ private async applyOwnedPolicy(
+ name: string,
+ namespace: string,
+ policy: k8s.V1NetworkPolicy,
+ lane: SafetyInventoryLane,
+ isCurrent?: () => boolean
+ ): Promise<void> {
+ await applyNetworkPolicy(
+ this.networkingApi,
+ name,
+ namespace,
+ policy,
+ '[NetPol]',
+ isCurrent,
+ existing => this.assertPolicyOwnership(name, existing, policy, lane)
+ )
+ }
+
+ private assertPolicyOwnership(
+ name: string,
+ existing: k8s.V1NetworkPolicy,
+ desired: k8s.V1NetworkPolicy,
+ lane: SafetyInventoryLane
+ ): void {
+ const classification = classifySafetyInventoryPolicy(existing, lane)
+ const existingLabels = existing.metadata?.labels ?? {}
+ const desiredLabels = desired.metadata?.labels ?? {}
+ const ownerLabels =
+ lane === 'external-egress' ? [MCPSERVER_LABEL] : [CONTEXT_LABEL, MCPSERVER_LABEL]
+ if (
+ (classification !== 'owned' && classification !== 'repairable') ||
+ ownerLabels.some(label => existingLabels[label] !== desiredLabels[label])
+ ) {
+ throw new Error(`NetworkPolicy "${name}" has conflicting ownership for the ${lane} lane`)
+ }
+ }
+
+ private safetyPolicyIdentity(policy: k8s.V1NetworkPolicy): {
+ name: string
+ uid: string
+ resourceVersion: string
+ } {
+ const name = policy.metadata?.name
+ const uid = policy.metadata?.uid
+ const resourceVersion = policy.metadata?.resourceVersion
+ if (!name || !uid || !resourceVersion) {
+ throw new Error(
+ `NetworkPolicy safety mutation requires name, uid, and resourceVersion for "${name ?? '<unnamed>'}"`
+ )
+ }
+ return { name, uid, resourceVersion }
+ }
+
+ /**
+ * Delete exactly the policy version classified by the authoritative safety
+ * inventory. Kubernetes rejects the preconditions if the name was recreated
+ * or changed after LIST, so HCC cannot delete a new or foreign object.
+ */
+ private async deleteSafetyPolicySnapshot(
+ namespace: string,
+ policy: k8s.V1NetworkPolicy,
+ isCurrent?: () => boolean,
+ onDeleted?: () => void,
+ lostFenceOutcome: 'throw' | 'report' = 'throw'
+ ): Promise<boolean> {
+ if (isCurrent && !isCurrent()) return false
+ const { name, uid, resourceVersion } = this.safetyPolicyIdentity(policy)
try {
await this.networkingApi.deleteNamespacedNetworkPolicy({
name,
- namespace: config.namespace,
+ namespace,
+ body: { preconditions: { uid, resourceVersion } },
})
- console.log(`[NetPol] Deleted policy "${name}"`)
+ console.log(`[NetPol] Deleted safety-inventory policy "${name}" in ${namespace}`)
+ onDeleted?.()
+ return true
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[NetPol] Policy "${name}" already gone`)
- } else {
- console.error(`[NetPol] Failed to delete policy "${name}":`, error)
- throw error
+ console.log(`[NetPol] Safety-inventory policy "${name}" already gone`)
+ return true
+ }
+ if (getErrorCode(error) === 409 && lostFenceOutcome === 'report') {
+ // Kubernetes rejected the uid/resourceVersion preconditions, so the
+ // object live under this name is not the one the authoritative
+ // inventory classified. Nothing was revoked, and nothing may be
+ // adopted. Callers that opt in act on this outcome by declining to
+ // certify, which is the expected racy result rather than a failure.
+ // Callers that cannot act on it keep the loud default.
+ console.warn(
+ `[NetPol] Lost the delete fence on safety-inventory policy "${name}" in ${namespace}; it changed identity after the authoritative inventory and was not revoked`
+ )
+ return false
+ }
+ console.error(
+ `[NetPol] Failed identity-bound delete of safety-inventory policy "${name}" in ${namespace}:`,
+ error
+ )
+ throw error
+ }
+ }
+
+ /**
+ * Replace only the exact live object classified by the safety inventory.
+ * The fresh read closes ownership drift before mutation; resourceVersion on
+ * the replace body closes the remaining read-to-write race.
+ */
+ private async replaceSafetyPolicySnapshot(
+ namespace: string,
+ snapshot: k8s.V1NetworkPolicy,
+ desired: k8s.V1NetworkPolicy,
+ lane: SafetyInventoryLane,
+ isCurrent: () => boolean
+ ): Promise<boolean> {
+ if (!isCurrent()) return false
+ const expected = this.safetyPolicyIdentity(snapshot)
+ const live = await this.networkingApi.readNamespacedNetworkPolicy({
+ name: expected.name,
+ namespace,
+ })
+ if (!isCurrent()) return false
+ const actual = this.safetyPolicyIdentity(live)
+ if (actual.uid !== expected.uid || actual.resourceVersion !== expected.resourceVersion) {
+ throw new Error(
+ `NetworkPolicy "${expected.name}" changed identity or ownership after the authoritative safety inventory`
+ )
+ }
+ this.assertPolicyOwnership(expected.name, live, desired, lane)
+ await this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: expected.name,
+ namespace,
+ body: {
+ ...desired,
+ metadata: {
+ ...desired.metadata,
+ name: expected.name,
+ namespace,
+ resourceVersion: actual.resourceVersion,
+ },
+ },
+ })
+ return isCurrent()
+ }
+
+ private async createSafetyPolicy(
+ namespace: string,
+ desired: k8s.V1NetworkPolicy,
+ lane: SafetyInventoryLane,
+ isCurrent: () => boolean
+ ): Promise<boolean> {
+ if (!isCurrent()) return false
+ let created: k8s.V1NetworkPolicy
+ try {
+ created = await this.networkingApi.createNamespacedNetworkPolicy({
+ namespace,
+ body: desired,
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 409) throw error
+ // This is the only creation route that does not go through
+ // applyNetworkPolicy, so a same-name object used to abort the whole
+ // safety pass and hold readiness down. Converge it exactly like every
+ // other lane does: adopt it only under the ownership guard, which still
+ // refuses a foreign or ambiguous object.
+ const name = desired.metadata?.name
+ if (!name) {
+ throw new Error(`Cannot converge a conflicting ${lane} policy without a desired name`)
}
+ console.log(
+ `[NetPol] Safety policy "${name}" in ${namespace} already exists — converging it to the desired spec`
+ )
+ // Deliberately not delegating to applyOwnedPolicy here: that helper owns
+ // the create-then-conflict flow, so calling it from inside our own
+ // create's catch would issue a second create. Delegating from the top is
+ // not an option either — the desired-state fence below needs the object
+ // this request created, which applyNetworkPolicy does not return. The
+ // wiring below is kept identical to it on purpose.
+ await replaceWithConflictRetry<k8s.V1NetworkPolicy>({
+ description: `policy "${name}" in ${namespace}`,
+ logPrefix: '[NetPol]',
+ body: desired,
+ read: () => this.networkingApi.readNamespacedNetworkPolicy({ name, namespace }),
+ replace: body =>
+ this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
+ mutationAllowed: isCurrent,
+ validateExisting: existing => this.assertPolicyOwnership(name, existing, desired, lane),
+ })
+ return isCurrent()
}
+ if (isCurrent()) return true
+
+ // The mutation crossed its desired-state fence. Remove exactly the object
+ // this request created, even though the old reconciliation is no longer
+ // authoritative. The UID/resourceVersion preconditions ensure a
+ // replacement or foreign policy with the same name is never deleted.
+ await this.deleteSafetyPolicySnapshot(namespace, created)
+ return false
}
/** List L2 egress counterpart policies in mcp-host namespace for a given context. */
@@ -1457,37 +2853,15 @@ export class NetworkPolicyReconciler {
}
}
- /** Delete a NetworkPolicy in mcp-host namespace (L2 egress counterpart). */
- private async deleteEgressPolicy(name: string): Promise<void> {
- try {
- await this.networkingApi.deleteNamespacedNetworkPolicy({
- name,
- namespace: config.hostNamespace,
- })
- console.log(`[NetPol] Deleted L2 egress policy "${name}"`)
- } catch (error: unknown) {
- if (getErrorCode(error) === 404) {
- console.log(`[NetPol] L2 egress policy "${name}" already gone`)
- } else {
- console.error(`[NetPol] Failed to delete L2 egress policy "${name}":`, error)
- throw error
- }
- }
- }
-
// ─── RPC-Proxy Egress (L2) ──────────────────────────────────────────
- /**
- * Create or update an egress policy in the rpc-proxy namespace allowing
- * rpc-proxy pods to reach a specific MCP server on a given port.
- */
- private async ensureRpcProxyEgress(
+ private buildRpcProxyEgressPolicy(
contextId: string,
serverName: string,
port: number
- ): Promise<void> {
+ ): k8s.V1NetworkPolicy {
const name = `rpc-egress-${contextId}-${serverName}`
- const policy: k8s.V1NetworkPolicy = {
+ return {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
@@ -1518,16 +2892,18 @@ export class NetworkPolicyReconciler {
],
},
}
- await applyNetworkPolicy(this.networkingApi, name, config.rpcProxyNamespace, policy, '[NetPol]')
}
/** Delete all rpc-proxy egress policies for a given context. */
- private async deleteRpcProxyPoliciesForContext(contextId: string): Promise<void> {
+ private async deleteRpcProxyPoliciesForContext(
+ contextId: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
try {
const policies = await this.listRpcProxyEgressPoliciesForContext(contextId)
for (const policy of policies) {
- const name = policy.metadata?.name || ''
- await this.deletePolicyInNamespace(config.rpcProxyNamespace, name)
+ if (deleteAllowed && !(await deleteAllowed())) return
+ await this.deleteSafetyPolicySnapshot(config.rpcProxyNamespace, policy)
}
} catch (error) {
console.error(
@@ -1571,34 +2947,19 @@ export class NetworkPolicyReconciler {
}
}
- /** Delete a NetworkPolicy in an arbitrary namespace. */
- private async deletePolicyInNamespace(namespace: string, name: string): Promise<void> {
- try {
- await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
- console.log(`[NetPol] Deleted rpc-proxy egress policy "${name}"`)
- } catch (error: unknown) {
- if (getErrorCode(error) === 404) {
- console.log(`[NetPol] rpc-proxy egress policy "${name}" already gone`)
- } else {
- console.error(`[NetPol] Failed to delete policy "${name}" in ${namespace}:`, error)
- throw error
- }
- }
- }
-
/** Delete a formerly static NetworkPolicy that is now generated dynamically. */
private async deleteLegacyStaticPolicy(namespace: string, name: string): Promise<void> {
+ let existing: k8s.V1NetworkPolicy
try {
- await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
- console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)
+ existing = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
} catch (error: unknown) {
- if (getErrorCode(error) !== 404) {
- console.error(
- `[NetPol] Failed to delete legacy static policy "${name}" in "${namespace}":`,
- error
- )
- throw error
- }
+ if (getErrorCode(error) === 404) return
+ throw error
+ }
+ if (existing.metadata?.labels?.[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE) {
+ throw new Error(`NetworkPolicy "${name}" has conflicting ownership for legacy cleanup`)
}
+ await this.deleteSafetyPolicySnapshot(namespace, existing)
+ console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)
}
}
diff --git a/host-context-controller/src/readinessGate.test.ts b/host-context-controller/src/readinessGate.test.ts
new file mode 100644
index 000000000..a9b522ab0
--- /dev/null
+++ b/host-context-controller/src/readinessGate.test.ts
@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest'
+import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
+
+describe('resolveProviderAuthoritativeFn (dev-mode readiness gate — B1)', () => {
+ it('dev provider (no watcher) is unconditionally authoritative', () => {
+ // Regression for R3-B1: main.ts left this undefined for DevMcpServerProvider,
+ // so the ContextMapperServer fail-closed default (() => false) pinned /ready
+ // and every /api/v1/* endpoint at 503 forever in dev mode. There is no
+ // NetworkPolicy inventory to certify in dev, so authority must be granted.
+ expect(resolveProviderAuthoritativeFn(null)()).toBe(true)
+ })
+
+ it('production watcher delegates to the live inventory certification', () => {
+ let asked = false
+ const watcher = {
+ isReadinessInventoryAuthoritative: () => {
+ asked = true
+ return false
+ },
+ }
+ // Delegates (returns the watcher's verdict) rather than a constant.
+ expect(resolveProviderAuthoritativeFn(watcher)()).toBe(false)
+ expect(asked).toBe(true)
+ watcher.isReadinessInventoryAuthoritative = () => true
+ expect(resolveProviderAuthoritativeFn(watcher)()).toBe(true)
+ })
+})
+
+describe('resolveHostAuthoritativeFn (dev-mode desktop gate — R2-M1)', () => {
+ it('dev provider (no watcher) is unconditionally authoritative', () => {
+ // Regression for R2-M1: main.ts left this undefined for DevMcpServerProvider,
+ // so the ContextMapperServer fail-closed default (() => false) pinned every
+ // /api/v1/desktop/* response at 503 after setReady(true) — where the base
+ // answered 200 {status:'inactive'}. There is no Host inventory to certify in
+ // dev, so authority must mirror the provider gate and be granted.
+ expect(resolveHostAuthoritativeFn(null)()).toBe(true)
+ })
+
+ it('production watcher delegates to the live Host inventory certification', () => {
+ let asked = false
+ const watcher = {
+ isHostInventoryAuthoritative: () => {
+ asked = true
+ return false
+ },
+ }
+ // Delegates (returns the watcher's verdict) rather than a constant.
+ expect(resolveHostAuthoritativeFn(watcher)()).toBe(false)
+ expect(asked).toBe(true)
+ watcher.isHostInventoryAuthoritative = () => true
+ expect(resolveHostAuthoritativeFn(watcher)()).toBe(true)
+ })
+})
diff --git a/host-context-controller/src/readinessGate.ts b/host-context-controller/src/readinessGate.ts
new file mode 100644
index 000000000..88ff77ee6
--- /dev/null
+++ b/host-context-controller/src/readinessGate.ts
@@ -0,0 +1,41 @@
+/**
+ * Resolve the readiness-authority gate for the server from the provider.
+ *
+ * Production (McpServerWatcher present): readiness follows the live inventory
+ * certification — /ready is 200 only once stale allows are revoked.
+ *
+ * Dev (DevMcpServerProvider — no watcher): there is NO NetworkPolicy inventory
+ * to certify, so authority is unconditional. The ContextMapperServer default
+ * for a missing gate is fail-closed (`() => false`), which is correct for a
+ * caller that forgets to wire one — but in dev it would pin /ready (and every
+ * data endpoint) at 503 forever. Dev must therefore pass an explicit
+ * always-authoritative gate; this function is that single, named decision so
+ * the affirmative literal never gets copy-pasted into a production path.
+ */
+export function resolveProviderAuthoritativeFn(
+ watcher: { isReadinessInventoryAuthoritative(): boolean } | null
+): () => boolean {
+ return watcher ? () => watcher.isReadinessInventoryAuthoritative() : () => true
+}
+
+/**
+ * Resolve the Host-inventory-authority gate the desktop route uses.
+ *
+ * Production (McpServerWatcher present): desktop status follows the live Host
+ * inventory certification — a degraded Host lane 503s desktop rather than
+ * answering a stale 200 'inactive'.
+ *
+ * Dev (DevMcpServerProvider — no watcher): there is NO Host inventory to
+ * certify, exactly as there is no NetworkPolicy inventory for the provider
+ * gate. The ContextMapperServer default for a missing gate is fail-closed
+ * (`() => false`), which is correct for a caller that forgets to wire one — but
+ * in dev it would pin every /api/v1/desktop/* response at 503 forever (R2-M1).
+ * Dev must therefore mirror the provider gate and pass an explicit
+ * always-authoritative gate; this function is that single, named decision so
+ * the affirmative literal never gets copy-pasted into a production path.
+ */
+export function resolveHostAuthoritativeFn(
+ watcher: { isHostInventoryAuthoritative(): boolean } | null
+): () => boolean {
+ return watcher ? () => watcher.isHostInventoryAuthoritative() : () => true
+}
diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.ts
new file mode 100644
index 000000000..4bd14f5e6
--- /dev/null
+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts
@@ -0,0 +1,513 @@
+/**
+ * Crosses the whole readiness chain with a real NetworkPolicyReconciler.
+ *
+ * Every other test of this behaviour mocks one side. `k8sClient.test.ts` mocks
+ * `./networkPolicyReconciler`, so it can prove the readiness predicate *reads*
+ * the certification getter but not that the reconciler ever *degrades* it.
+ * `networkPolicyReconciler.test.ts` drives the real fence but never reaches an
+ * HTTP response. Each half is verified by mutation; the seam between them was
+ * not, and the seam is where the guarantee actually lives.
+ *
+ * Here `@kubernetes/client-node` is the only mock. The reconciler, the watcher
+ * predicate, and the HTTP server are the production classes, so the assertion
+ * is end-to-end: a delete that loses its uid/resourceVersion precondition
+ * during the additive phase — the 409 a second writer produces, and every
+ * rollout opens a two-replica window for one — must turn /ready from 200 into
+ * 503, and a later clean pass must turn it back.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import * as http from 'http'
+import { McpServerWatcher } from './k8sClient'
+import { ContextMapperServer } from './server'
+
+const mocks = vi.hoisted(() => ({
+ listNamespacedNetworkPolicy: vi.fn(),
+ deleteNamespacedNetworkPolicy: vi.fn(),
+ readNamespacedNetworkPolicy: vi.fn(),
+ createNamespacedNetworkPolicy: vi.fn(),
+ replaceNamespacedNetworkPolicy: vi.fn(),
+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
+ getNamespacedCustomObject: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ getNamespacedCustomObjectStatus: vi
+ .fn()
+ .mockResolvedValue({ metadata: { generation: 1, resourceVersion: 'rv-1' }, status: {} }),
+ patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
+ watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),
+}))
+
+vi.mock('./config', () => ({
+ config: {
+ devMode: false,
+ namespace: 'mcp-server',
+ hostNamespace: 'mcp-host',
+ rpcProxyNamespace: 'rpc-proxy',
+ channelsNamespace: 'channels',
+ port: 0,
+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],
+ hostK8sRequestTimeoutMs: 30_000,
+ hostResyncIntervalSec: 0,
+ externalEgressResyncIntervalSec: 0,
+ controlApiBaseUrl: 'http://control-api.test:8090',
+ governedTracingEnabled: false,
+ },
+}))
+
+vi.mock('@kubernetes/client-node', () => {
+ class CustomObjectsApi {}
+ class CoreV1Api {}
+ class NetworkingV1Api {}
+ class PolicyV1Api {}
+ class AppsV1Api {}
+ class BatchV1Api {}
+ class RbacAuthorizationV1Api {}
+ class Watch {
+ constructor(_kc: unknown) {}
+ watch = mocks.watch
+ }
+ class KubeConfig {
+ loadFromDefault(): void {}
+ loadFromOptions(_o: unknown): void {}
+ makeApiClient(api: unknown): unknown {
+ if (api === NetworkingV1Api) {
+ return {
+ listNamespacedNetworkPolicy: mocks.listNamespacedNetworkPolicy,
+ deleteNamespacedNetworkPolicy: mocks.deleteNamespacedNetworkPolicy,
+ readNamespacedNetworkPolicy: mocks.readNamespacedNetworkPolicy,
+ createNamespacedNetworkPolicy: mocks.createNamespacedNetworkPolicy,
+ replaceNamespacedNetworkPolicy: mocks.replaceNamespacedNetworkPolicy,
+ }
+ }
+ if (api === CustomObjectsApi) {
+ return {
+ listNamespacedCustomObject: mocks.listNamespacedCustomObject,
+ getNamespacedCustomObject: mocks.getNamespacedCustomObject,
+ getNamespacedCustomObjectStatus: mocks.getNamespacedCustomObjectStatus,
+ patchNamespacedCustomObjectStatus: mocks.patchNamespacedCustomObjectStatus,
+ }
+ }
+ return new Proxy({}, { get: () => vi.fn().mockResolvedValue({ items: [] }) })
+ }
+ }
+ return {
+ CustomObjectsApi,
+ CoreV1Api,
+ NetworkingV1Api,
+ PolicyV1Api,
+ AppsV1Api,
+ BatchV1Api,
+ RbacAuthorizationV1Api,
+ Watch,
+ KubeConfig,
+ }
+})
+
+const MANAGED = 'clerum.io/managed-by'
+const POLICY_TYPE = 'clerum.io/policy-type'
+const CONTEXT_LABEL = 'clerum.io/context'
+const SERVER_LABEL = 'clerum.io/mcpserver'
+
+/** A stale context-allow: HCC-owned, for a server the Context no longer lists. */
+const stalePolicy = {
+ metadata: {
+ name: 'ctx-alpha-ghost',
+ namespace: 'mcp-server',
+ uid: 'ghost-uid',
+ resourceVersion: '7',
+ labels: {
+ [MANAGED]: 'host-context-controller',
+ [POLICY_TYPE]: 'context-allow',
+ [CONTEXT_LABEL]: 'alpha',
+ [SERVER_LABEL]: 'ghost',
+ },
+ },
+ spec: { podSelector: {}, policyTypes: ['Ingress'] },
+}
+
+const alphaContext = {
+ name: 'alpha',
+ namespace: 'mcp-server',
+ spec: { contextId: 'alpha', mcpServers: [] },
+}
+
+async function readyStatus(server: InstanceType<typeof ContextMapperServer>): Promise<number> {
+ const listener = (server as unknown as { server: http.Server | null }).server
+ const address = listener?.address()
+ if (!address || typeof address === 'string') throw new Error('readiness listener never bound')
+ return new Promise((resolve, reject) => {
+ http
+ .get({ host: '127.0.0.1', port: address.port, path: '/ready' }, res => {
+ res.resume()
+ res.on('end', () => resolve(res.statusCode ?? 0))
+ })
+ .on('error', reject)
+ })
+}
+
+/** Aligns every revocation counter so only the fence can gate readiness. */
+function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>): void {
+ const w = watcher as unknown as Record<string, unknown>
+ w.mcpServerCacheSynced = true
+ w.contextCacheSynced = true
+ w.hostCacheSynced = true
+ w.networkPolicyRevocationContextGeneration = w.contextWatchGeneration
+ w.networkPolicyRevocationServerGeneration = w.mcpWatchGeneration
+ w.networkPolicyRevocationContextRevision = w.contextDesiredRevision
+ w.networkPolicyRevocationServerRevision = w.mcpServerDesiredRevision
+}
+
+describe('readiness withdrawal on a real lost delete fence', () => {
+ let watcher: InstanceType<typeof McpServerWatcher>
+ let server: InstanceType<typeof ContextMapperServer>
+
+ beforeEach(async () => {
+ vi.clearAllMocks()
+ mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })
+ mocks.getNamespacedCustomObject.mockRejectedValue(
+ Object.assign(new Error('not found'), { code: 404 })
+ )
+ mocks.createNamespacedNetworkPolicy.mockResolvedValue({})
+ mocks.replaceNamespacedNetworkPolicy.mockResolvedValue({})
+ mocks.readNamespacedNetworkPolicy.mockResolvedValue(stalePolicy)
+ watcher = new McpServerWatcher()
+ server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
+ watcher.isReadinessInventoryAuthoritative()
+ )
+ await server.start()
+ server.setReady(true)
+ })
+
+ afterEach(async () => {
+ await server.stop()
+ await watcher.stop()
+ })
+
+ it('turns /ready from 200 to 503 when the additive phase loses a delete fence, and back on a clean pass', async () => {
+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
+ alignRevocationCounters(watcher)
+
+ // Baseline: nothing stale anywhere, so the pass certifies and the gate opens.
+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ await reconciler.fullReconcile([alphaContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ alignRevocationCounters(watcher)
+ expect(await readyStatus(server)).toBe(200)
+
+ // The stale allow only becomes visible after the authoritative phase has
+ // certified, so the safety phase is clean by construction and the additive
+ // re-LIST is what finds it. Its delete then loses the uid/resourceVersion
+ // precondition — a 409, exactly what a concurrent writer produces.
+ let certified = false
+ mocks.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace }: { namespace?: string }) => ({
+ items: certified && namespace === 'mcp-server' ? [stalePolicy] : [],
+ })
+ )
+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('the object has been modified'), { code: 409 })
+ )
+
+ await expect(
+ reconciler.fullReconcile([alphaContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {
+ certified = true
+ },
+ })
+ ).rejects.toThrow()
+
+ // Nothing moved: no watch generation bumped, no desired revision changed.
+ // Only the fence knows this pass left an allow it classified as stale.
+ alignRevocationCounters(watcher)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ expect(await readyStatus(server)).toBe(503)
+
+ // The retry lands: the allow is gone and the pass certifies again.
+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ await reconciler.fullReconcile([alphaContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ alignRevocationCounters(watcher)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
+ expect(await readyStatus(server)).toBe(200)
+ })
+
+ it('withholds readiness when the authoritative pass loses a delete fence under a provided safety snapshot', async () => {
+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
+ alignRevocationCounters(watcher)
+
+ // Baseline: a clean authoritative pass certifies and the gate opens.
+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ await reconciler.fullReconcile([alphaContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ alignRevocationCounters(watcher)
+ expect(await readyStatus(server)).toBe(200)
+
+ // Unlike the additive-phase test above, the stale allow is visible to the
+ // authoritative LISTs from the very start of the pass, so fullReconcile
+ // classifies it into the safety snapshot it provides to the revocation
+ // lane. Its identity-bound delete then loses the uid/resourceVersion
+ // precondition (409). The external-egress selector stays empty so the
+ // fence is lost in the provided-snapshot lane and nowhere else.
+ mocks.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
+ items:
+ namespace === 'mcp-server' && !(labelSelector ?? '').includes('external-egress')
+ ? [stalePolicy]
+ : [],
+ })
+ )
+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('the object has been modified'), { code: 409 })
+ )
+
+ // The guarantee under test is that the fence degrades at the point of
+ // loss, while the condemned pass is still unwinding — not only when the
+ // pass finally reports. Sample it through the authority callback the pass
+ // keeps polling: every sample before the lost delete must be certified,
+ // and every later sample must already be withdrawn.
+ let certifications = 0
+ const fenceSeenDuringPass: boolean[] = []
+ await expect(
+ reconciler.fullReconcile([alphaContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => {
+ fenceSeenDuringPass.push(reconciler.hasCertifiedSafetyInventory())
+ return true
+ },
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {
+ certifications += 1
+ },
+ })
+ ).rejects.toThrow()
+
+ expect(certifications).toBe(0)
+ expect(fenceSeenDuringPass.length).toBeGreaterThan(1)
+ expect(fenceSeenDuringPass[0]).toBe(true)
+ expect(fenceSeenDuringPass[fenceSeenDuringPass.length - 1]).toBe(false)
+
+ alignRevocationCounters(watcher)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ expect(await readyStatus(server)).toBe(503)
+
+ // A clean authoritative pass re-certifies and reopens the gate.
+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ await reconciler.fullReconcile([alphaContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ alignRevocationCounters(watcher)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
+ expect(await readyStatus(server)).toBe(200)
+ })
+
+ it('short-circuits a scoped delta at the certificate source while the last authoritative pass left the inventory uncertified', async () => {
+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
+ const w = watcher as unknown as Record<string, any>
+
+ // Register the real Context watch so the production watch handler — the
+ // scoped-delta certification lane — can be driven through its callback.
+ await w.startContextWatch('ctx-rv-1')
+ const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>
+ String(path).endsWith('/namespaces/mcp-server/contexts')
+ )
+ if (!contextWatchCall) throw new Error('Context watch was never registered')
+ const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>
+
+ // Seed the prior Context so the MODIFIED event below is a same-identity
+ // delta: same uid, same contextId, new generation.
+ const priorContext = {
+ name: 'alpha',
+ namespace: 'mcp-server',
+ uid: 'ctx-alpha-uid',
+ generation: 1,
+ spec: { contextId: 'alpha', mcpServers: [] },
+ }
+ w.contexts.set('alpha', priorContext)
+ alignRevocationCounters(watcher)
+
+ // Baseline clean authoritative pass: certified, gate open.
+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ await reconciler.fullReconcile([priorContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ alignRevocationCounters(watcher)
+ expect(await readyStatus(server)).toBe(200)
+
+ // The authoritative pass loses its delete fence under the provided
+ // snapshot, exactly as in the test above: the broad safety LISTs see the
+ // stale allow and its delete keeps 409ing. Every context-scoped LIST
+ // stays empty so the scoped delta fired below completes its own
+ // revocation cleanly — and because the 409 never clears, the convergence
+ // pass the watch event kicks off in the background can never certify
+ // either.
+ mocks.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ const selector = labelSelector ?? ''
+ const scoped =
+ selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')
+ return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }
+ }
+ )
+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('the object has been modified'), { code: 409 })
+ )
+ await expect(
+ reconciler.fullReconcile([priorContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ ).rejects.toThrow()
+ alignRevocationCounters(watcher)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ expect(await readyStatus(server)).toBe(503)
+
+ // Fire a same-identity MODIFIED delta. Its scoped revocation completes
+ // (its label-scoped LISTs are clean) and it carries a delta certificate,
+ // so the only thing standing between it and
+ // recordNetworkPolicySafetyCertificate is the uncertified fence left by
+ // the authoritative pass.
+ const warnSpy = vi.spyOn(console, 'warn')
+ try {
+ await fireContextEvent('MODIFIED', {
+ metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },
+ spec: { contextId: 'alpha', mcpServers: [] },
+ })
+
+ // M1: the fence was lost before this delta, so
+ // currentNetworkPolicySafetyCertificate() returns null at the capture and no
+ // delta certificate is ever minted — the delta short-circuits at the source
+ // and never reaches the per-delta certification branch. No warn is emitted;
+ // nothing is recorded; readiness stays withdrawn via the independent
+ // readiness fence even after every counter is realigned.
+ expect(
+ warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))
+ ).toBe(false)
+ expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ alignRevocationCounters(watcher)
+ expect(await readyStatus(server)).toBe(503)
+ } finally {
+ warnSpy.mockRestore()
+ }
+ })
+
+ it('declines to certify a scoped delta whose certificate outlived a fence lost mid-flight, and warns', async () => {
+ // Coverage preservation for the per-delta fence check (k8sClient.ts :3801):
+ // the delta certificate is minted while the fence is still certified, then a
+ // CONCURRENT authoritative pass loses its delete fence mid-flight. M1's
+ // source short-circuit cannot catch this — only the per-delta check can. This
+ // test is green with AND without M1; it guards the TOCTOU window, not M1.
+ const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
+ const w = watcher as unknown as Record<string, any>
+
+ await w.startContextWatch('ctx-rv-2')
+ const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>
+ String(path).endsWith('/namespaces/mcp-server/contexts')
+ )
+ if (!contextWatchCall) throw new Error('Context watch was never registered')
+ const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>
+
+ const priorContext = {
+ name: 'alpha',
+ namespace: 'mcp-server',
+ uid: 'ctx-alpha-uid',
+ generation: 1,
+ spec: { contextId: 'alpha', mcpServers: [] },
+ }
+ w.contexts.set('alpha', priorContext)
+ alignRevocationCounters(watcher)
+
+ // Baseline clean authoritative pass: certified, gate open.
+ mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
+ mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
+ await reconciler.fullReconcile([priorContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ alignRevocationCounters(watcher)
+ expect(await readyStatus(server)).toBe(200)
+
+ // The delta certificate is captured now, while the fence is still certified.
+ // The FIRST scoped LIST the delta issues runs a concurrent authoritative pass
+ // that loses its own delete fence — degrading the fence AFTER the delta minted
+ // its certificate but BEFORE the delta reaches its certification branch.
+ let fenceBrokenMidFlight = false
+ mocks.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
+ const selector = labelSelector ?? ''
+ const scoped =
+ selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')
+ if (scoped && !fenceBrokenMidFlight) {
+ fenceBrokenMidFlight = true
+ mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
+ Object.assign(new Error('the object has been modified'), { code: 409 })
+ )
+ await reconciler
+ .fullReconcile([priorContext], [], {
+ ensureDefaults: false,
+ contextInventoryAuthoritative: () => true,
+ serverInventoryAuthoritative: () => true,
+ onAuthoritativeRevocationComplete: () => {},
+ })
+ .then(
+ () => {
+ throw new Error('interleaved authoritative pass should have lost its fence')
+ },
+ () => {}
+ )
+ }
+ return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }
+ }
+ )
+ const warnSpy = vi.spyOn(console, 'warn')
+ try {
+ await fireContextEvent('MODIFIED', {
+ metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },
+ spec: { contextId: 'alpha', mcpServers: [] },
+ })
+
+ // The interleave actually ran (no vacuous pass) and the per-delta check
+ // withheld certification: it warned, recorded nothing, readiness stays 503.
+ expect(fenceBrokenMidFlight).toBe(true)
+ expect(
+ warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))
+ ).toBe(true)
+ expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)
+ expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
+ alignRevocationCounters(watcher)
+ expect(await readyStatus(server)).toBe(503)
+ } finally {
+ warnSpy.mockRestore()
+ }
+ })
+})
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index 92e74d6e8..26a7f5ab0 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import {
asAppsApi,
@@ -14,6 +14,17 @@ import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './const
import { McpServerReconciler } from './reconciler'
import { McpServerCRD } from './types'
+function deferred<T = void>(): {
+ promise: Promise<T>
+ resolve: (value: T | PromiseLike<T>) => void
+} {
+ let resolve!: (value: T | PromiseLike<T>) => void
+ const promise = new Promise<T>(res => {
+ resolve = res
+ })
+ return { promise, resolve }
+}
+
/**
* Tests for the managed:false guard in the reconciler (Risk 1.7).
*
@@ -30,6 +41,7 @@ vi.mock('./config', () => ({
controlPlaneNamespace: 'control-plane',
hostNamespace: 'mcp-host',
rpcProxyNamespace: 'rpc-proxy',
+ egressProxyImage: 'clerum/nginx-egress-proxy:test',
devMcpServers: [],
devContexts: [],
devAuthTokens: new Map(),
@@ -64,6 +76,7 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -155,6 +168,55 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
})
+ it('completes a remote proxy reconcile without rewriting the desired image', async () => {
+ const server = makeServer({
+ name: 'remote-api',
+ image: 'vendor/original-image:1',
+ remote: { baseUrl: 'https://api.example.com' },
+ })
+ const originalSpec = structuredClone(server.spec)
+ let desiredRevisionIsCurrent = true
+ customApi.patchNamespacedCustomObject.mockImplementation(
+ ({ body }: { body?: { spec?: unknown } }) => {
+ if (body?.spec !== undefined) {
+ desiredRevisionIsCurrent = false
+ }
+ return Promise.resolve({})
+ }
+ )
+
+ await reconciler.reconcile(server, {
+ isCurrent: () => desiredRevisionIsCurrent,
+ })
+
+ expect(server.spec).toEqual(originalSpec)
+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(
+ expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),
+ expect.anything()
+ )
+ expect(coreApi.createNamespacedConfigMap).toHaveBeenCalledOnce()
+ expect(coreApi.createNamespacedService).toHaveBeenCalledOnce()
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledOnce()
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(
+ expect.objectContaining({
+ body: expect.objectContaining({
+ spec: expect.objectContaining({
+ template: expect.objectContaining({
+ spec: expect.objectContaining({
+ containers: [
+ expect.objectContaining({
+ name: 'egress-proxy',
+ image: 'clerum/nginx-egress-proxy:test',
+ }),
+ ],
+ }),
+ }),
+ }),
+ }),
+ })
+ )
+ })
+
it('should log WRC-owned ownership when managed: false', async () => {
const consoleSpy = vi.spyOn(console, 'log')
const server = makeServer({ name: 'workflow-recipes', managed: false })
@@ -223,6 +285,7 @@ describe('PR-B B1 — validateSecret result shape', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -407,7 +470,12 @@ describe('PR-B B1 — writeStatusCondition', () => {
const getPatchedConditions = () => {
const patch = customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0]
- const op = patch.body[0]
+ const op = patch.body.find(
+ (candidate: { op?: string; path?: string }) =>
+ candidate.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ if (!op) throw new Error('status condition add operation was not emitted')
const conditions = op.path === '/status' ? op.value.conditions : op.value
return { patch, op, conditions }
}
@@ -415,6 +483,7 @@ describe('PR-B B1 — writeStatusCondition', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -509,6 +578,216 @@ describe('PR-B B1 — writeStatusCondition', () => {
expect(types).toContain('Ready')
})
+ it('records the reconciled McpServer generation on status conditions', async () => {
+ const server = {
+ ...makeServer({ name: 'pg' }),
+ generation: 17,
+ }
+
+ await reconciler.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ })
+
+ const { conditions } = getPatchedConditions()
+ expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(
+ expect.objectContaining({
+ status: 'True',
+ observedGeneration: 17,
+ })
+ )
+ })
+
+ it('re-reads and preserves a concurrent condition after a resourceVersion conflict', async () => {
+ const networkReady = {
+ type: 'NetworkReady',
+ status: 'True',
+ reason: 'NetworkPoliciesApplied',
+ message: 'network ready',
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ }
+ const externalEgressReady = {
+ type: 'ExternalEgressReady',
+ status: 'True',
+ reason: 'EgressPolicyApplied',
+ message: 'external egress ready',
+ lastTransitionTime: '2020-01-02T00:00:00.000Z',
+ }
+ customApi.getNamespacedCustomObjectStatus
+ .mockResolvedValueOnce({
+ metadata: { resourceVersion: '100' },
+ status: { conditions: [networkReady] },
+ })
+ .mockResolvedValueOnce({
+ metadata: { resourceVersion: '101' },
+ status: { conditions: [networkReady, externalEgressReady] },
+ })
+ customApi.patchNamespacedCustomObjectStatus
+ .mockRejectedValueOnce(Object.assign(new Error('resourceVersion conflict'), { code: 409 }))
+ .mockResolvedValueOnce({})
+
+ const server: McpServerCRD = {
+ ...makeServer({ name: 'pg' }),
+ uid: 'uid-pg',
+ generation: 17,
+ }
+ await reconciler.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ })
+
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
+ expect(customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body).toContainEqual({
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: '100',
+ })
+ const secondPatch = customApi.patchNamespacedCustomObjectStatus.mock.calls[1][0].body
+ expect(secondPatch).toContainEqual({
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: '101',
+ })
+ const conditionPatch = secondPatch.find(
+ (candidate: { op?: string; path?: string }) =>
+ candidate.op === 'add' && candidate.path === '/status/conditions'
+ )
+ expect(conditionPatch?.value.map((written: { type: string }) => written.type)).toEqual([
+ 'NetworkReady',
+ 'ExternalEgressReady',
+ 'Ready',
+ ])
+ })
+
+ it('bounds resourceVersion conflict retries', async () => {
+ const currentStatus = {
+ metadata: { resourceVersion: '100' },
+ status: { conditions: [] },
+ }
+ customApi.getNamespacedCustomObjectStatus
+ .mockResolvedValueOnce(currentStatus)
+ .mockResolvedValueOnce(currentStatus)
+ .mockResolvedValueOnce(currentStatus)
+ const conflict = Object.assign(new Error('resourceVersion conflict'), { code: 422 })
+ customApi.patchNamespacedCustomObjectStatus
+ .mockRejectedValueOnce(conflict)
+ .mockRejectedValueOnce(conflict)
+ .mockRejectedValueOnce(conflict)
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+
+ await reconciler.writeStatusCondition(
+ {
+ ...makeServer({ name: 'pg' }),
+ uid: 'uid-pg',
+ generation: 17,
+ },
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ }
+ )
+
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining('Failed to write status condition Ready=True'),
+ conflict
+ )
+ warn.mockRestore()
+ })
+
+ it('guards the status patch with the captured UID, generation, and resourceVersion', async () => {
+ const staleServer: McpServerCRD = {
+ ...makeServer({ name: 'recreated' }),
+ uid: 'uid-before-recreation',
+ generation: 1,
+ }
+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ metadata: { resourceVersion: '100' },
+ status: { conditions: [] },
+ })
+ const writeFailure = Object.assign(new Error('status API unavailable'), {
+ code: 500,
+ })
+ customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async request => {
+ expect(request.body.slice(0, 3)).toEqual([
+ {
+ op: 'test',
+ path: '/metadata/uid',
+ value: 'uid-before-recreation',
+ },
+ {
+ op: 'test',
+ path: '/metadata/generation',
+ value: 1,
+ },
+ {
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: '100',
+ },
+ ])
+ throw writeFailure
+ })
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+
+ await reconciler.writeStatusCondition(staleServer, {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'stale reconcile result',
+ })
+
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining('Failed to write status condition Ready=True'),
+ writeFailure
+ )
+ warn.mockRestore()
+ })
+
+ it('refreshes observedGeneration even when the condition payload is otherwise unchanged', async () => {
+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ status: {
+ conditions: [
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ observedGeneration: 16,
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ },
+ ],
+ },
+ })
+ const server = {
+ ...makeServer({ name: 'pg' }),
+ generation: 17,
+ }
+
+ await reconciler.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ })
+
+ const { conditions } = getPatchedConditions()
+ expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(
+ expect.objectContaining({
+ observedGeneration: 17,
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ })
+ )
+ })
+
it('creates the status object when the CRD has no status yet', async () => {
customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({})
@@ -653,6 +932,170 @@ describe('PR-B B1 — writeStatusCondition', () => {
})
})
+describe('restart-safe discovery status', () => {
+ const appsApi = createMockAppsApi()
+ const coreApi = createMockCoreApi()
+ const customApi = createMockCustomApi()
+ let reconciler: McpServerReconciler
+
+ beforeEach(() => {
+ vi.clearAllMocks()
+ reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(coreApi),
+ customApi: asCustomApi(customApi),
+ })
+ })
+
+ it('uses a persisted Ready condition only when it observes the current generation', () => {
+ const server: McpServerCRD = {
+ ...makeServer({ name: 'restart-safe' }),
+ generation: 7,
+ status: {
+ conditions: [
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'Deployment created',
+ observedGeneration: 7,
+ },
+ ],
+ },
+ }
+
+ expect(reconciler.getStatus(server)).toEqual({
+ deployed: true,
+ ready: true,
+ message: 'Deployment created',
+ authoritative: true,
+ })
+ })
+
+ it.each([
+ ['stale', 6],
+ ['unversioned', undefined],
+ ])('fails closed for a %s persisted Ready condition', (_label, observedGeneration) => {
+ const server: McpServerCRD = {
+ ...makeServer({ name: 'restart-unsafe' }),
+ generation: 7,
+ status: {
+ conditions: [
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'Deployment created',
+ observedGeneration,
+ },
+ ],
+ },
+ }
+
+ expect(reconciler.getStatus(server)).toEqual({
+ deployed: false,
+ ready: false,
+ message: 'Not reconciled yet',
+ authoritative: false,
+ })
+ })
+
+ it('fails closed when persisted Ready conditions conflict for the current generation', () => {
+ const server: McpServerCRD = {
+ ...makeServer({ name: 'restart-conflict' }),
+ generation: 7,
+ status: {
+ conditions: [
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ observedGeneration: 7,
+ },
+ {
+ type: 'Ready',
+ status: 'False',
+ reason: 'SecretValidationFailed',
+ observedGeneration: 7,
+ },
+ ],
+ },
+ }
+
+ expect(reconciler.getStatus(server)).toEqual({
+ deployed: false,
+ ready: false,
+ message: 'Not reconciled yet',
+ authoritative: false,
+ })
+ })
+
+ it('prefers fresh in-memory status over persisted status from the same generation', async () => {
+ const server: McpServerCRD = {
+ ...makeServer({ name: 'live-status' }),
+ generation: 3,
+ status: {
+ conditions: [
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ observedGeneration: 3,
+ },
+ ],
+ },
+ }
+ appsApi.readNamespacedDeployment.mockResolvedValueOnce({
+ status: { readyReplicas: 0 },
+ })
+
+ await reconciler.reconcile(server)
+
+ expect(reconciler.getStatus(server)).toEqual(
+ expect.objectContaining({
+ deployed: true,
+ ready: false,
+ authoritative: true,
+ })
+ )
+ })
+
+ it.each([
+ ['generation changes', 'stable-uid', 4],
+ ['the same name is recreated', 'replacement-uid', 3],
+ ])('does not reuse in-memory status when %s', async (_case, uid, generation) => {
+ const original: McpServerCRD = {
+ ...makeServer({ name: 'identity-fenced' }),
+ uid: 'original-uid',
+ generation: 3,
+ }
+ await reconciler.reconcile(original)
+ // #223 (credential rotation) makes readiness immediately False until the poll
+ // upgrades it, so the original caches deployed/authoritative here — not ready.
+ // This test's subject is the identity fencing asserted below, not readiness
+ // timing; it only needs a real (non-default) cached status to prove it is not
+ // reused across an identity change.
+ expect(reconciler.getStatus(original)).toEqual(
+ expect.objectContaining({ deployed: true, ready: false, authoritative: true })
+ )
+
+ const current: McpServerCRD = {
+ ...original,
+ uid,
+ generation,
+ status: undefined,
+ }
+
+ expect(reconciler.getStatus(current)).toEqual({
+ deployed: false,
+ ready: false,
+ message: 'Not reconciled yet',
+ authoritative: false,
+ })
+ })
+})
+
describe('ownership-safe fail-closed cleanup', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
@@ -662,6 +1105,7 @@ describe('ownership-safe fail-closed cleanup', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -764,6 +1208,480 @@ describe('ownership-safe fail-closed cleanup', () => {
})
})
+describe('full reconciliation inventory authority', () => {
+ const appsApi = createMockAppsApi()
+ const coreApi = createMockCoreApi()
+ const customApi = createMockCustomApi()
+ let reconciler: McpServerReconciler
+
+ beforeEach(() => {
+ vi.clearAllMocks()
+ customApi.getNamespacedCustomObject.mockRejectedValue(
+ Object.assign(new Error('not found'), { code: 404 })
+ )
+ reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(coreApi),
+ customApi: asCustomApi(customApi),
+ })
+ })
+
+ it('rejects when Deployment inventory LIST fails so startup retry remains possible', async () => {
+ const inventoryError = new Error('deployment inventory unavailable')
+ const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ appsApi.listNamespacedDeployment.mockRejectedValueOnce(inventoryError)
+
+ await expect(reconciler.fullReconcile([])).rejects.toBe(inventoryError)
+
+ expect(error).toHaveBeenCalledWith(
+ '[Reconciler] Failed to list managed deployments:',
+ inventoryError
+ )
+ error.mockRestore()
+ })
+
+ it('defaults to fail-closed orphan cleanup until an authority source is wired', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ },
+ },
+ ],
+ })
+
+ await reconciler.fullReconcile([])
+
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))
+ warn.mockRestore()
+ })
+
+ it('finishes desired reconciliation but skips orphan cleanup if inventory authority is lost mid-pass', async () => {
+ let authority = { known: true, generation: 7 }
+ reconciler.setInventoryAuthority(() => authority)
+ const desired = makeServer({ name: 'desired-server' })
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {
+ authority = { known: false, generation: 8 }
+ })
+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ },
+ },
+ ],
+ })
+
+ await reconciler.fullReconcile([desired])
+
+ expect(reconcile).toHaveBeenCalledOnce()
+ expect(reconcile).toHaveBeenCalledWith(desired, {
+ isCurrent: expect.any(Function),
+ })
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))
+ warn.mockRestore()
+ })
+
+ it('skips orphan cleanup if the authoritative inventory generation changes mid-pass', async () => {
+ let authority = { known: true, generation: 11 }
+ reconciler.setInventoryAuthority(() => authority)
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {
+ authority = { known: true, generation: 12 }
+ })
+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ },
+ },
+ ],
+ })
+
+ await reconciler.fullReconcile([makeServer({ name: 'desired-server' })])
+
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))
+ warn.mockRestore()
+ })
+
+ it('deletes an orphan when inventory authority remains known and stable', async () => {
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 21 }))
+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ },
+ },
+ ],
+ })
+
+ await reconciler.fullReconcile([])
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ })
+ expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ })
+ })
+
+ it('skips a startup runtime revision superseded before its keyed effect is admitted', async () => {
+ const snapshot: McpServerCRD = {
+ ...makeServer({ name: 'updated-server' }),
+ generation: 1,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/updated-server:old',
+ transport: { type: 'streamableHttp', port: 3000 },
+ },
+ }
+ const current: McpServerCRD = {
+ ...makeServer({ name: 'updated-server' }),
+ generation: 2,
+ spec: {
+ contextRef: 'default',
+ image: 'clerum/updated-server:new',
+ transport: { type: 'streamableHttp', port: 3000 },
+ egressBindings: [{ dns: 'new.example', port: 443 }],
+ },
+ }
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 40 }))
+ reconciler.setResolveCurrentServer(() => current)
+ const reconcile = vi.spyOn(reconciler, 'reconcile')
+
+ await reconciler.fullReconcile([snapshot])
+
+ expect(reconcile).not.toHaveBeenCalled()
+ })
+
+ it('does not mutate runtime or status after startup authority retires during Secret validation', async () => {
+ let authority = { known: true, generation: 51 }
+ const server = {
+ ...makeServer({
+ name: 'secret-authority-race',
+ envSecret: {
+ name: 'runtime-secret',
+ keys: [{ secretKey: 'token', envVar: 'TOKEN' }],
+ },
+ }),
+ uid: 'secret-authority-race-uid',
+ generation: 4,
+ }
+ reconciler.setInventoryAuthority(() => authority)
+ reconciler.setResolveCurrentServer(name => (name === server.name ? server : undefined))
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred<{ data: Record<string, string> }>()
+ coreApi.readNamespacedSecret.mockImplementationOnce(async () => {
+ secretReadStarted.resolve(undefined)
+ return releaseSecretRead.promise
+ })
+
+ const convergence = reconciler.fullReconcile([server])
+ await secretReadStarted.promise
+
+ authority = { known: true, generation: 53 }
+ releaseSecretRead.resolve({ data: { token: 'encoded-token' } })
+ await convergence
+
+ expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ })
+
+ it('retires a same-watch runtime revision superseded during Secret validation', async () => {
+ let current: McpServerCRD
+ const original: McpServerCRD = {
+ ...makeServer({
+ name: 'same-watch-runtime',
+ image: 'clerum/same-watch-runtime:v1',
+ envSecret: {
+ name: 'runtime-secret',
+ keys: [{ secretKey: 'token', envVar: 'TOKEN' }],
+ },
+ }),
+ uid: 'same-watch-runtime-uid',
+ generation: 1,
+ }
+ const replacement: McpServerCRD = {
+ ...original,
+ generation: 2,
+ spec: {
+ ...original.spec,
+ image: 'clerum/same-watch-runtime:v2',
+ },
+ }
+ current = original
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 61 }))
+ reconciler.setResolveCurrentServer(name => (name === original.name ? current : undefined))
+
+ const originalSecretReadStarted = deferred()
+ const releaseOriginalSecretRead = deferred<{ data: Record<string, string> }>()
+ coreApi.readNamespacedSecret
+ .mockImplementationOnce(async () => {
+ originalSecretReadStarted.resolve(undefined)
+ return releaseOriginalSecretRead.promise
+ })
+ .mockResolvedValueOnce({ data: { token: 'encoded-token' } })
+
+ const originalReconcile = reconciler.reconcile(original)
+ await originalSecretReadStarted.promise
+ current = replacement
+ const replacementReconcile = reconciler.reconcile(replacement)
+ releaseOriginalSecretRead.resolve({ data: { token: 'encoded-token' } })
+
+ await Promise.all([originalReconcile, replacementReconcile])
+
+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith({
+ namespace: replacement.namespace,
+ body: expect.objectContaining({
+ spec: expect.objectContaining({
+ template: expect.objectContaining({
+ spec: expect.objectContaining({
+ containers: expect.arrayContaining([
+ expect.objectContaining({
+ image: 'clerum/same-watch-runtime:v2',
+ }),
+ ]),
+ }),
+ }),
+ }),
+ }),
+ })
+ })
+
+ it('admits independent desired effects while another server effect is blocked', async () => {
+ const blocked = makeServer({ name: 'blocked-server' })
+ const independent = makeServer({ name: 'independent-server' })
+ let markBlockedStarted!: () => void
+ let releaseBlocked!: () => void
+ const blockedStarted = new Promise<void>(resolve => {
+ markBlockedStarted = resolve
+ })
+ const blockedRelease = new Promise<void>(resolve => {
+ releaseBlocked = resolve
+ })
+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
+ const fullReconcile = reconciler.fullReconcile([blocked, independent], {
+ runEffect: async (serverName, work) => {
+ if (serverName === blocked.name) {
+ markBlockedStarted()
+ await blockedRelease
+ }
+ await work()
+ },
+ })
+
+ await blockedStarted
+ await Promise.resolve()
+
+ expect(reconcile).toHaveBeenCalledWith(independent, {
+ isCurrent: expect.any(Function),
+ })
+ expect(reconcile.mock.calls.some(([server]) => server === blocked)).toBe(false)
+
+ releaseBlocked()
+ await fullReconcile
+
+ expect(reconcile).toHaveBeenCalledWith(blocked, {
+ isCurrent: expect.any(Function),
+ })
+ })
+
+ it('bounds desired-effect admission without serializing independent siblings', async () => {
+ const desired = Array.from({ length: 25 }, (_, index) =>
+ makeServer({ name: `bounded-server-${index}` })
+ )
+ let active = 0
+ let maxActive = 0
+ let startedBeforeRelease = 0
+ let releaseAll!: () => void
+ const release = new Promise<void>(resolve => {
+ releaseAll = resolve
+ })
+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
+ const options = {
+ maxConcurrency: 3,
+ runEffect: async (_serverName: string, work: () => Promise<void>) => {
+ active += 1
+ startedBeforeRelease += 1
+ maxActive = Math.max(maxActive, active)
+ await release
+ try {
+ await work()
+ } finally {
+ active -= 1
+ }
+ },
+ }
+
+ const fullReconcile = reconciler.fullReconcile(desired, options)
+ await vi.waitFor(() => expect(active).toBeGreaterThanOrEqual(3))
+ await Promise.resolve()
+ const observedMaxActive = maxActive
+ const observedStartedBeforeRelease = startedBeforeRelease
+
+ releaseAll()
+ await fullReconcile
+
+ expect(observedMaxActive).toBe(3)
+ expect(observedStartedBeforeRelease).toBe(3)
+ expect(reconcile).toHaveBeenCalledTimes(desired.length)
+ })
+
+ it('waits for every admitted desired effect before reporting a sibling failure', async () => {
+ const failed = makeServer({ name: 'failed-server' })
+ const slow = makeServer({ name: 'slow-server' })
+ let markSlowStarted!: () => void
+ let releaseSlow!: () => void
+ const slowStarted = new Promise<void>(resolve => {
+ markSlowStarted = resolve
+ })
+ const slowRelease = new Promise<void>(resolve => {
+ releaseSlow = resolve
+ })
+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
+
+ const fullReconcile = reconciler.fullReconcile([failed, slow], {
+ runEffect: async (serverName, work) => {
+ if (serverName === failed.name) {
+ throw new Error('failed server gate')
+ }
+ markSlowStarted()
+ await slowRelease
+ await work()
+ },
+ })
+ let settled = false
+ const observed = fullReconcile.then(
+ () => undefined,
+ error => error
+ )
+ void observed.then(() => {
+ settled = true
+ })
+
+ await slowStarted
+ await Promise.resolve()
+ await Promise.resolve()
+
+ expect(settled).toBe(false)
+
+ releaseSlow()
+ const error = await observed
+
+ expect(error).toBeInstanceOf(AggregateError)
+ expect(error).toMatchObject({
+ message: 'Failed to reconcile desired McpServers',
+ })
+ expect(reconcile).toHaveBeenCalledWith(slow, {
+ isCurrent: expect.any(Function),
+ })
+ })
+
+ it('aggregates every desired-effect failure after all bounded workers settle', async () => {
+ const failures = [
+ new Error('first desired failure'),
+ new Error('second desired failure'),
+ new Error('third desired failure'),
+ ]
+ const desired = failures.map((_, index) => makeServer({ name: `failed-${index}` }))
+ const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
+
+ const error = await reconciler
+ .fullReconcile(desired, {
+ maxConcurrency: 2,
+ runEffect: async serverName => {
+ throw failures[Number(serverName.replace('failed-', ''))]
+ },
+ })
+ .then(
+ () => undefined,
+ reason => reason
+ )
+
+ expect(error).toBeInstanceOf(AggregateError)
+ expect((error as AggregateError).errors).toEqual(failures)
+ expect(reconcile).not.toHaveBeenCalled()
+ })
+
+ it('does not delete an orphan candidate that was added to the same-generation cache mid-pass', async () => {
+ const currentServers = new Map<string, McpServerCRD>()
+ ;(reconciler as any).setResolveCurrentServer((name: string) => currentServers.get(name))
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 31 }))
+ appsApi.listNamespacedDeployment.mockImplementationOnce(async () => {
+ currentServers.set('orphan-server', makeServer({ name: 'orphan-server' }))
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ },
+ },
+ ],
+ }
+ })
+
+ await reconciler.fullReconcile([])
+
+ expect(customApi.getNamespacedCustomObject).not.toHaveBeenCalled()
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ })
+
+ it('does not delete an orphan candidate when a fresh API GET sees a same-name CRD before ADDED arrives', async () => {
+ ;(reconciler as any).setResolveCurrentServer(() => undefined)
+ reconciler.setInventoryAuthority(() => ({ known: true, generation: 32 }))
+ customApi.getNamespacedCustomObject.mockResolvedValueOnce({
+ metadata: { name: 'orphan-server', namespace: 'mcp-server', uid: 'recreated-uid' },
+ })
+ appsApi.listNamespacedDeployment.mockResolvedValueOnce({
+ items: [
+ {
+ metadata: {
+ name: 'orphan-server',
+ namespace: 'mcp-server',
+ },
+ },
+ ],
+ })
+
+ await reconciler.fullReconcile([])
+
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledWith(
+ expect.objectContaining({
+ namespace: 'mcp-server',
+ plural: 'mcpservers',
+ name: 'orphan-server',
+ })
+ )
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ })
+})
+
describe('updateStatusConditions', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
@@ -773,6 +1691,7 @@ describe('updateStatusConditions', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -835,3 +1754,53 @@ describe('updateStatusConditions', () => {
expect(types).toContain('DeploymentReady')
})
})
+
+describe('unconfigured inventory authority fails closed (G1)', () => {
+ const appsApi = createMockAppsApi()
+ const coreApi = createMockCoreApi()
+ const customApi = createMockCustomApi()
+ let errorSpy: ReturnType<typeof vi.spyOn>
+
+ // Deliberately NO assumeInventoryAuthorityWhenUnconfigured and NO
+ // setInventoryAuthority/setResolveCurrentServer wiring: the fence must deny.
+ function makeUnwiredReconciler(): McpServerReconciler {
+ return new McpServerReconciler({} as k8s.KubeConfig, {
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(coreApi),
+ customApi: asCustomApi(customApi),
+ })
+ }
+
+ beforeEach(() => {
+ vi.clearAllMocks()
+ errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ })
+
+ afterEach(() => {
+ errorSpy.mockRestore()
+ })
+
+ it('T-G1a: reconcile() denies (no Deployment) and warns once when authority is unwired', async () => {
+ const reconciler = makeUnwiredReconciler()
+ await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('fail closed'))
+ })
+
+ it('T-G1b: fullReconcile() denies when authority is unwired', async () => {
+ const reconciler = makeUnwiredReconciler()
+ await reconciler.fullReconcile([makeServer({ name: 'g1-mcp', managed: true })])
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ })
+
+ it('T-G1c: the explicit opt-in restores assume-current', async () => {
+ const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(coreApi),
+ customApi: asCustomApi(customApi),
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ })
+ await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))
+ expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
+ })
+})
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index caba6be91..4056c621c 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -16,6 +16,10 @@ import { classifyPluginImage } from '@clerum/image-policy'
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
import { config } from './config'
import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'
+import {
+ confirmAuthoritativeMcpServerAbsence,
+ sameMcpServerDesiredRevision,
+} from './mcpServerSafety'
import { mcpserverMissingSecret } from './metrics'
import { McpServerCRD, McpServerStatus } from './types'
import {
@@ -62,6 +66,7 @@ export interface McpServerCondition {
reason: string
message: string
lastTransitionTime: string
+ observedGeneration?: number
}
type McpServerReconcilerDeps = {
@@ -69,6 +74,28 @@ type McpServerReconcilerDeps = {
coreApi?: k8s.CoreV1Api
customApi?: k8s.CustomObjectsApi
networkingApi?: k8s.NetworkingV1Api
+ /**
+ * TEST-ONLY escape hatch. When the inventory-authority fence is not wired
+ * (setInventoryAuthority / setResolveCurrentServer never called), mutation
+ * fences fail CLOSED by default. Unit tests that exercise the reconciler
+ * without a watcher opt in to the legacy assume-current behavior here.
+ * Production construction (k8sClient) must never set this.
+ */
+ assumeInventoryAuthorityWhenUnconfigured?: boolean
+}
+
+export type McpServerInventoryAuthoritySnapshot = {
+ known: boolean
+ generation: number
+}
+
+export type McpServerFullReconcileOptions = {
+ runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
+ maxConcurrency?: number
+}
+
+export type McpServerMutationOptions = {
+ isCurrent?: () => boolean
}
// G2: Pre-deploy handshake constants
@@ -88,6 +115,13 @@ const HEADER_NAME_RE = /^[A-Za-z0-9-]+$/
// Bound the value length so a pathological CRD cannot blow up the ConfigMap.
const MAX_AUTH_HEADER_VALUE_LENGTH = 2048
const WRC_OWNED_RUNTIME_READY_REASON = 'WrcOwnedRuntimeRegistered'
+// Keep fleet admission aligned with the external-egress convergence lane:
+// enough parallelism for independent servers without allocating one pending
+// effect promise per fleet member.
+const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10
+// Status is a read/merge/write transaction shared with other controllers.
+// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.
+const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3
function throwCleanupFailures(failures: unknown[], message: string): void {
if (failures.length > 0) {
@@ -95,6 +129,37 @@ function throwCleanupFailures(failures: unknown[], message: string): void {
}
}
+async function mapSettledWithConcurrency<T, TResult>(
+ items: readonly T[],
+ maxConcurrency: number,
+ work: (item: T, index: number) => Promise<TResult>
+): Promise<Array<PromiseSettledResult<TResult>>> {
+ if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
+ throw new RangeError('maxConcurrency must be a positive integer')
+ }
+
+ const results = new Array<PromiseSettledResult<TResult>>(items.length)
+ let nextIndex = 0
+ const worker = async (): Promise<void> => {
+ while (nextIndex < items.length) {
+ const index = nextIndex
+ nextIndex += 1
+ try {
+ results[index] = {
+ status: 'fulfilled',
+ value: await work(items[index], index),
+ }
+ } catch (reason) {
+ results[index] = { status: 'rejected', reason }
+ }
+ }
+ }
+
+ const workerCount = Math.min(maxConcurrency, items.length)
+ await Promise.all(Array.from({ length: workerCount }, () => worker()))
+ return results
+}
+
/**
* State of the readiness poll window for one McpServer (issue #223).
*
@@ -146,6 +211,35 @@ interface ReadinessPollState {
* WaitingForReplicas by the very reconcile the terminal write triggers.
*/
detail: string
+ /**
+ * Mutation fence and CRD object of the LATEST reconcile that owns this
+ * window. pollReadiness() REBINDS both on every call that finds the window
+ * live: a superseding reconcile (e.g. the fullReconcile after a routine
+ * McpServer watch recovery bumps the inventory-authority generation) becomes
+ * the window's owner, so the next tick evaluates the CURRENT fence instead of
+ * the arming reconcile's stale closure. Without the rebind the stale fence
+ * self-terminates the window with no re-arm and — status-only MODIFIED events
+ * being suppressed, with no periodic McpServer resync — DeploymentReady=False
+ * sticks forever for a healthy rollout (the #223 rotation-timeout symptom,
+ * reintroduced under a routine reconnect).
+ *
+ * Two ORTHOGONAL supersession guards protect this window; do not assume one
+ * covers the other (that assumption is exactly how the F1 race was born):
+ * - Identity retirement: beginStatusTracking()/clearStatus() retire the
+ * window on any CRD uid or CRD-generation change (spec edit, recreate),
+ * so a surviving window always has the caller's status identity — a
+ * rebind can never adopt a recreate or a superseded desired revision.
+ * - In-tick generation reset + recheckRequested: a credential rotation
+ * bumps neither CRD uid nor CRD generation (it changes the Deployment's
+ * metadata.generation only), so identity retirement does NOT fire; the
+ * rotation is instead guarded by the tick's live Deployment read (a new
+ * generation resets attempts/exhausted) and by recheckRequested deferring
+ * a mid-flight tick's terminal verdict past the rotation's write.
+ * A rebind never touches the budget fields (generation/attempts/exhausted/
+ * detail) above.
+ */
+ isCurrent: () => boolean
+ server: McpServerCRD
}
export class McpServerReconciler {
@@ -154,8 +248,26 @@ export class McpServerReconciler {
private readonly customApi: k8s.CustomObjectsApi
private readonly networkingApi?: k8s.NetworkingV1Api
+ /**
+ * Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.
+ * The fail-closed default prevents orphan cleanup until the watcher wires
+ * the real authority source.
+ */
+ private inventoryAuthority: () => McpServerInventoryAuthoritySnapshot = () => ({
+ known: false,
+ generation: 0,
+ })
+ private inventoryAuthorityConfigured = false
+ private resolveCurrentServer?: (name: string) => McpServerCRD | undefined
+ /** See McpServerReconcilerDeps: test-only opt-in to assume-current when the
+ * inventory-authority fence is unwired. Production leaves it false. */
+ private readonly assumeInventoryAuthorityWhenUnconfigured: boolean
+ private warnedUnconfiguredAuthority = false
+
/** Tracks the deployment status of each managed McpServer. */
private readonly statusMap: Map<string, McpServerStatus> = new Map()
+ private readonly statusIdentities: Map<string, Pick<McpServerCRD, 'uid' | 'generation'>> =
+ new Map()
/** G7: Tracks the initial `managed` value per McpServer to enforce immutability. */
private readonly managedSnapshot: Map<string, boolean> = new Map()
@@ -190,20 +302,120 @@ export class McpServerReconciler {
this.networkingApi =
deps?.networkingApi ??
(typeof kc.makeApiClient === 'function' ? kc.makeApiClient(k8s.NetworkingV1Api) : undefined)
+ this.assumeInventoryAuthorityWhenUnconfigured =
+ deps?.assumeInventoryAuthorityWhenUnconfigured ?? false
+ }
+
+ /**
+ * Late-bound authority source for destructive full-reconcile cleanup.
+ * A full pass may finish reconciling desired servers, but it may delete an
+ * orphan only while the same authoritative inventory generation remains
+ * active from pass start through delete admission.
+ */
+ setInventoryAuthority(fn: () => McpServerInventoryAuthoritySnapshot): void {
+ this.inventoryAuthority = fn
+ this.inventoryAuthorityConfigured = true
+ }
+
+ /**
+ * Resolve current cache membership at effect admission. Full-reconcile
+ * snapshots select candidates only; they must never overwrite or delete a
+ * same-name CRD accepted by the continuing watch.
+ */
+ setResolveCurrentServer(fn: (name: string) => McpServerCRD | undefined): void {
+ this.resolveCurrentServer = fn
+ }
+
+ /**
+ * Verdict for a mutation fence evaluated while the inventory authority is NOT
+ * wired. Fail closed (deny) unless a test explicitly opted in; deny loudly
+ * once, because an unconfigured fence in a running controller is a
+ * construction bug, never a runtime condition.
+ */
+ private currentWhenUnconfigured(): boolean {
+ if (this.assumeInventoryAuthorityWhenUnconfigured) return true
+ if (!this.warnedUnconfiguredAuthority) {
+ this.warnedUnconfiguredAuthority = true
+ console.error(
+ '[Reconciler] inventory authority not configured — mutation fences fail closed. ' +
+ 'Wire setInventoryAuthority() and setResolveCurrentServer() before reconciling.'
+ )
+ }
+ return false
}
// ─── Status ────────────────────────────────────────────────────────
/**
* Get deployment status for an McpServer.
- * Returns unknown status if the server hasn't been reconciled yet.
+ * Returns current in-memory status when this process has reconciled the
+ * server. After a controller restart, a generation-matched persisted Ready
+ * condition is the only safe fallback: a legacy condition without
+ * observedGeneration, or a condition for an older spec generation, cannot
+ * prove that the current desired runtime is ready.
*/
- getStatus(name: string): McpServerStatus {
+ getStatus(serverOrName: string | McpServerCRD): McpServerStatus {
+ const name = typeof serverOrName === 'string' ? serverOrName : serverOrName.name
+ const current = this.statusMap.get(name)
+ if (current && (typeof serverOrName === 'string' || this.statusIdentityMatches(serverOrName))) {
+ return typeof serverOrName === 'string' ? current : { ...current, authoritative: true }
+ }
+
+ if (typeof serverOrName !== 'string' && serverOrName.generation !== undefined) {
+ const readyConditions =
+ serverOrName.status?.conditions?.filter(condition => condition.type === 'Ready') ?? []
+ const persistedReady =
+ readyConditions.length === 1 &&
+ readyConditions[0].observedGeneration === serverOrName.generation
+ ? readyConditions[0]
+ : undefined
+ if (persistedReady) {
+ const ready = persistedReady.status === 'True'
+ return {
+ deployed: ready,
+ ready,
+ message: persistedReady.message,
+ authoritative: true,
+ }
+ }
+ }
+
+ return {
+ deployed: false,
+ ready: false,
+ message: 'Not reconciled yet',
+ authoritative: false,
+ }
+ }
+
+ private statusIdentityMatches(server: McpServerCRD): boolean {
+ const identity = this.statusIdentities.get(server.name)
return (
- this.statusMap.get(name) ?? { deployed: false, ready: false, message: 'Not reconciled yet' }
+ identity !== undefined &&
+ identity.uid === server.uid &&
+ identity.generation === server.generation
)
}
+ private beginStatusTracking(server: McpServerCRD): void {
+ if (!this.statusIdentityMatches(server)) {
+ this.statusMap.delete(server.name)
+ // A new CRD identity (uid/generation) retires any poll window armed for
+ // the previous desired revision: its verdicts would describe a rollout
+ // that no longer represents the desired state. The reconcile for the
+ // new identity arms its own window after its Deployment write.
+ const poll = this.readinessPolls.get(server.name)
+ if (poll) {
+ if (poll.timer !== undefined) clearTimeout(poll.timer)
+ this.readinessPolls.delete(server.name)
+ }
+ }
+ this.statusIdentities.set(server.name, {
+ uid: server.uid,
+ generation: server.generation,
+ })
+ }
+
/**
* Update the status of an McpServer after reconciliation.
*/
@@ -216,6 +428,7 @@ export class McpServerReconciler {
*/
private clearStatus(name: string): void {
this.statusMap.delete(name)
+ this.statusIdentities.delete(name)
this.managedSnapshot.delete(name)
const poll = this.readinessPolls.get(name)
if (poll) {
@@ -307,14 +520,26 @@ export class McpServerReconciler {
* window with a superseded generation's numbers and the new rollout would
* never be observed (see ReadinessPollState.recheckRequested).
*/
- private pollReadiness(server: McpServerCRD, intervalMs = 5000, maxAttempts = 24): void {
+ private pollReadiness(
+ server: McpServerCRD,
+ intervalMs = 5000,
+ maxAttempts = 24,
+ isCurrent: () => boolean = () => true
+ ): void {
const { name, namespace } = server
const previous = this.readinessPolls.get(name)
if (previous !== undefined && (previous.timer !== undefined || previous.ticking)) {
- // A window is already running for this server — leave it alone
- // (re-arming would reset the attempt budget: the E2 status-update-loop).
- // But if its tick is executing RIGHT NOW, the tick's Deployment read may
+ // A window is already running for this server — leave its TIMER and
+ // BUDGET alone (re-arming would reset the attempt budget: the E2
+ // status-update-loop). But this caller is now the window's owner: rebind
+ // the mutation fence (and the same-identity server object) so the next
+ // tick evaluates the CURRENT fence instead of the arming reconcile's
+ // stale closure, which would otherwise self-terminate the window with no
+ // re-arm after a routine watch-generation bump.
+ previous.isCurrent = isCurrent
+ previous.server = server
+ // If its tick is executing RIGHT NOW, the tick's Deployment read may
// predate the Deployment write that triggered THIS call (a new
// generation), so a silent no-op could let the tick end the window on
// stale data. Record that a re-check is owed; the tick honors it before
@@ -337,6 +562,8 @@ export class McpServerReconciler {
attempts: 0,
exhausted: previous?.exhausted ?? false,
detail: previous?.detail ?? '',
+ isCurrent,
+ server,
}
this.readinessPolls.set(name, state)
@@ -346,7 +573,21 @@ export class McpServerReconciler {
// can tell "tick running" apart from "window ended" during our awaits.
state.timer = undefined
state.ticking = true
+ // Mutation fencing: a superseded reconcile (desired-revision or
+ // inventory-authority change) must not keep observing — or writing
+ // status — on behalf of a stale desired state. End the window quietly;
+ // the current reconcile arms its own window after its Deployment write.
+ if (!state.isCurrent()) {
+ state.ticking = false
+ if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
+ return
+ }
const rollout = await this.readDeploymentRollout(name, namespace)
+ if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {
+ state.ticking = false
+ if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
+ return
+ }
// The McpServer may have been deleted (clearStatus) while we were
// reading. A cancelled window must not keep ticking or write status
@@ -395,7 +636,20 @@ export class McpServerReconciler {
// always false, so this is the ONLY place a converged rollout is ever
// recorded — without it every successful credential rotation would sit
// at DeploymentReady=False forever and the UI would report a timeout.
- await this.updateStatusConditions(server, true)
+ await this.updateStatusConditions(state.server, true, undefined, state.isCurrent)
+ // Readiness is decoupled from fleet convergence: the poll also
+ // upgrades the persisted, generation-matched Ready condition so a
+ // controller restart can trust the recorded readiness (see getStatus).
+ await this.writeStatusCondition(
+ state.server,
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'Deployment created',
+ },
+ state.isCurrent
+ )
return
}
@@ -415,7 +669,7 @@ export class McpServerReconciler {
// Give up loudly, on the CRD: a stuck rollout must be visible to the
// operator with the numbers that prove it, never left as a stale
// condition or a silent Unknown.
- await this.updateStatusConditions(server, false, rollout.detail)
+ await this.updateStatusConditions(state.server, false, rollout.detail, state.isCurrent)
return
}
@@ -679,8 +933,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
/**
* Ensure ConfigMap exists and is up to date (create or replace).
*/
- private async ensureConfigMap(server: McpServerCRD): Promise<void> {
+ private async ensureConfigMap(
+ server: McpServerCRD,
+ isCurrent: () => boolean = () => true
+ ): Promise<void> {
const cm = this.buildNginxConfigMap(server)
+ if (!isCurrent()) return
try {
await this.coreApi.createNamespacedConfigMap({
namespace: server.namespace,
@@ -693,6 +951,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
name: `${server.name}-nginx-conf`,
namespace: server.namespace,
})
+ if (!isCurrent()) return
const next = preserveObjectAnnotations(
{
...cm,
@@ -703,6 +962,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
existing
)
+ if (!isCurrent()) return
await this.coreApi.replaceNamespacedConfigMap({
name: `${server.name}-nginx-conf`,
namespace: server.namespace,
@@ -810,7 +1070,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
private buildDeployment(server: McpServerCRD, credentialsRevision = ''): k8s.V1Deployment {
- // ── Platform hardening: sanitize CRD before building PodSpec ──
+ // ── Platform hardening: sanitize an ephemeral copy before building PodSpec ──
+ // The caller retains this object as the LIST -> WATCH desired-state cache.
+ // Mutating it here makes a later status-only watch event look like a
+ // user-driven spec revision, which invalidates the authoritative policy
+ // revocation marker during startup. Sanitization remains mandatory for the
+ // generated PodSpec, but must never rewrite the observed CRD inventory.
+ server = structuredClone(server)
this.sanitizeCrdSpec(server)
const transportPort = server.spec.transport.port || 3000
@@ -1206,9 +1472,14 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
/**
* Ensure a Deployment exists and is up to date.
*/
- private async ensureDeployment(server: McpServerCRD, credentialsRevision = ''): Promise<void> {
+ private async ensureDeployment(
+ server: McpServerCRD,
+ credentialsRevision = '',
+ isCurrent: () => boolean = () => true
+ ): Promise<void> {
const deployment = this.buildDeployment(server, credentialsRevision)
+ if (!isCurrent()) return
try {
await this.appsApi.createNamespacedDeployment({
namespace: server.namespace,
@@ -1243,6 +1514,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
namespace: server.namespace,
body,
}),
+ mutationAllowed: isCurrent,
})
}
}
@@ -1250,9 +1522,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
/**
* Ensure a Service exists and is up to date.
*/
- private async ensureService(server: McpServerCRD): Promise<void> {
+ private async ensureService(
+ server: McpServerCRD,
+ isCurrent: () => boolean = () => true
+ ): Promise<void> {
const service = this.buildService(server)
+ if (!isCurrent()) return
try {
await this.coreApi.createNamespacedService({
namespace: server.namespace,
@@ -1276,6 +1552,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
namespace: server.namespace,
body,
}),
+ mutationAllowed: isCurrent,
})
} else {
throw error
@@ -1288,12 +1565,16 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* Name matches are not enough: WRC-owned `managed:false` runtimes can share
* names with McpServer CRDs, so every delete verifies HCC ownership labels.
*/
- private async deleteResources(name: string, namespace: string): Promise<void> {
+ private async deleteResources(
+ name: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
const failures: unknown[] = []
for (const cleanup of [
- () => this.deleteDeploymentIfHccOwned(name, namespace),
- () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name),
- () => this.deleteServiceIfHccOwned(name, namespace),
+ () => this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed),
+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
+ () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
]) {
try {
await cleanup()
@@ -1315,7 +1596,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[MCPSERVER_LABEL] === serverName
}
- private async deleteDeploymentIfHccOwned(name: string, namespace: string): Promise<void> {
+ private async deleteDeploymentIfHccOwned(
+ name: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
let deployment: k8s.V1Deployment
try {
deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
@@ -1333,6 +1618,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(`[Reconciler] Skipping Deployment "${name}" delete — not HCC-owned`)
return
}
+ if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.appsApi.deleteNamespacedDeployment({ name, namespace })
@@ -1350,7 +1636,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
private async deleteConfigMapIfHccOwned(
configMapName: string,
namespace: string,
- serverName: string
+ serverName: string,
+ deleteAllowed?: () => Promise<boolean>
): Promise<void> {
let configMap: k8s.V1ConfigMap
try {
@@ -1369,6 +1656,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(`[Reconciler] Skipping ConfigMap "${configMapName}" delete — not HCC-owned`)
return
}
+ if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.coreApi.deleteNamespacedConfigMap({ name: configMapName, namespace })
@@ -1383,7 +1671,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async deleteServiceIfHccOwned(name: string, namespace: string): Promise<void> {
+ private async deleteServiceIfHccOwned(
+ name: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
let service: k8s.V1Service
try {
service = await this.coreApi.readNamespacedService({ name, namespace })
@@ -1401,6 +1693,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(`[Reconciler] Skipping Service "${name}" delete — not HCC-owned`)
return
}
+ if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.coreApi.deleteNamespacedService({ name, namespace })
@@ -1415,7 +1708,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async deleteNetworkPoliciesForServer(name: string, namespace: string): Promise<void> {
+ private async deleteNetworkPoliciesForServer(
+ name: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
if (!this.networkingApi) return
const failures: unknown[] = []
const namespaces = [...new Set([namespace, config.hostNamespace, config.rpcProxyNamespace])]
@@ -1429,6 +1726,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
const policyName = policy.metadata.name
if (!policyName) continue
try {
+ if (deleteAllowed && !(await deleteAllowed())) return
await this.networkingApi.deleteNamespacedNetworkPolicy({
name: policyName,
namespace: policyNamespace,
@@ -1460,15 +1758,19 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
throwCleanupFailures(failures, `Failed to delete NetworkPolicies for McpServer "${name}"`)
}
- private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {
+ private async deleteRuntimeResources(
+ name: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
const failures: unknown[] = []
try {
- await this.deleteResources(name, namespace)
+ await this.deleteResources(name, namespace, deleteAllowed)
} catch (error) {
failures.push(error)
}
try {
- await this.deleteNetworkPoliciesForServer(name, namespace)
+ await this.deleteNetworkPoliciesForServer(name, namespace, deleteAllowed)
} catch (error) {
failures.push(error)
}
@@ -1481,13 +1783,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* G2: Set `clerum.io/network-ready: "true"` annotation on McpServer CRD.
* Completes the pre-deploy handshake so WRC knows NetworkPolicies are applied.
*/
- private async setNetworkReadyAnnotation(server: McpServerCRD): Promise<void> {
+ private async setNetworkReadyAnnotation(
+ server: McpServerCRD,
+ isCurrent: () => boolean = () => true
+ ): Promise<void> {
// Use merge-patch (plain object body) instead of JSON Patch (array body).
// JSON Patch op:"add" on /metadata/annotations/<key> requires /metadata/annotations
// to already exist — it fails with 422 when the CRD was created without annotations.
// Merge-patch creates the map if absent. Errors are re-thrown so the caller (reconcile)
// knows the handshake was not completed — WRC will time out and fail rather than proceed
// silently without NetworkPolicies in place.
+ if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObject(
{
group: 'clerum.io',
@@ -1509,74 +1815,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
}
- /**
- * Keep remote egress proxy CRDs aligned with the platform-owned runtime image.
- *
- * Remote servers are rendered as nginx egress proxies and never run
- * `server.spec.image`, but stale CR specs can mislead scanners or future
- * tooling. Patch only the declarative image field; Deployment rendering still
- * uses `config.egressProxyImage` as the authority.
- *
- * This stays internally non-throwing because image canonicalization is
- * best-effort cleanup; a failed CRD patch must not block ConfigMap,
- * Service, or Deployment reconciliation for the actual proxy runtime.
- */
- private async canonicalizeRemoteEgressProxyImage(server: McpServerCRD): Promise<void> {
- if (!this.isRemote(server)) {
- return
- }
- if (server.spec.image === config.egressProxyImage) {
- await this.writeStatusCondition(server, {
- type: 'ImageCanonicalized',
- status: 'True',
- reason: 'RemoteEgressProxyImageMatches',
- message: 'Remote McpServer spec.image matches the platform egress proxy image',
- })
- return
- }
-
- try {
- await this.customApi.patchNamespacedCustomObject(
- {
- group: 'clerum.io',
- version: 'v1alpha1',
- namespace: server.namespace,
- plural: 'mcpservers',
- name: server.name,
- body: {
- spec: {
- image: config.egressProxyImage,
- },
- },
- },
- {
- middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
- }
- )
- server.spec.image = config.egressProxyImage
- console.log(
- `[Reconciler] Canonicalized remote McpServer "${server.name}" image to ${config.egressProxyImage}`
- )
- await this.writeStatusCondition(server, {
- type: 'ImageCanonicalized',
- status: 'True',
- reason: 'RemoteEgressProxyImageMatches',
- message: 'Remote McpServer spec.image matches the platform egress proxy image',
- })
- } catch (err) {
- console.warn(
- `[Reconciler] Failed to canonicalize remote McpServer "${server.name}" image to ${config.egressProxyImage}:`,
- err
- )
- await this.writeStatusCondition(server, {
- type: 'ImageCanonicalized',
- status: 'False',
- reason: 'RemoteEgressProxyImagePatchFailed',
- message: 'Failed to patch remote McpServer spec.image to the platform egress proxy image',
- })
- }
- }
-
/**
* G11: Update status conditions on McpServer CRD.
* Sets NetworkReady and DeploymentReady conditions so WRC can watch
@@ -1590,29 +1828,38 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* message so a stuck rollout reports WHAT is stuck (issue #223). Callers
* that only know the verdict omit it.
*/
- rolloutDetail?: string
+ rolloutDetail?: string,
+ isCurrent: () => boolean = () => true
): Promise<void> {
try {
- await this.writeStatusCondition(server, {
- type: 'NetworkReady',
- status: 'True',
- reason: 'NetworkPoliciesApplied',
- message: 'NetworkPolicies and Service created',
- })
- await this.writeStatusCondition(server, {
- type: 'DeploymentReady',
- status: deploymentReady ? 'True' : 'False',
- reason: deploymentReady
- ? 'ReplicasAvailable'
- : rolloutDetail
- ? 'RolloutIncomplete'
- : 'WaitingForReplicas',
- message: rolloutDetail
- ? `Rollout did not converge — ${rolloutDetail}`
- : deploymentReady
- ? 'Deployment has ready replicas'
- : 'Waiting for pods to become ready',
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'NetworkReady',
+ status: 'True',
+ reason: 'NetworkPoliciesApplied',
+ message: 'NetworkPolicies and Service created',
+ },
+ isCurrent
+ )
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'DeploymentReady',
+ status: deploymentReady ? 'True' : 'False',
+ reason: deploymentReady
+ ? 'ReplicasAvailable'
+ : rolloutDetail
+ ? 'RolloutIncomplete'
+ : 'WaitingForReplicas',
+ message: rolloutDetail
+ ? `Rollout did not converge — ${rolloutDetail}`
+ : deploymentReady
+ ? 'Deployment has ready replicas'
+ : 'Waiting for pods to become ready',
+ },
+ isCurrent
+ )
console.log(
`[Reconciler] Updated status conditions on "${server.name}" (DeploymentReady=${deploymentReady})`
)
@@ -1637,12 +1884,29 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
*
* Failure modes:
* - 404 (CRD deleted mid-reconcile) → log + swallow.
+ * - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.
* - Other errors → log + swallow (status writes are best-effort).
*/
async writeStatusCondition(
server: McpServerCRD,
- condition: Omit<McpServerCondition, 'lastTransitionTime'>
+ condition: Omit<McpServerCondition, 'lastTransitionTime'>,
+ isCurrent: () => boolean = () => true
): Promise<void> {
+ if (!isCurrent()) return
+ const now = new Date().toISOString()
+
+ // Update the missing-secret gauge for SecretResolved conditions.
+ // Do this before any no-op short-circuit so the gauge stays current
+ // across reconciles even when we skip the API patch.
+ if (condition.type === 'SecretResolved' && server.spec.envSecret) {
+ const labels = {
+ namespace: server.namespace,
+ name: server.name,
+ secret_name: server.spec.envSecret.name,
+ }
+ mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)
+ }
+
// Optimistic-concurrency loop (M2). The patch REPLACES the whole
// /status/conditions array with a value built from a snapshot read just
// before. Without a version guard, a concurrent writer — the readiness poll
@@ -1652,12 +1916,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// op on metadata.resourceVersion makes the API server reject the patch (422)
// if anything changed since our read; we then re-read and retry, so no
// writer's condition is lost.
- const MAX_ATTEMPTS = 4
- for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
- const now = new Date().toISOString()
-
- // Fetch current status (and its resourceVersion) to preserve existing
- // conditions and to fence the write below.
+ for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {
+ if (!isCurrent()) return
+ // Re-read on every optimistic-conflict retry. Merging against the
+ // latest resourceVersion preserves conditions written by peer
+ // controllers between our read and patch.
let existingConditions: McpServerCondition[] = []
let hasStatusObject = false
let resourceVersion: string | undefined
@@ -1672,9 +1935,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
metadata?: { resourceVersion?: string }
status?: { conditions?: McpServerCondition[] }
}
- resourceVersion = current.metadata?.resourceVersion
hasStatusObject = typeof current.status === 'object' && current.status !== null
existingConditions = current.status?.conditions ?? []
+ resourceVersion = current.metadata?.resourceVersion
} catch (error) {
if (getErrorCode(error) === 404) {
console.warn(
@@ -1688,26 +1951,16 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
return
}
+ if (!isCurrent()) return
const prior = existingConditions.find(c => c.type === condition.type)
+ const observedGeneration = condition.observedGeneration ?? server.generation
const unchanged =
prior !== undefined &&
prior.status === condition.status &&
prior.reason === condition.reason &&
- prior.message === condition.message
-
- // Update the missing-secret gauge for SecretResolved conditions. Do this
- // BEFORE the no-op short-circuit so the gauge stays current across
- // reconciles even when we skip the API patch. Idempotent, so a retry
- // re-running it is harmless.
- if (condition.type === 'SecretResolved' && server.spec.envSecret) {
- const labels = {
- namespace: server.namespace,
- name: server.name,
- secret_name: server.spec.envSecret.name,
- }
- mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)
- }
+ prior.message === condition.message &&
+ prior.observedGeneration === observedGeneration
// Skip the patch when the condition is identical to what's already on
// the CRD. Without this, every reconcile bumps the McpServer's
@@ -1727,26 +1980,49 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
reason: condition.reason,
message: condition.message,
lastTransitionTime,
+ ...(observedGeneration !== undefined && { observedGeneration }),
}
const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]
- // The `test` op fences the write to the exact revision we read. If a
- // concurrent writer bumped resourceVersion in between, the API server
- // fails the patch (422) and we retry with a fresh read.
+ const writePreconditions: Array<{
+ op: 'test'
+ path: string
+ value: string | number
+ }> = []
+ if (server.uid !== undefined) {
+ writePreconditions.push({
+ op: 'test',
+ path: '/metadata/uid',
+ value: server.uid,
+ })
+ }
+ if (server.generation !== undefined) {
+ writePreconditions.push({
+ op: 'test',
+ path: '/metadata/generation',
+ value: server.generation,
+ })
+ }
+ if (resourceVersion !== undefined) {
+ writePreconditions.push({
+ op: 'test',
+ path: '/metadata/resourceVersion',
+ value: resourceVersion,
+ })
+ }
const statusPatch = [
- ...(resourceVersion
- ? [{ op: 'test', path: '/metadata/resourceVersion', value: resourceVersion }]
- : []),
- hasStatusObject
- ? { op: 'add', path: '/status/conditions', value: nextConditions }
- : { op: 'add', path: '/status', value: { conditions: nextConditions } },
+ ...writePreconditions,
+ ...(hasStatusObject
+ ? [{ op: 'add', path: '/status/conditions', value: nextConditions }]
+ : [{ op: 'add', path: '/status', value: { conditions: nextConditions } }]),
]
try {
// The generated client prefers JSON Patch for this endpoint. Send an
// actual patch document so status writes keep working across client
// upgrades instead of relying on merge-patch object bodies.
+ if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObjectStatus({
group: 'clerum.io',
version: 'v1alpha1',
@@ -1757,22 +2033,25 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
})
return
} catch (error) {
- if (getErrorCode(error) === 404) {
+ const errorCode = getErrorCode(error)
+ if (errorCode === 404) {
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`
)
return
}
- // 409 (Conflict) or 422 (the resourceVersion `test` op failed) mean a
- // concurrent writer won the race. Re-read and retry rather than lose
- // our condition. Any other error, or the last attempt, falls through to
- // the best-effort log-and-swallow below.
- const code = getErrorCode(error)
- if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) {
+ // 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer
+ // won the race. Re-read and retry rather than lose our condition. Any
+ // other error, or the last attempt, falls through to the best-effort
+ // log-and-swallow below.
+ if (
+ (errorCode === 409 || errorCode === 422) &&
+ attempt < STATUS_CONDITION_WRITE_MAX_ATTEMPTS
+ ) {
continue
}
console.warn(
- `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${MAX_ATTEMPTS}):`,
+ `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,
error
)
return
@@ -1790,10 +2069,24 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* A prior failure does not block the next call; the chain absorbs it so
* the next reconcile still runs.
*/
- async reconcile(server: McpServerCRD): Promise<void> {
+ async reconcile(server: McpServerCRD, options: McpServerMutationOptions = {}): Promise<void> {
+ const capturedAuthority = this.inventoryAuthority()
+ const callerIsCurrent = options.isCurrent ?? (() => true)
+ const isCurrent = () => {
+ if (!callerIsCurrent()) return false
+ if (!this.inventoryAuthorityConfigured) return this.currentWhenUnconfigured()
+ if (!this.authorityMatches(capturedAuthority)) return false
+ const current = this.resolveCurrentServer?.(server.name)
+ return current !== undefined && sameMcpServerDesiredRevision(server, current)
+ }
const key = server.name
const prev = this.inFlight.get(key) ?? Promise.resolve()
- const next = prev.catch(() => undefined).then(() => this.performReconcile(server))
+ const next = prev
+ .catch(() => undefined)
+ .then(async () => {
+ if (!isCurrent()) return
+ await this.performReconcile(server, isCurrent)
+ })
this.inFlight.set(key, next)
try {
await next
@@ -1804,8 +2097,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async performReconcile(server: McpServerCRD): Promise<void> {
+ private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<void> {
+ if (!isCurrent()) return
+ this.beginStatusTracking(server)
+
// G7: Enforce managed field immutability — once set, cannot change
+ if (!isCurrent()) return
const currentManaged = server.spec.managed ?? true
const previousManaged = this.managedSnapshot.get(server.name)
if (previousManaged !== undefined && previousManaged !== currentManaged) {
@@ -1813,6 +2110,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +
`This is not allowed — delete and recreate the McpServer to change ownership.`
)
+ if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -1820,6 +2118,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
})
return
}
+ if (!isCurrent()) return
this.managedSnapshot.set(server.name, currentManaged)
// If disabled, respect ownership before any cleanup. managed:false means
@@ -1829,57 +2128,72 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.log(
`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`
)
- await this.deleteRuntimeResources(server.name, server.namespace)
+ await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
} else {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`
)
}
+ if (!isCurrent()) return
this.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })
- await this.writeStatusCondition(server, {
- type: 'Ready',
- status: 'False',
- reason: 'Disabled',
- message: 'McpServer is disabled',
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'Ready',
+ status: 'False',
+ reason: 'Disabled',
+ message: 'McpServer is disabled',
+ },
+ isCurrent
+ )
return
}
if (!currentManaged) {
- await this.reconcileWrcOwnedServer(server)
+ await this.reconcileWrcOwnedServer(server, isCurrent)
return
}
// Validate secret before creating/updating
const secretResult = await this.validateSecret(server)
+ if (!isCurrent()) return
if (!secretResult.ok) {
console.error(
`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
)
if (this.shouldFailClosedForSecretFailure(secretResult)) {
- await this.deleteRuntimeResources(server.name, server.namespace)
+ await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
} else {
console.warn(
`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`
)
}
+ if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
message: secretResult.message,
})
- await this.writeStatusCondition(server, {
- type: 'SecretResolved',
- status: 'False',
- reason: secretResult.reason,
- message: secretResult.message,
- })
- await this.writeStatusCondition(server, {
- type: 'Ready',
- status: 'False',
- reason: 'SecretValidationFailed',
- message: secretResult.message,
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'SecretResolved',
+ status: 'False',
+ reason: secretResult.reason,
+ message: secretResult.message,
+ },
+ isCurrent
+ )
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'Ready',
+ status: 'False',
+ reason: 'SecretValidationFailed',
+ message: secretResult.message,
+ },
+ isCurrent
+ )
return
}
@@ -1887,8 +2201,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
try {
// Remote egress proxy: create nginx ConfigMap before Deployment
if (this.isRemote(server)) {
- await this.canonicalizeRemoteEgressProxyImage(server)
- await this.ensureConfigMap(server)
+ // The CRD image remains desired-state input owned by its author. Remote
+ // workloads use the platform-owned proxy image selected in
+ // buildDeployment(); rewriting spec.image here would bump generation
+ // and retire this reconcile before it can create the runtime resources.
+ await this.ensureConfigMap(server, isCurrent)
} else {
// Phase 2.3: plugin image-host allowlist. Audit mode (default) logs a
// would-be denial and still builds; enforce mode blocks the workload.
@@ -1900,12 +2217,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
const message = `Image "${server.spec.image}" is not permitted by the plugin image allowlist (${decision.reason})`
if (config.enforcePluginImageAllowlist) {
console.warn(`[Reconciler] ${message} — blocking "${server.name}" (enforce mode)`)
- await this.writeStatusCondition(server, {
- type: 'Ready',
- status: 'False',
- reason: 'ImageNotAllowed',
- message,
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'Ready',
+ status: 'False',
+ reason: 'ImageNotAllowed',
+ message,
+ },
+ isCurrent
+ )
+ if (!isCurrent()) return
this.setStatus(server.name, { deployed: false, ready: false, message })
return
}
@@ -1914,9 +2236,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
}
}
- await this.ensureService(server)
- await this.ensureDeployment(server, secretResult.revision)
+ if (!isCurrent()) return
+ await this.ensureService(server, isCurrent)
+ if (!isCurrent()) return
+ await this.ensureDeployment(server, secretResult.revision, isCurrent)
} catch (err) {
+ if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -1933,8 +2258,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
server.annotations?.[NETWORK_READY_ANNOTATION] !== 'true'
) {
try {
- await this.setNetworkReadyAnnotation(server)
+ await this.setNetworkReadyAnnotation(server, isCurrent)
} catch (err) {
+ if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -1946,6 +2272,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// Check readiness after reconciliation
const rollout = await this.readDeploymentRollout(server.name, server.namespace)
+ if (!isCurrent()) return
const ready = rollout.ready
this.setStatus(server.name, {
deployed: true,
@@ -1970,28 +2297,36 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
pollState.generation === rollout.generation
? pollState.detail
: undefined
- await this.updateStatusConditions(server, ready, exhaustedDetail)
+ await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)
// PR-B B1: record SecretResolved=True + Ready=True after a successful
// reconcile. This is additive to updateStatusConditions (NetworkReady /
// DeploymentReady); the merge logic in writeStatusCondition preserves
// the existing conditions by type.
if (server.spec.envSecret) {
- await this.writeStatusCondition(server, {
- type: 'SecretResolved',
- status: 'True',
- reason: 'SecretFound',
- message: 'Secret resolved and validated',
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'SecretResolved',
+ status: 'True',
+ reason: 'SecretFound',
+ message: 'Secret resolved and validated',
+ },
+ isCurrent
+ )
}
- await this.writeStatusCondition(server, {
- type: 'Ready',
- status: ready ? 'True' : 'Unknown',
- reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',
- message: ready
- ? 'Deployment created'
- : 'Deployment created, waiting for pods to become ready',
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'Ready',
+ status: ready ? 'True' : 'Unknown',
+ reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',
+ message: ready
+ ? 'Deployment created'
+ : 'Deployment created, waiting for pods to become ready',
+ },
+ isCurrent
+ )
// If not ready yet, poll until it becomes ready. With generation-aware
// readiness this is the normal path right after any Deployment write —
@@ -2008,8 +2343,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// is a deployment whose replica tuple perpetually OSCILLATES, and that is
// bounded to one reconcile per poll window (~120s) — not a tight loop, and (per
// pollReadiness's atomic check-then-set) not a timer leak.
- if (!ready) {
- this.pollReadiness(server)
+ if (!ready && isCurrent()) {
+ this.pollReadiness(server, 5000, 24, isCurrent)
}
}
@@ -2023,13 +2358,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
}
- private async reconcileWrcOwnedServer(server: McpServerCRD): Promise<void> {
+ private async reconcileWrcOwnedServer(
+ server: McpServerCRD,
+ isCurrent: () => boolean
+ ): Promise<void> {
+ if (!isCurrent()) return
console.log(
`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +
'skipping HCC runtime creation/deletion and updating discovery status.'
)
const secretResult = await this.validateSecret(server)
+ if (!isCurrent()) return
if (!secretResult.ok) {
this.setStatus(server.name, {
deployed: true,
@@ -2037,41 +2377,58 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
message: secretResult.message,
})
if (server.spec.envSecret) {
- await this.writeStatusCondition(server, {
- type: 'SecretResolved',
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'SecretResolved',
+ status: 'False',
+ reason: secretResult.reason,
+ message: secretResult.message,
+ },
+ isCurrent
+ )
+ }
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'Ready',
status: 'False',
- reason: secretResult.reason,
+ reason: 'SecretValidationFailed',
message: secretResult.message,
- })
- }
- await this.writeStatusCondition(server, {
- type: 'Ready',
- status: 'False',
- reason: 'SecretValidationFailed',
- message: secretResult.message,
- })
+ },
+ isCurrent
+ )
return
}
+ if (!isCurrent()) return
this.setStatus(server.name, {
deployed: true,
ready: true,
message: 'WRC-owned runtime registered',
})
if (server.spec.envSecret) {
- await this.writeStatusCondition(server, {
- type: 'SecretResolved',
- status: 'True',
- reason: 'SecretFound',
- message: 'Secret resolved and validated',
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'SecretResolved',
+ status: 'True',
+ reason: 'SecretFound',
+ message: 'Secret resolved and validated',
+ },
+ isCurrent
+ )
}
- await this.writeStatusCondition(server, {
- type: 'Ready',
- status: 'True',
- reason: WRC_OWNED_RUNTIME_READY_REASON,
- message: 'WRC-owned runtime registered for HCC discovery',
- })
+ await this.writeStatusCondition(
+ server,
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: WRC_OWNED_RUNTIME_READY_REASON,
+ message: 'WRC-owned runtime registered for HCC discovery',
+ },
+ isCurrent
+ )
}
/**
@@ -2079,18 +2436,55 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
*/
async reconcileDelete(name: string, namespace: string): Promise<void> {
console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)
- await this.deleteRuntimeResources(name, namespace)
+ const capturedAuthority = this.inventoryAuthority()
+ await this.deleteRuntimeResources(name, namespace, () =>
+ this.orphanDeleteAllowed(name, namespace, capturedAuthority)
+ )
this.clearStatus(name)
}
+ private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {
+ const current = this.inventoryAuthority()
+ return captured.known && current.known && captured.generation === current.generation
+ }
+
+ /**
+ * A cache miss is not proof of absence: the API watch may have accepted an
+ * ADDED event that has not reached this process yet. Confirm a 404 from the
+ * API and then recheck both cache membership and LIST→WATCH authority
+ * immediately before every destructive delete.
+ */
+ private async orphanDeleteAllowed(
+ name: string,
+ namespace: string,
+ capturedAuthority: McpServerInventoryAuthoritySnapshot
+ ): Promise<boolean> {
+ return confirmAuthoritativeMcpServerAbsence({
+ inventoryAuthoritative: () => this.authorityMatches(capturedAuthority),
+ resolveCurrent: () => this.resolveCurrentServer?.(name),
+ readCurrent: () =>
+ this.customApi.getNamespacedCustomObject({
+ group: 'clerum.io',
+ version: 'v1alpha1',
+ namespace,
+ plural: 'mcpservers',
+ name,
+ }),
+ })
+ }
+
/**
* Full reconciliation pass — sync desired state with actual state.
* Called on startup to handle CRDs that were created while the skill-mapper was down.
*/
- async fullReconcile(desiredServers: McpServerCRD[]): Promise<void> {
+ async fullReconcile(
+ desiredServers: McpServerCRD[],
+ options: McpServerFullReconcileOptions = {}
+ ): Promise<void> {
console.log(
`[Reconciler] Running full reconciliation for ${desiredServers.length} McpServer(s)`
)
+ const capturedAuthority = this.inventoryAuthority()
// Get all deployments managed by skill-mapper
const existingDeployments = await this.listManagedDeployments()
@@ -2100,18 +2494,62 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
.map(s => s.name)
)
- // Create or update desired servers
- for (const server of desiredServers) {
- await this.reconcile(server)
- }
+ const runEffect =
+ options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
+ const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY
+
+ // Create or update desired servers independently. runEffect owns each
+ // per-server lane, so a slow fleet member cannot become a process-wide
+ // startup barrier. The finite worker pool admits no more than the configured
+ // number of effects at once, while still waiting for every desired result
+ // before reporting aggregated failures or moving to orphan cleanup.
+ const desiredResults = await mapSettledWithConcurrency(desiredServers, maxConcurrency, server =>
+ runEffect(server.name, async () => {
+ const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server
+ // Cache mutation happens synchronously before its keyed watch effect is
+ // queued. If this startup item was superseded, skip it and let the
+ // newer queued event perform external-egress-before-runtime using its
+ // own spec; reconciling the newer runtime here could otherwise race
+ // ahead of that safety policy.
+ if (!current || !sameMcpServerDesiredRevision(server, current)) return
+ const isCurrent = () => {
+ if (!this.inventoryAuthorityConfigured) {
+ if (!this.currentWhenUnconfigured()) return false
+ } else if (!this.authorityMatches(capturedAuthority)) {
+ return false
+ }
+ const latest = this.resolveCurrentServer?.(current.name)
+ return latest
+ ? sameMcpServerDesiredRevision(current, latest)
+ : this.resolveCurrentServer === undefined && this.currentWhenUnconfigured()
+ }
+ await this.reconcile(current, { isCurrent })
+ })
+ )
+ throwCleanupFailures(
+ desiredResults
+ .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+ .map(result => result.reason),
+ 'Failed to reconcile desired McpServers'
+ )
// Delete orphaned deployments (managed by us but no longer desired)
for (const deployment of existingDeployments) {
const name = deployment.metadata?.name || ''
if (!desiredNames.has(name)) {
const namespace = deployment.metadata?.namespace || config.namespace
- console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
- await this.deleteRuntimeResources(name, namespace)
+ await runEffect(name, async () => {
+ if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {
+ console.warn(
+ `[Reconciler] Deferring orphan cleanup for "${name}": McpServer exists or inventory authority is unknown or changed`
+ )
+ return
+ }
+ console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
+ await this.deleteRuntimeResources(name, namespace, () =>
+ this.orphanDeleteAllowed(name, namespace, capturedAuthority)
+ )
+ })
}
}
@@ -2130,7 +2568,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
return response.items || []
} catch (error) {
console.error('[Reconciler] Failed to list managed deployments:', error)
- return []
+ throw error
}
}
}
diff --git a/host-context-controller/src/server.test.ts b/host-context-controller/src/server.test.ts
index 633a48992..0ab7d03aa 100644
--- a/host-context-controller/src/server.test.ts
+++ b/host-context-controller/src/server.test.ts
@@ -1,5 +1,6 @@
-import { afterEach, describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
import { McpServerProvider } from './k8sClient'
+import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
import { ContextMapperServer } from './server'
class FakeProvider implements McpServerProvider {
@@ -75,14 +76,30 @@ describe('ContextMapperServer readiness', () => {
afterEach(async () => {
await server?.stop()
server = null
+ vi.restoreAllMocks()
})
- it('serves health immediately but gates readiness and API responses until warm-up completes', async () => {
+ it('withholds readiness when no inventory-authority gate is wired', async () => {
+ // The gate decides whether the provider's inventory is authoritative. A
+ // caller that forgets to wire it must not get a permanently Ready server:
+ // this endpoint is the contract that no stale allow is live, so the default
+ // has to fail closed. There is no type error to catch the omission.
server = new ContextMapperServer(new FakeProvider(), 0)
+ server.setReady(true)
+
+ const response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ })
+
+ it('serves health immediately but gates readiness and API responses until warm-up completes', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
let response = await invoke(server, '/health')
expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toEqual({ status: 'ok', ready: false })
+ expect(JSON.parse(response.body)).toEqual({
+ status: 'ok',
+ ready: false,
+ })
response = await invoke(server, '/ready')
expect(response.statusCode).toBe(503)
@@ -108,4 +125,188 @@ describe('ContextMapperServer readiness', () => {
contextRef: '*',
})
})
+
+ it('tracks provider authority dynamically after warm-up and fails API requests closed', async () => {
+ let providerAuthoritative = true
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ undefined,
+ undefined,
+ () => providerAuthoritative
+ )
+
+ let response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })
+
+ server.setReady(true)
+
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
+
+ providerAuthoritative = false
+
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
+
+ response = await invoke(server, '/health')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({
+ status: 'ok',
+ ready: false,
+ })
+
+ response = await invoke(server, '/api/v1/mcpservers')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({
+ error: 'Service Unavailable',
+ message: 'Context mapper provider inventory is not authoritative',
+ })
+
+ providerAuthoritative = true
+
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
+
+ response = await invoke(server, '/api/v1/mcpservers')
+ expect(response.statusCode).toBe(200)
+ })
+
+ it('fails readiness closed when the provider authority check throws', async () => {
+ const authorityError = new Error('authority unavailable')
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => {
+ throw authorityError
+ })
+ server.setReady(true)
+
+ const response = await invoke(server, '/ready')
+
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
+ expect(errorLog).toHaveBeenCalledWith(
+ '[Server] Provider authority readiness check failed:',
+ authorityError
+ )
+ })
+
+ it('withdraws readiness as soon as shutdown begins', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
+ server.setReady(true)
+
+ expect((await invoke(server, '/ready')).statusCode).toBe(200)
+
+ await server.stop()
+
+ const response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })
+ })
+})
+
+describe('ContextMapperServer desktop status authority', () => {
+ let server: ContextMapperServer | null = null
+
+ // A Host reconciler that reports a running desktop, so the only thing that can
+ // withhold a 200 is the Host-authority gate under test.
+ const runningReconciler = {
+ getStatus: () => ({ deployed: true, ready: true, message: '' }),
+ hasDesktop: () => true,
+ } as unknown as ConstructorParameters<typeof ContextMapperServer>[2]
+ const hasDesktopFn = () => true
+
+ afterEach(async () => {
+ await server?.stop()
+ server = null
+ vi.restoreAllMocks()
+ })
+
+ it('serves desktop status on Host authority alone even when the provider inventory is not authoritative', async () => {
+ // The heart of H1: desktop needs only Host authority. A degraded
+ // McpServer/Context lane (providerAuthoritative=false) must NOT 503 desktop,
+ // while /api/v1/mcpservers still 503s on the same server.
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ () => false, // provider inventory NOT authoritative
+ () => true // Host inventory authoritative
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(200)
+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })
+
+ const mcpservers = await invoke(server, '/api/v1/mcpservers')
+ expect(mcpservers.statusCode).toBe(503)
+ })
+
+ it('fails desktop closed (503, never 200 inactive) when Host inventory is not authoritative', async () => {
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ () => true, // provider authoritative — must not rescue desktop
+ () => false // Host inventory NOT authoritative
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(503)
+ expect(JSON.parse(desktop.body)).toEqual({
+ error: 'Service Unavailable',
+ message: 'Host inventory is not authoritative',
+ })
+ })
+
+ it('serves desktop status 200 in the dev wiring (no watcher), never 503 (R2-M1)', async () => {
+ // Wiring regression for R2-M1: main.ts resolves both authority gates from the
+ // same watcher. In dev (DevMcpServerProvider, watcher=null) the OLD wiring left
+ // the Host gate undefined, so the server's fail-closed default pinned every
+ // /api/v1/desktop/* at 503 after setReady(true) — where the base answered 200
+ // {status:'inactive'}. Exercise the REAL resolvers with a null watcher, exactly
+ // as main.ts does, so a revert to the undefined wiring turns this 200 into 503.
+ const providerAuthoritativeFn = resolveProviderAuthoritativeFn(null)
+ const hostAuthoritativeFn = resolveHostAuthoritativeFn(null)
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ providerAuthoritativeFn,
+ hostAuthoritativeFn
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(200)
+ expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })
+ })
+
+ it('fails desktop closed when no Host-authority gate is wired (default)', async () => {
+ // Omitting the 6th arg must default to fail-closed, exactly like the
+ // provider gate — no type error catches the omission.
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ runningReconciler,
+ hasDesktopFn,
+ () => true
+ )
+ server.setReady(true)
+
+ const desktop = await invoke(server, '/api/v1/desktop/host-a')
+ expect(desktop.statusCode).toBe(503)
+ expect(JSON.parse(desktop.body)).toEqual({
+ error: 'Service Unavailable',
+ message: 'Host inventory is not authoritative',
+ })
+ })
})
diff --git a/host-context-controller/src/server.ts b/host-context-controller/src/server.ts
index 7be800a8e..03f576dcf 100644
--- a/host-context-controller/src/server.ts
+++ b/host-context-controller/src/server.ts
@@ -8,24 +8,53 @@ import { McpServerProvider } from './k8sClient'
import { registry } from './metrics'
import { ErrorResponse, McpServersResponse } from './types'
+type ReadinessState =
+ | { ready: true; status: 'ready' }
+ | { ready: false; status: 'starting' | 'degraded'; message: string }
+
export class ContextMapperServer {
private server: http.Server | null = null
private provider: McpServerProvider
private port: number
private hostReconciler: HostReconciler | null
private hasDesktopFn: ((hostRef: string) => boolean) | null
+ private providerAuthoritativeFn: () => boolean
+ private hostAuthoritativeFn: () => boolean
private ready = false
constructor(
provider: McpServerProvider,
port: number = config.port,
hostReconciler?: HostReconciler,
- hasDesktopFn?: (hostRef: string) => boolean
+ hasDesktopFn?: (hostRef: string) => boolean,
+ // Fails closed. /ready is the assertion that no stale allow is live, and a
+ // caller that omits this gate gets no type error, so the default must
+ // withhold readiness rather than grant it.
+ providerAuthoritativeFn: () => boolean = () => false,
+ // Desktop status needs ONLY Host inventory authority, not the full
+ // readiness inventory. Fails closed like providerAuthoritativeFn: a caller
+ // that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.
+ hostAuthoritativeFn: () => boolean = () => false
) {
this.provider = provider
this.port = port
this.hostReconciler = hostReconciler ?? null
this.hasDesktopFn = hasDesktopFn ?? null
+ this.providerAuthoritativeFn = providerAuthoritativeFn
+ this.hostAuthoritativeFn = hostAuthoritativeFn
+ }
+
+ /**
+ * Host inventory authority, fail-closed on error — the desktop route's only
+ * authority gate. Same shape as the provider check in getReadinessState().
+ */
+ private safeHostAuthoritative(): boolean {
+ try {
+ return this.hostAuthoritativeFn()
+ } catch (err) {
+ console.error('[Server] Host authority check failed:', err)
+ return false
+ }
}
/**
@@ -35,6 +64,30 @@ export class ContextMapperServer {
this.ready = ready
}
+ private getReadinessState(): ReadinessState {
+ if (!this.ready) {
+ return {
+ ready: false,
+ status: 'starting',
+ message: 'Context mapper is still starting',
+ }
+ }
+
+ try {
+ if (this.providerAuthoritativeFn()) {
+ return { ready: true, status: 'ready' }
+ }
+ } catch (err) {
+ console.error('[Server] Provider authority readiness check failed:', err)
+ }
+
+ return {
+ ready: false,
+ status: 'degraded',
+ message: 'Context mapper provider inventory is not authoritative',
+ }
+ }
+
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// CORS headers
res.setHeader('Access-Control-Allow-Origin', '*')
@@ -57,15 +110,20 @@ export class ContextMapperServer {
// Health check
if (req.method === 'GET' && url.pathname === '/health') {
- this.sendJson(res, 200, { status: 'ok', ready: this.ready })
+ const readiness = this.getReadinessState()
+ this.sendJson(res, 200, {
+ status: 'ok',
+ ready: readiness.ready,
+ })
return
}
// Readiness check
if (req.method === 'GET' && url.pathname === '/ready') {
- this.sendJson(res, this.ready ? 200 : 503, {
- status: this.ready ? 'ready' : 'starting',
- ready: this.ready,
+ const readiness = this.getReadinessState()
+ this.sendJson(res, readiness.ready ? 200 : 503, {
+ status: readiness.status,
+ ready: readiness.ready,
})
return
}
@@ -78,10 +136,30 @@ export class ContextMapperServer {
return
}
- if (!this.ready) {
+ // Desktop status: Host inventory is the ONLY authority this route needs, so
+ // it is gated here — ABOVE the blanket readiness gate — on Host authority
+ // alone. A degraded McpServer/Context lane must NOT 503 desktop, and a
+ // degraded Host lane must 503 it (never answer 200 'inactive' from a stale
+ // cache). See the E3 design note's endpoint→authority matrix.
+ if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {
+ if (!this.ready || !this.safeHostAuthoritative()) {
+ this.sendJson(res, 503, {
+ error: 'Service Unavailable',
+ message: 'Host inventory is not authoritative',
+ })
+ return
+ }
+ if (!this.checkDesktopAuth(req, res)) return
+ const hostRef = url.pathname.replace('/api/v1/desktop/', '')
+ await this.handleDesktopStatus(res, hostRef)
+ return
+ }
+
+ const readiness = this.getReadinessState()
+ if (!readiness.ready) {
this.sendJson(res, 503, {
error: 'Service Unavailable',
- message: 'Context mapper is still starting',
+ message: readiness.message,
})
return
}
@@ -110,14 +188,6 @@ export class ContextMapperServer {
return
}
- // Desktop status (requires service token)
- if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {
- if (!this.checkDesktopAuth(req, res)) return
- const hostRef = url.pathname.replace('/api/v1/desktop/', '')
- await this.handleDesktopStatus(res, hostRef)
- return
- }
-
// 404
this.sendJson(res, 404, { error: 'Not Found', message: 'Endpoint not found' })
}
@@ -198,7 +268,8 @@ export class ContextMapperServer {
endpoints: {
'GET /': 'This information page',
'GET /health': "Health check endpoint - returns { status: 'ok', ready: boolean }",
- 'GET /ready': 'Readiness endpoint - returns 200 only after warm-up completes',
+ 'GET /ready':
+ 'Readiness endpoint - returns 200 after warm-up while provider inventory is authoritative',
'GET /api/v1/mcpservers': 'List all McpServer resources',
'GET /api/v1/mcpservers/context/:contextRef':
'List McpServers filtered by contextRef (only enabled servers)',
@@ -346,9 +417,12 @@ export class ContextMapperServer {
* Stop the server.
*/
stop(): Promise<void> {
+ this.ready = false
+ const activeServer = this.server
+ this.server = null
return new Promise(resolve => {
- if (this.server) {
- this.server.close(() => {
+ if (activeServer) {
+ activeServer.close(() => {
console.log('[Server] Context Mapper stopped')
resolve()
})
diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.ts
index 81542832a..f731391ca 100644
--- a/host-context-controller/src/statelessLifecycleExecutor.ts
+++ b/host-context-controller/src/statelessLifecycleExecutor.ts
@@ -42,6 +42,21 @@ const ACTIVE_COMMUNICATION_CHANNELS_REASON = 'ActiveCommunicationChannels'
*/
const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'
+/**
+ * H2: reflect a committed lifecycle outcome onto the CURRENT canonical cache
+ * entry for `name` — but only if that entry still has the admitted `uid`.
+ * The receiver (McpServerWatcher) owns the lookup and the uid guard; the
+ * executor never touches the cache directly. A same-name recreation with a
+ * different uid MUST be skipped silently (the mutation already committed
+ * server-side under the old uid; corrupting the new object would be worse
+ * than staleness, which the watch/resync repairs).
+ */
+export type ReflectHostOutcomeFn = (
+ name: string,
+ uid: string | undefined,
+ apply: (target: HostCRD) => void
+) => void
+
/** Constructor-injected seams from HostReconciler (late-bound where needed). */
export interface StatelessLifecycleExecutorDeps {
appsApi: k8s.AppsV1Api
@@ -71,7 +86,16 @@ export interface StatelessLifecycleExecutorDeps {
* serialization slot, so re-entering the per-host chain would
* self-deadlock.
*/
- reconcileCore: (host: HostCRD) => Promise<void>
+ reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>
+ /**
+ * Capture Host inventory authority at heartbeat dispatch time and return an
+ * admission closure that revalidates that authority and resolves the current
+ * same-identity Host inside this executor's per-Host serializer. The two-step
+ * shape is intentional: capturing only after the queue drains cannot detect
+ * an authority generation change that happened while the work was waiting.
+ */
+ prepareHostMutationAdmission?: (action: string, host: HostCRD) => () => HostCRD
+ reflectHostOutcome?: ReflectHostOutcomeFn
onLifecycleStatusCommitted?: (host: HostCRD, lifecycle: HostLifecycleStatus) => void
}
@@ -82,7 +106,9 @@ export class StatelessLifecycleExecutor {
private readonly now: () => Date
private readonly countCommunicationChannels: (hostName: string) => number
private readonly isCommunicationChannelCacheSynced: () => boolean
- private readonly reconcileCore: (host: HostCRD) => Promise<void>
+ private readonly reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>
+ private readonly prepareHostMutationAdmission: (action: string, host: HostCRD) => () => HostCRD
+ private readonly reflectHostOutcome: ReflectHostOutcomeFn
private readonly onLifecycleStatusCommitted: (
host: HostCRD,
lifecycle: HostLifecycleStatus
@@ -132,6 +158,9 @@ export class StatelessLifecycleExecutor {
this.countCommunicationChannels = deps.countCommunicationChannels
this.isCommunicationChannelCacheSynced = deps.isCommunicationChannelCacheSynced
this.reconcileCore = deps.reconcileCore
+ this.prepareHostMutationAdmission =
+ deps.prepareHostMutationAdmission ?? ((_action, host) => () => host)
+ this.reflectHostOutcome = deps.reflectHostOutcome ?? (() => {})
this.onLifecycleStatusCommitted = deps.onLifecycleStatusCommitted ?? (() => {})
StatelessLifecycleExecutor.warnSingleReplicaInvariantOnce()
}
@@ -671,15 +700,19 @@ export class StatelessLifecycleExecutor {
metadata: {
name: string
namespace?: string
+ uid?: string
+ generation?: number
resourceVersion?: string
annotations?: Record<string, string>
}
spec: HostCRD['spec']
status?: HostCRD['status']
}
- return {
+ const fresh: HostCRD = {
name: obj.metadata.name,
namespace: obj.metadata.namespace ?? host.namespace,
+ uid: obj.metadata.uid,
+ generation: obj.metadata.generation,
// Preserve resourceVersion so heartbeat-path status writers can pass it
// back as an optimistic-concurrency precondition (D3). Dropping it here
// would force every write to be an unconditioned last-write-wins patch.
@@ -688,6 +721,8 @@ export class StatelessLifecycleExecutor {
spec: obj.spec,
status: obj.status,
}
+ this.requireSameHostIdentity(host, fresh)
+ return fresh
}
/**
@@ -716,17 +751,24 @@ export class StatelessLifecycleExecutor {
reason: string,
entryWakeHandledGeneration: number
): Promise<SuspendFromHeartbeatOutcome> {
- // L2: serialize on the host chain so a tracker-driven suspend never
- // interleaves with a watch-driven wake reconcile for the same host.
- return this.serializeByHost(host.name, () =>
- this.suspendHostFromHeartbeatCore(host, reason, entryWakeHandledGeneration)
+ return this.serializeHeartbeatMutation(
+ 'heartbeat suspension',
+ host,
+ (admittedHost, revalidate) =>
+ this.suspendHostFromHeartbeatCore(
+ admittedHost,
+ reason,
+ entryWakeHandledGeneration,
+ revalidate
+ )
)
}
private async suspendHostFromHeartbeatCore(
host: HostCRD,
reason: string,
- entryWakeHandledGeneration: number
+ entryWakeHandledGeneration: number,
+ revalidate?: () => HostCRD
): Promise<SuspendFromHeartbeatOutcome> {
// FRESH GET (see readFreshHost): guard against the CURRENT server-side
// lifecycle, not the caller's watch-cache snapshot. The guard is re-run
@@ -741,6 +783,17 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
+ // UID proves object identity, not decision freshness. A same-identity
+ // Host may have advanced from stateless to stateful while the watch
+ // cache trails the API. Re-evaluate the complete effective stateless
+ // policy from every fresh read/retry before honoring old heartbeat
+ // evidence.
+ if (
+ !this.sameHostSpecRevision(host, fresh) ||
+ !this.effectiveLifecycleFromCache(fresh).stateless
+ ) {
+ return { skip: true }
+ }
const freshLifecycle = fresh.status?.lifecycle
if (freshLifecycle?.state !== 'draining') {
// Every legitimate suspend follows a durable `draining` write (the
@@ -802,12 +855,36 @@ export class StatelessLifecycleExecutor {
writtenStatus = { ...(fresh.status ?? {}), lifecycle }
return this.fullStatusOps(writtenStatus)
},
- initialFresh
+ initialFresh,
+ revalidate
)
if ('skipped' in result || writtenStatus === undefined) {
return skipOutcome
}
- host.status = writtenStatus
+ // Revalidate authority/UID immediately before follow-on effects, then read
+ // once more from the API. The successful PATCH attempt's fresh object is
+ // the exact spec on which the status write committed, but a wake/spec
+ // update can land immediately AFTER that commit and BEFORE runtime
+ // reconciliation. A linearizable follow-on GET observes both our durable
+ // suspended write and any such newer same-UID update; the watch cache can
+ // legitimately trail either one.
+ const currentAtFollowOn = revalidate?.()
+ if (currentAtFollowOn) this.requireSameHostIdentity(host, currentAtFollowOn)
+ this.reflectOutcome(host, target => {
+ target.status = writtenStatus
+ })
+ const reconcileHost = await this.readFreshHost(result.fresh)
+ if (!this.sameHostSpecRevision(host, reconcileHost)) {
+ const error = new Error(
+ `Host spec generation changed before heartbeat suspension follow-on for "${host.name}"`
+ )
+ error.name = 'HostMutationSpecRevisionChangedError'
+ throw error
+ }
+ const currentAfterFollowOnRead = revalidate?.()
+ if (currentAfterFollowOnRead) {
+ this.requireSameHostIdentity(host, currentAfterFollowOnRead)
+ }
this.lastWrittenLifecycleStatus.delete(host.name)
this.logSuspendedApplied(host.name)
console.log(
@@ -816,18 +893,32 @@ export class StatelessLifecycleExecutor {
// L2: call reconcileCore (not the serialized reconcile) — this method
// already runs inside this host's serialization slot, so re-entering the
// per-host chain here would self-deadlock.
- await this.reconcileCore(host)
+ await this.reconcileCore(reconcileHost, revalidate)
+ // Preserve the historical caller-visible reflection contract even when
+ // follow-on work correctly used a fresher API object. The tracker may hold
+ // the originally admitted instance until the watch delivers the next
+ // MODIFIED event; reflect the final authoritative outcome immediately so
+ // it cannot make a second decision from the pre-follow-on status.
+ this.reflectOutcome(host, target => {
+ target.uid = reconcileHost.uid
+ target.generation = reconcileHost.generation
+ target.resourceVersion = reconcileHost.resourceVersion
+ target.annotations = reconcileHost.annotations
+ target.spec = reconcileHost.spec
+ target.status = reconcileHost.status
+ })
// The wake fast-path / drained-pre-scale guard inside reconcile() may
// have flipped the host back to active (a wake raced the suspension) —
// in that case no scale-down happened, so no phase/metric is emitted.
const wakePending =
- this.wakeRequestedGeneration(host) > (host.status?.lifecycle?.wakeHandledGeneration ?? 0)
- if (host.status?.lifecycle?.state !== 'suspended' || wakePending) {
+ this.wakeRequestedGeneration(reconcileHost) >
+ (reconcileHost.status?.lifecycle?.wakeHandledGeneration ?? 0)
+ if (reconcileHost.status?.lifecycle?.state !== 'suspended' || wakePending) {
// The durable suspended write DID commit (a racing wake revived the
// host afterwards), so the drained evidence was honored: 'suspended'.
return 'suspended'
}
- this.recordSuspendedApplied(host.name)
+ this.recordSuspendedApplied(reconcileHost.name)
return 'suspended'
}
@@ -838,12 +929,19 @@ export class StatelessLifecycleExecutor {
* Throws on API failure — the tracker logs it loudly.
*/
async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {
- // L2: serialize on the host chain (see serializeByHost) so this status
- // write never interleaves with a concurrent reconcile for the same host.
- return this.serializeByHost(host.name, () => this.publishSuspendBlockedReasonCore(host, reason))
+ return this.serializeHeartbeatMutation(
+ 'heartbeat suspend-blocked reason publication',
+ host,
+ (admittedHost, revalidate) =>
+ this.publishSuspendBlockedReasonCore(admittedHost, reason, revalidate)
+ )
}
- private async publishSuspendBlockedReasonCore(host: HostCRD, reason: string): Promise<void> {
+ private async publishSuspendBlockedReasonCore(
+ host: HostCRD,
+ reason: string,
+ revalidate?: () => HostCRD
+ ): Promise<void> {
// FRESH GET (see readFreshHost): this is a suspend-BLOCKED reason
// ANNOTATOR, not a state writer — it must NEVER change state, only stamp
// the reason on top of the CURRENT server-side state. AP-1 (the 9th
@@ -866,6 +964,12 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
+ if (
+ !this.sameHostSpecRevision(host, fresh) ||
+ !this.effectiveLifecycleFromCache(fresh).stateless
+ ) {
+ return { skip: true }
+ }
const freshLifecycle = fresh.status?.lifecycle
const freshState = freshLifecycle?.state ?? 'active'
if (freshState !== 'active') {
@@ -889,12 +993,16 @@ export class StatelessLifecycleExecutor {
writtenLifecycle = lifecycle
return [{ op: 'add', path: '/status/lifecycle', value: lifecycle }]
},
- initialFresh
+ initialFresh,
+ revalidate
)
if ('skipped' in result || writtenLifecycle === undefined) {
return
}
- host.status = { ...(host.status ?? {}), lifecycle: writtenLifecycle }
+ const reflectedStatus = { ...(host.status ?? {}), lifecycle: writtenLifecycle }
+ this.reflectOutcome(host, target => {
+ target.status = reflectedStatus
+ })
this.lastWrittenLifecycleStatus.delete(host.name)
}
@@ -926,17 +1034,18 @@ export class StatelessLifecycleExecutor {
host: HostCRD,
entryWakeHandledGeneration: number
): Promise<void> {
- // L2: serialize on the host chain (see serializeByHost) so the drain
- // status write never interleaves with a concurrent reconcile for the
- // same host.
- return this.serializeByHost(host.name, () =>
- this.markHostDrainingFromHeartbeatCore(host, entryWakeHandledGeneration)
+ return this.serializeHeartbeatMutation(
+ 'heartbeat draining transition',
+ host,
+ (admittedHost, revalidate) =>
+ this.markHostDrainingFromHeartbeatCore(admittedHost, entryWakeHandledGeneration, revalidate)
)
}
private async markHostDrainingFromHeartbeatCore(
host: HostCRD,
- entryWakeHandledGeneration: number
+ entryWakeHandledGeneration: number,
+ revalidate?: () => HostCRD
): Promise<void> {
// FRESH GET (see readFreshHost): a stale `active` snapshot over an
// actually-`suspended` CR would write `draining` (a running state) and
@@ -948,6 +1057,12 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
+ if (
+ !this.sameHostSpecRevision(host, fresh) ||
+ !this.effectiveLifecycleFromCache(fresh).stateless
+ ) {
+ return { skip: true }
+ }
const freshLifecycle = fresh.status?.lifecycle
if ((freshLifecycle?.state ?? 'active') !== 'active') {
return { skip: true }
@@ -983,12 +1098,15 @@ export class StatelessLifecycleExecutor {
writtenStatus = { ...(fresh.status ?? {}), lifecycle }
return this.fullStatusOps(writtenStatus)
},
- initialFresh
+ initialFresh,
+ revalidate
)
if ('skipped' in result || writtenStatus === undefined) {
return
}
- host.status = writtenStatus
+ this.reflectOutcome(host, target => {
+ target.status = writtenStatus
+ })
this.lastWrittenLifecycleStatus.delete(host.name)
console.log(
`[HostReconciler] Marked stateless Host "${host.name}" draining — control-api now answers {drain:true} from Host.status`
@@ -1010,13 +1128,17 @@ export class StatelessLifecycleExecutor {
* heartbeat retries.
*/
async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {
- // L2: serialize on the host chain (see serializeByHost) so the
- // cancel-drain status write never interleaves with a concurrent
- // reconcile for the same host.
- return this.serializeByHost(host.name, () => this.markHostActiveFromHeartbeatCore(host))
+ return this.serializeHeartbeatMutation(
+ 'heartbeat cancel-drain transition',
+ host,
+ (admittedHost, revalidate) => this.markHostActiveFromHeartbeatCore(admittedHost, revalidate)
+ )
}
- private async markHostActiveFromHeartbeatCore(host: HostCRD): Promise<void> {
+ private async markHostActiveFromHeartbeatCore(
+ host: HostCRD,
+ revalidate?: () => HostCRD
+ ): Promise<void> {
// FRESH GET (see readFreshHost): a stale `draining` snapshot over an
// actually-`suspended` CR must never resurrect it (and would regress
// wakeHandledGeneration). Proceed only when the CURRENT server-side
@@ -1029,6 +1151,12 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
+ if (
+ !this.sameHostSpecRevision(host, fresh) ||
+ !this.effectiveLifecycleFromCache(fresh).stateless
+ ) {
+ return { skip: true }
+ }
const freshLifecycle = fresh.status?.lifecycle
if (freshLifecycle?.state !== 'draining') {
skippedNotDraining = freshLifecycle?.state ?? 'active'
@@ -1041,7 +1169,8 @@ export class StatelessLifecycleExecutor {
writtenStatus = { ...(fresh.status ?? {}), lifecycle }
return this.fullStatusOps(writtenStatus)
},
- initialFresh
+ initialFresh,
+ revalidate
)
if ('skipped' in result || writtenStatus === undefined) {
console.log(
@@ -1049,7 +1178,9 @@ export class StatelessLifecycleExecutor {
)
return
}
- host.status = writtenStatus
+ this.reflectOutcome(host, target => {
+ target.status = writtenStatus
+ })
this.lastWrittenLifecycleStatus.delete(host.name)
console.log(
`[StatelessLifecycle] host=${host.name} cancel-drain: activity evidence while draining — reverting status to active`
@@ -1165,8 +1296,10 @@ export class StatelessLifecycleExecutor {
// Reflect the fresh truth on the in-memory object so the heavy
// reconcile body derives replicas from the current durable state
// instead of the stale event payload.
- host.annotations = initialFresh.annotations
- host.status = initialFresh.status
+ this.reflectOutcome(host, target => {
+ target.annotations = initialFresh.annotations
+ target.status = initialFresh.status
+ })
return false
}
@@ -1225,8 +1358,10 @@ export class StatelessLifecycleExecutor {
initialFresh
)
if ('skipped' in result || writtenLifecycle === undefined) {
- host.annotations = latestFresh.annotations
- host.status = latestFresh.status
+ this.reflectOutcome(host, target => {
+ target.annotations = latestFresh.annotations
+ target.status = latestFresh.status
+ })
return false
}
} catch (err) {
@@ -1245,8 +1380,11 @@ export class StatelessLifecycleExecutor {
// (assessLifecycle reads host.status) derives replicas from the woken
// state instead of re-suspending from the stale event payload. The base
// is the FRESH status the write committed over, never the cached spread.
- host.annotations = latestFresh.annotations
- host.status = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }
+ const reflectedStatus = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }
+ this.reflectOutcome(host, target => {
+ target.annotations = latestFresh.annotations
+ target.status = reflectedStatus
+ })
console.log(
`[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=status_flipped ts=${this.now().getTime()}`
)
@@ -1417,19 +1555,31 @@ export class StatelessLifecycleExecutor {
private async patchStatusWithPrecondition(
host: HostCRD,
build: (fresh: HostCRD) => Array<{ op: 'add'; path: string; value: unknown }> | { skip: true },
- initialFresh?: HostCRD
+ initialFresh?: HostCRD,
+ revalidate?: () => HostCRD
): Promise<{ fresh: HostCRD } | { skipped: true }> {
const maxAttempts = 5
let fresh = initialFresh
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+ const currentAtAttempt = revalidate?.()
+ if (currentAtAttempt) this.requireSameHostIdentity(host, currentAtAttempt)
if (fresh === undefined) {
fresh = await this.readFreshHost(host)
}
+ if (host.uid !== undefined && fresh.resourceVersion === undefined) {
+ const error = new Error(
+ `Host status mutation for "${host.name}" has no fresh resourceVersion; refusing an unconditioned write`
+ )
+ error.name = 'HostMutationPreconditionUnavailableError'
+ throw error
+ }
const built = build(fresh)
if ('skip' in built) {
return { skipped: true }
}
try {
+ const currentBeforePatch = revalidate?.()
+ if (currentBeforePatch) this.requireSameHostIdentity(host, currentBeforePatch)
await this.customApi.patchNamespacedCustomObjectStatus({
group: GROUP,
version: VERSION,
@@ -1499,6 +1649,79 @@ export class StatelessLifecycleExecutor {
return result
}
+ /**
+ * Capture admission before enqueueing, then resolve the current Host only
+ * after this mutation reaches the front of its own serializer. Keeping the
+ * check here avoids wrapping a lifecycle call in HostReconciler's serializer,
+ * which would recursively acquire the same Host chain and deadlock.
+ */
+ private serializeHeartbeatMutation<T>(
+ action: string,
+ host: HostCRD,
+ work: (admittedHost: HostCRD, revalidate: () => HostCRD) => Promise<T>
+ ): Promise<T> {
+ const revalidate = this.prepareHostMutationAdmission(action, host)
+ return this.serializeByHost(host.name, () => work(revalidate(), revalidate))
+ }
+
+ /**
+ * A resourceVersion conflict can be caused by ordinary writes or by a
+ * delete/recreate. Every fresh read, including every bounded 409 retry, must
+ * remain on the Kubernetes UID that was admitted. Name equality is not an
+ * identity fence. Production watch snapshots always carry UID; if only one
+ * side carries it, fail closed rather than guessing.
+ */
+ /**
+ * H2: apply a committed outcome to BOTH visible surfaces:
+ * 1. the local working object (`host`) — with a resolver wired this is the
+ * admitted CLONE (serializeHeartbeatMutation), so this keeps same-pass
+ * logic (assessLifecycle, drained-pre-scale guard, follow-on reconcile)
+ * reading the outcome exactly as before; standalone callers (no resolver)
+ * get the legacy caller-visible reflection unchanged;
+ * 2. the CURRENT cache entry, via the uid-guarded reflector (no-op unless
+ * McpServerWatcher wired it).
+ * `host.uid` is the admitted identity (admission proved requested.uid ===
+ * current.uid; readFreshHost re-proves it), so passing it after mutate() ran
+ * is equivalent to passing it before.
+ *
+ * `mutate` closures MUST only ASSIGN precomputed values — never derive from
+ * `target` — so both surfaces receive the identical outcome.
+ */
+ private reflectOutcome(host: HostCRD, mutate: (target: HostCRD) => void): void {
+ mutate(host)
+ this.reflectHostOutcome(host.name, host.uid, mutate)
+ }
+
+ private requireSameHostIdentity(expected: HostCRD, current: HostCRD): void {
+ if (expected.uid === undefined && current.uid === undefined) return
+ if (expected.uid === undefined || current.uid === undefined || expected.uid !== current.uid) {
+ const error = new Error(
+ `Host identity changed while mutating "${expected.name}"; refusing to cross a same-name recreation`
+ )
+ error.name = 'HostMutationIdentityChangedError'
+ throw error
+ }
+ }
+
+ /**
+ * Heartbeat evidence belongs to one Host spec epoch. UID equality allows
+ * status-only updates and retries, but a generation change can replace the
+ * runtime inputs (context, Secret, image, lifecycle policy, etc.) while
+ * remaining stateless. Never apply an older runtime's heartbeat decision to
+ * that new spec. UID-less standalone fixtures retain their legacy behavior;
+ * production objects (which carry UID) fail closed on a missing generation.
+ */
+ private sameHostSpecRevision(expected: HostCRD, current: HostCRD): boolean {
+ if (expected.generation === undefined && current.generation === undefined) {
+ return expected.uid === undefined && current.uid === undefined
+ }
+ return (
+ expected.generation !== undefined &&
+ current.generation !== undefined &&
+ expected.generation === current.generation
+ )
+ }
+
/**
* Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
* suspension IMMEDIATELY before replicas:0 derives from the assessment.
@@ -1579,8 +1802,10 @@ export class StatelessLifecycleExecutor {
// A wake is pending in the fresh read — run the wake transition
// instead of the scale-down. Reflect the fresh wake generation on the
// in-memory object so downstream writes preserve it.
- host.annotations = fresh.annotations
- host.status = fresh.status
+ this.reflectOutcome(host, target => {
+ target.annotations = fresh.annotations
+ target.status = fresh.status
+ })
return {
effective: { stateless: true, state: 'active' },
lifecycle: { state: 'active', wakeHandledGeneration: freshWakeRequested },
@@ -1606,8 +1831,10 @@ export class StatelessLifecycleExecutor {
// Reflect fresh onto the in-memory host (mirroring the wake fast-path's
// epoch guard) so every downstream write derives from the current
// server-side truth instead of the stale event payload.
- host.annotations = fresh.annotations
- host.status = fresh.status
+ this.reflectOutcome(host, target => {
+ target.annotations = fresh.annotations
+ target.status = fresh.status
+ })
const freshReason = fresh.status?.lifecycle?.reason
const preservedReason = isHeartbeatManagedLifecycleReason(freshReason) ? freshReason : undefined
return {
diff --git a/host-context-controller/src/types.ts b/host-context-controller/src/types.ts
index a7488dc7d..552ed3f0f 100644
--- a/host-context-controller/src/types.ts
+++ b/host-context-controller/src/types.ts
@@ -350,6 +350,8 @@ export interface ContextStatus {
export interface ContextCRD {
name: string
namespace: string
+ uid?: string
+ generation?: number
spec: ContextSpec
status?: ContextStatus
}
@@ -509,6 +511,12 @@ export interface McpServerStatus {
ready: boolean
/** Human-readable status message. */
message?: string
+ /**
+ * Whether HCC can tie this status to the current McpServer identity and
+ * generation. False means discovery is authoritative but runtime status has
+ * not yet been re-established after controller startup.
+ */
+ authoritative?: boolean
}
/**
diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.ts
index 22d63b8b2..a6992eeb8 100644
--- a/host-context-controller/src/utils.ts
+++ b/host-context-controller/src/utils.ts
@@ -36,6 +36,10 @@ export async function replaceWithConflictRetry<
* when a meaningful change is still required.
*/
isUpToDate?: (body: T, existing: T) => boolean
+ /** Reject an object whose identity or ownership is unsafe to replace. */
+ validateExisting?: (existing: T) => void
+ /** Rechecked immediately before every Kubernetes write attempt. */
+ mutationAllowed?: () => boolean
maxAttempts?: number
}): Promise<void> {
const {
@@ -47,10 +51,13 @@ export async function replaceWithConflictRetry<
replace,
mergeExisting,
isUpToDate,
+ validateExisting,
+ mutationAllowed,
maxAttempts = 3,
} = opts
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
const existing = await read()
+ validateExisting?.(existing)
const desired = resolveBody ? await resolveBody() : body
const base: T = {
...desired,
@@ -58,6 +65,7 @@ export async function replaceWithConflictRetry<
}
const next = mergeExisting ? mergeExisting(base, existing) : base
if (isUpToDate?.(next, existing)) return
+ if (mutationAllowed && !mutationAllowed()) return
try {
await replace(next)
const suffix = attempt > 1 ? ` (after ${attempt} attempts)` : ''
@@ -171,8 +179,11 @@ export async function applyNetworkPolicy(
name: string,
namespace: string,
policy: k8s.V1NetworkPolicy,
- logPrefix = '[NetPol]'
+ logPrefix = '[NetPol]',
+ mutationAllowed?: () => boolean,
+ validateExisting?: (existing: k8s.V1NetworkPolicy) => void
): Promise<void> {
+ if (mutationAllowed && !mutationAllowed()) return
try {
await api.createNamespacedNetworkPolicy({ namespace, body: policy })
console.log(`${logPrefix} Created policy "${name}" in ${namespace}`)
@@ -188,6 +199,8 @@ export async function applyNetworkPolicy(
body: policy,
read: () => api.readNamespacedNetworkPolicy({ name, namespace }),
replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),
+ mutationAllowed,
+ validateExisting,
})
}
diff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.ts
index 4e6593024..17ac53f0f 100644
--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts
+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts
@@ -136,6 +136,20 @@ function createReconciler() {
return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }
}
+function deferred<T = void>(): {
+ promise: Promise<T>
+ resolve: (value: T) => void
+ reject: (reason?: unknown) => void
+} {
+ let resolve!: (value: T) => void
+ let reject!: (reason?: unknown) => void
+ const promise = new Promise<T>((res, rej) => {
+ resolve = res
+ reject = rej
+ })
+ return { promise, resolve, reject }
+}
+
/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */
function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {
const calls = [
@@ -207,7 +221,755 @@ function freshHostRead(lifecycle?: {
}
}
+describe('HostReconciler heartbeat mutation admission', () => {
+ const heartbeatMutations = [
+ {
+ name: 'suspend',
+ core: 'suspendHostFromHeartbeatCore',
+ coreArgs: ['idle', 0] as const,
+ freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,
+ invoke: (reconciler: HostReconciler, host: HostCRD) =>
+ reconciler.suspendHostFromHeartbeat(host, 'idle', 0),
+ },
+ {
+ name: 'suspend-blocked reason',
+ core: 'publishSuspendBlockedReasonCore',
+ coreArgs: ['suspend-blocked: activeTask'] as const,
+ freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,
+ invoke: (reconciler: HostReconciler, host: HostCRD) =>
+ reconciler.publishSuspendBlockedReason(host, 'suspend-blocked: activeTask'),
+ },
+ {
+ name: 'draining',
+ core: 'markHostDrainingFromHeartbeatCore',
+ coreArgs: [0] as const,
+ freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,
+ invoke: (reconciler: HostReconciler, host: HostCRD) =>
+ reconciler.markHostDrainingFromHeartbeat(host, 0),
+ },
+ {
+ name: 'cancel-drain',
+ core: 'markHostActiveFromHeartbeatCore',
+ coreArgs: [] as const,
+ freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,
+ invoke: (reconciler: HostReconciler, host: HostCRD) =>
+ reconciler.markHostActiveFromHeartbeat(host),
+ },
+ ] as const
+
+ it.each(heartbeatMutations)(
+ 'rejects queued $name work when Host authority changes before serializer admission',
+ async ({ core, invoke }) => {
+ const { reconciler } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 1 }
+ const authority = { current: { known: true, generation: 7 } }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => authority.current)
+ const lifecycle = (reconciler as any).lifecycle
+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = lifecycle.serializeByHost(host.name, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }) as Promise<void>
+ await blockerStarted.promise
+
+ const pending = invoke(reconciler, host)
+ authority.current = { known: false, generation: 8 }
+ releaseBlocker.resolve(undefined)
+ await occupied
+
+ await expect(pending).rejects.toThrow(/Host inventory authority/)
+ expect(coreSpy).not.toHaveBeenCalled()
+ }
+ )
+
+ it.each(heartbeatMutations)(
+ 'rejects queued $name work when the Host UID changes before serializer admission',
+ async ({ core, invoke }) => {
+ const { reconciler } = createReconciler()
+ const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-old-uid', generation: 4 }
+ let current = requested
+ const authority = { current: { known: true, generation: 9 } }
+ reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => authority.current)
+ const lifecycle = (reconciler as any).lifecycle
+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = lifecycle.serializeByHost(requested.name, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }) as Promise<void>
+ await blockerStarted.promise
+
+ const pending = invoke(reconciler, requested)
+ current = { ...requested, uid: 'heartbeat-host-new-uid', generation: 1 }
+ releaseBlocker.resolve(undefined)
+ await occupied
+
+ await expect(pending).rejects.toThrow(/Host identity/)
+ expect(coreSpy).not.toHaveBeenCalled()
+ }
+ )
+
+ it.each(heartbeatMutations)(
+ 'admits the latest same-UID same-spec Host snapshot for $name inside the heartbeat serializer',
+ async ({ core, coreArgs, invoke }) => {
+ const { reconciler } = createReconciler()
+ const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ const current = {
+ ...requested,
+ resourceVersion: 'cache-rv-latest',
+ status: { lifecycle: { state: 'active' as const, wakeHandledGeneration: 2 } },
+ }
+ reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))
+ const lifecycle = (reconciler as any).lifecycle
+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
+
+ await invoke(reconciler, requested)
+
+ expect(coreSpy).toHaveBeenCalledOnce()
+ expect(coreSpy).toHaveBeenCalledWith(current, ...coreArgs, expect.any(Function))
+ }
+ )
+
+ it.each(heartbeatMutations)(
+ 'rejects a $name 409 retry that resolves to a recreated same-name Host',
+ async ({ freshLifecycle, invoke }) => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),
+ uid: 'heartbeat-host-old-uid',
+ generation: 4,
+ }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 15 }))
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead(freshLifecycle),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-old',
+ },
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead(freshLifecycle),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: 'heartbeat-host-new-uid',
+ generation: 1,
+ resourceVersion: 'rv-new',
+ },
+ })
+ customApi.patchNamespacedCustomObjectStatus
+ .mockRejectedValueOnce({ code: 409 })
+ .mockResolvedValueOnce(undefined)
+
+ await expect(invoke(reconciler, host)).rejects.toThrow(/Host identity/)
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ }
+ )
+
+ it.each(heartbeatMutations)(
+ 'rejects queued $name work when the same Host advances to a new spec generation',
+ async ({ core, invoke }) => {
+ const { reconciler } = createReconciler()
+ const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ let current = requested
+ reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))
+ const lifecycle = (reconciler as any).lifecycle
+ const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
+ const blockerStarted = deferred()
+ const releaseBlocker = deferred()
+ const occupied = lifecycle.serializeByHost(requested.name, async () => {
+ blockerStarted.resolve(undefined)
+ await releaseBlocker.promise
+ }) as Promise<void>
+ await blockerStarted.promise
+
+ const pending = invoke(reconciler, requested)
+ current = {
+ ...requested,
+ generation: 5,
+ spec: { ...requested.spec, contextRef: 'new-runtime-context' },
+ }
+ releaseBlocker.resolve(undefined)
+ await occupied
+
+ await expect(pending).rejects.toThrow(/Host spec generation/)
+ expect(coreSpy).not.toHaveBeenCalled()
+ }
+ )
+
+ it.each(heartbeatMutations)(
+ 'does not apply stale $name evidence to a newer same-UID stateless spec',
+ async ({ freshLifecycle, invoke }) => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),
+ uid: 'heartbeat-host-uid',
+ generation: 4,
+ }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead(freshLifecycle),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-new-spec',
+ },
+ spec: { ...host.spec, contextRef: 'new-runtime-context' },
+ })
+ const reconcileCore = vi
+ .spyOn(reconciler as any, 'reconcileCore')
+ .mockResolvedValue(undefined)
+
+ await invoke(reconciler, host)
+
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(reconcileCore).not.toHaveBeenCalled()
+ }
+ )
+
+ it.each(heartbeatMutations)(
+ 'does not apply stale $name evidence after the same-UID Host becomes stateful',
+ async ({ invoke }) => {
+ const { reconciler, customApi } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 16 }))
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-stateful',
+ },
+ spec: {
+ ...host.spec,
+ lifecycle: { stateless: false },
+ },
+ })
+ const reconcileCore = vi
+ .spyOn(reconciler as any, 'reconcileCore')
+ .mockResolvedValue(undefined)
+
+ await invoke(reconciler, host)
+
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(reconcileCore).not.toHaveBeenCalled()
+ }
+ )
+
+ it('does not retry stale heartbeat evidence after a 409 reveals a same-UID stateful spec', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-stateless',
+ },
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-stateful',
+ },
+ spec: {
+ ...host.spec,
+ lifecycle: { stateless: false },
+ },
+ })
+ customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })
+
+ await reconciler.markHostDrainingFromHeartbeat(host, 0)
+
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ })
+
+ it('does not retry stale heartbeat evidence after a 409 reveals a newer stateless spec', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-old-spec',
+ },
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-new-spec',
+ },
+ spec: { ...host.spec, contextRef: 'new-runtime-context' },
+ })
+ customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })
+
+ await reconciler.markHostDrainingFromHeartbeat(host, 0)
+
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ })
+
+ it('revalidates authority after the fresh read and before the first status patch', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ const authority = { current: { known: true, generation: 18 } }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => authority.current)
+ customApi.getNamespacedCustomObject.mockImplementation(async () => {
+ authority.current = { known: false, generation: 19 }
+ return {
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ resourceVersion: 'rv-authority-lost',
+ },
+ }
+ })
+
+ await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
+ /Host inventory authority/
+ )
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ })
+
+ it('revalidates authority before a 409 retry performs another fresh read', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ const authority = { current: { known: true, generation: 21 } }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => authority.current)
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-before-conflict',
+ },
+ })
+ customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {
+ authority.current = { known: false, generation: 22 }
+ throw { code: 409 }
+ })
+
+ await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
+ /Host inventory authority/
+ )
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ })
+
+ it('does not reconcile runtime effects after suspend when the Host UID changes', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'heartbeat-host-old-uid',
+ generation: 4,
+ }
+ let current = host
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 24 }))
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-suspend',
+ },
+ })
+ customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {
+ current = { ...host, uid: 'heartbeat-host-new-uid', generation: 1 }
+ })
+
+ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
+ /Host identity/
+ )
+ expect(reconcileCore).not.toHaveBeenCalled()
+ })
+
+ it('reconciles suspend follow-on effects from the successful fresh same-UID Host spec', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({
+ spec: { contextRef: 'latest-context' },
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'heartbeat-host-uid',
+ generation: 5,
+ }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 26 }))
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-suspend-current',
+ },
+ spec: { ...host.spec, contextRef: 'latest-context' },
+ })
+ .mockResolvedValue({
+ ...freshHostRead({
+ state: 'suspended',
+ wakeHandledGeneration: 0,
+ reason: 'idle',
+ }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-suspend-committed',
+ },
+ spec: { ...host.spec, contextRef: 'latest-context' },
+ })
+
+ await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
+
+ expect(reconcileCore).toHaveBeenCalledOnce()
+ const reconciledHost = reconcileCore.mock.calls[0][0] as HostCRD
+ expect(reconciledHost).toMatchObject({
+ uid: host.uid,
+ generation: 5,
+ spec: expect.objectContaining({ contextRef: 'latest-context' }),
+ })
+ expect(reconciledHost.status?.lifecycle).toEqual({
+ state: 'suspended',
+ wakeHandledGeneration: 0,
+ reason: 'idle',
+ })
+ })
+
+ it('rejects a known-identity status write without a fresh resourceVersion', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 28 }))
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: { name: host.name, namespace: host.namespace, uid: host.uid },
+ })
+
+ await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
+ /no fresh resourceVersion/
+ )
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ })
+
+ it('retries a 409 on the same UID and uses the latest Host for suspend follow-on effects', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'heartbeat-host-uid',
+ generation: 4,
+ }
+ let current = host
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 30 }))
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-1',
+ },
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-2',
+ },
+ spec: host.spec,
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead({
+ state: 'suspended',
+ wakeHandledGeneration: 0,
+ reason: 'idle',
+ }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-3',
+ },
+ spec: host.spec,
+ })
+ customApi.patchNamespacedCustomObjectStatus
+ .mockImplementationOnce(async () => {
+ current = {
+ ...host,
+ resourceVersion: 'cache-rv-after-conflict',
+ }
+ throw { code: 409 }
+ })
+ .mockResolvedValueOnce(undefined)
+
+ await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
+
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(3)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
+ expect(reconcileCore.mock.calls[0][0]).toMatchObject({
+ uid: host.uid,
+ generation: 4,
+ spec: expect.objectContaining({ contextRef: 'context-a' }),
+ })
+ })
+
+ it('does not apply or record suspension when only the follow-on GET observes a newer spec', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'heartbeat-host-uid',
+ generation: 4,
+ }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 33 }))
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-suspend-write',
+ },
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead({
+ state: 'suspended',
+ wakeHandledGeneration: 0,
+ reason: 'idle',
+ }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 5,
+ resourceVersion: 'rv-new-spec',
+ },
+ spec: {
+ ...host.spec,
+ contextRef: 'replacement-context',
+ lifecycle: { stateless: false },
+ },
+ })
+ const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
+ const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')
+
+ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
+ /Host spec generation/
+ )
+
+ expect(reconcileCore).not.toHaveBeenCalled()
+ expect(recordSuspended).not.toHaveBeenCalled()
+ })
+
+ it('rejects stale suspend follow-on work when the Host spec changes during Secret validation', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'heartbeat-host-uid',
+ generation: 4,
+ }
+ let current = host
+ reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 34 }))
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-suspend-write',
+ },
+ })
+ .mockResolvedValueOnce({
+ ...freshHostRead({
+ state: 'suspended',
+ wakeHandledGeneration: 0,
+ reason: 'idle',
+ }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-suspend-follow-on',
+ },
+ })
+ const secretReadStarted = deferred()
+ const releaseSecretRead = deferred()
+ vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
+ secretReadStarted.resolve(undefined)
+ await releaseSecretRead.promise
+ return { ok: true }
+ })
+ const staleMutation = vi
+ .spyOn(reconciler as any, 'ensureHostServiceAccount')
+ .mockRejectedValue(new Error('stale follow-on mutation was admitted'))
+
+ const pending = reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
+ await secretReadStarted.promise
+ current = {
+ ...host,
+ generation: 5,
+ spec: {
+ ...host.spec,
+ contextRef: 'replacement-context',
+ lifecycle: { stateless: false },
+ },
+ }
+ releaseSecretRead.resolve(undefined)
+
+ await expect(pending).rejects.toThrow(/Host spec generation/)
+ expect(staleMutation).not.toHaveBeenCalled()
+ })
+
+ it('does not record a suspended scale-down when follow-on reconcile handles a wake', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'heartbeat-host-uid',
+ generation: 4,
+ }
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 32 }))
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: host.uid,
+ generation: 4,
+ resourceVersion: 'rv-wake-follow-on',
+ },
+ })
+ vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async (...args: unknown[]) => {
+ const reconcileHost = args[0] as HostCRD
+ reconcileHost.annotations = { 'clerum.io/wake-requested': '1' }
+ reconcileHost.status = {
+ lifecycle: { state: 'active', wakeHandledGeneration: 1 },
+ }
+ })
+ const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')
+
+ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).resolves.toBe('suspended')
+
+ expect(recordSuspended).not.toHaveBeenCalled()
+ })
+})
+
describe('HostReconciler.suspendHostFromHeartbeat', () => {
+ it('does not mutate the admitted watch-cache snapshot while applying a suspension', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
+ })
+ host.uid = 'stateless-host-uid'
+ host.generation = 1
+ host.resourceVersion = 'cache-rv'
+ const snapshot = structuredClone(host)
+ Object.freeze(host)
+ reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
+ reconciler.setHostMutationAuthority(() => ({
+ known: true,
+ hostGeneration: 1,
+ contextGeneration: 1,
+ }))
+ const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })
+ Object.assign(draining.metadata, {
+ uid: host.uid,
+ generation: host.generation,
+ resourceVersion: 'draining-rv',
+ })
+ const suspended = freshHostRead({
+ state: 'suspended',
+ wakeHandledGeneration: 2,
+ reason: 'idle',
+ })
+ Object.assign(suspended.metadata, {
+ uid: host.uid,
+ generation: host.generation,
+ resourceVersion: 'suspended-rv',
+ })
+ customApi.getNamespacedCustomObject.mockResolvedValueOnce(draining).mockResolvedValue(suspended)
+
+ await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe('suspended')
+ expect(host).toEqual(snapshot)
+ })
+
it("writes state='suspended' + reason 'idle' durably, then reconciles to replicas=0", async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const host = makeStatelessHost({
@@ -1313,3 +2075,89 @@ describe('HostReconciler reconcile — stateless replicas derive from FRESH stat
}
})
})
+
+describe('H2: guarded cache reflection', () => {
+ it('reflects a committed lifecycle outcome onto the current cache entry without a watch event', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const entry = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'uid-1',
+ generation: 1,
+ resourceVersion: 'rv-1',
+ }
+ const hosts = new Map<string, HostCRD>([[entry.name, entry]])
+ reconciler.setResolveCurrentHost(name => hosts.get(name))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
+ // Exactly the k8sClient.ts wiring shape (uid-guarded apply onto the cache).
+ reconciler.setReflectHostOutcome((name, uid, apply) => {
+ const cached = hosts.get(name)
+ if (!cached || uid === undefined || cached.uid !== uid) return
+ apply(cached)
+ })
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: entry.name,
+ namespace: entry.namespace,
+ uid: 'uid-1',
+ generation: 1,
+ resourceVersion: 'fresh-rv',
+ },
+ })
+
+ await reconciler.markHostDrainingFromHeartbeat(entry, 0)
+
+ // The status write committed to the API…
+ expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')
+ // …and — the H2 fix — the committed outcome is visible on the CANONICAL cache
+ // entry without waiting for a watch MODIFIED. With the resolver wired, the
+ // heartbeat core mutates the admitted clone (Shape A), so this can ONLY be
+ // true if the guarded reflector routed the outcome onto the cache entry.
+ expect(hosts.get(entry.name)?.status?.lifecycle?.state).toBe('draining')
+ })
+
+ it('never reflects a committed outcome onto a same-name recreation with a different uid', async () => {
+ const { reconciler, customApi } = createReconciler()
+ const entry = {
+ ...makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
+ }),
+ uid: 'uid-1',
+ generation: 1,
+ resourceVersion: 'rv-1',
+ }
+ const hosts = new Map<string, HostCRD>([[entry.name, entry]])
+ const replacement = { ...makeStatelessHost({ name: entry.name }), uid: 'uid-2' }
+ reconciler.setResolveCurrentHost(name => hosts.get(name))
+ reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
+ reconciler.setReflectHostOutcome((name, uid, apply) => {
+ const cached = hosts.get(name)
+ if (!cached || uid === undefined || cached.uid !== uid) return
+ apply(cached)
+ })
+ customApi.getNamespacedCustomObject.mockResolvedValue({
+ ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
+ metadata: {
+ name: entry.name,
+ namespace: entry.namespace,
+ uid: 'uid-1',
+ generation: 1,
+ resourceVersion: 'fresh-rv',
+ },
+ })
+ // A watch DELETE+ADD recreates the entry (new uid) mid-commit.
+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(async () => {
+ hosts.set(entry.name, replacement)
+ return {}
+ })
+
+ await reconciler.markHostDrainingFromHeartbeat(entry, 0)
+
+ // The write committed…
+ expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')
+ // …but the recreation (uid-2) was NOT corrupted by the stale reflection.
+ expect(replacement.status?.lifecycle).toBeUndefined()
+ })
+})
diff --git a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts
index a06465eeb..f8696e57e 100644
--- a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts
@@ -34,9 +34,12 @@ describe('ensureK8sApiEgress', () => {
mockApply.mockClear()
// ensureDefaultPolicies() also prunes legacy static policies via the
// networking API directly (not applyNetworkPolicy), so the mocked client
- // must expose deleteNamespacedNetworkPolicy.
+ // must expose the read-before-delete legacy cleanup path.
const mockKc = {
makeApiClient: () => ({
+ readNamespacedNetworkPolicy: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
}),
} as unknown as k8s.KubeConfig
diff --git a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts
index 6a89edc00..04e7b3b33 100644
--- a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts
@@ -26,7 +26,9 @@ const mockApply = vi.mocked(applyNetworkPolicy)
function createMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),
+ readNamespacedNetworkPolicy: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
diff --git a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts
index ae59d2dbd..9d0db5860 100644
--- a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts
@@ -32,7 +32,9 @@ const mockApply = vi.mocked(applyNetworkPolicy)
function createMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),
+ readNamespacedNetworkPolicy: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
diff --git a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts
index a93e20eed..6acb165b3 100644
--- a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts
@@ -83,7 +83,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('generates egress policy in rpc-proxy namespace when context has servers', async () => {
const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])
- await reconciler.reconcileContext(ctx)
+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)
@@ -91,7 +91,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('egress policy targets correct server pod by mcpserver label', async () => {
const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])
- await reconciler.reconcileContext(ctx)
+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)
@@ -103,7 +103,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('egress policy uses correct port from McpServer spec', async () => {
const ctx = makeContext('ctx-beta', ['airtable-mcp'])
- await reconciler.reconcileContext(ctx)
+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })
const rpcProxyCalls = mockApply.mock.calls.filter(
call => call[1] === 'rpc-egress-ctx-beta-airtable-mcp'
@@ -122,6 +122,8 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
metadata: {
name: 'rpc-egress-ctx-gamma-mongodb-mcp',
namespace: 'rpc-proxy',
+ uid: 'rpc-delete-uid',
+ resourceVersion: '7',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/policy-type': 'rpc-proxy-egress',
@@ -144,7 +146,22 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('deletes orphaned egress policies when server removed from context', async () => {
// First reconcile with both servers
const ctx1 = makeContext('ctx-delta', ['mongodb-mcp', 'airtable-mcp'])
- await reconciler.reconcileContext(ctx1)
+ await reconciler.reconcileContext(ctx1, { isCurrent: () => true })
+ const existingRpcProxyPolicies = mockApply.mock.calls
+ .filter(call => call[2] === 'rpc-proxy')
+ .map(call => {
+ const policy = call[3] as k8s.V1NetworkPolicy
+ const name = policy.metadata?.name
+ return {
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ uid: `rpc-proxy:${name}:uid`,
+ resourceVersion: '1',
+ },
+ }
+ })
+ expect(existingRpcProxyPolicies).toHaveLength(2)
vi.clearAllMocks()
mockNetApi.listNamespacedNetworkPolicy.mockImplementation(
@@ -154,10 +171,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
String(labelSelector).includes('clerum.io/context=ctx-delta')
) {
return {
- items: [
- { metadata: { name: 'rpc-egress-ctx-delta-mongodb-mcp' } },
- { metadata: { name: 'rpc-egress-ctx-delta-airtable-mcp' } },
- ],
+ items: existingRpcProxyPolicies,
}
}
return { items: [] }
@@ -166,7 +180,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
// Now reconcile with only one server — orphaned rpc-proxy policy should be cleaned
const ctx2 = makeContext('ctx-delta', ['mongodb-mcp'])
- await reconciler.reconcileContext(ctx2)
+ await reconciler.reconcileContext(ctx2, { isCurrent: () => true })
// applyNetworkPolicy should be called for mongodb-mcp in rpc-proxy but not airtable-mcp
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
@@ -176,6 +190,12 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
expect(mockNetApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'rpc-egress-ctx-delta-airtable-mcp',
namespace: 'rpc-proxy',
+ body: {
+ preconditions: {
+ uid: 'rpc-proxy:rpc-egress-ctx-delta-airtable-mcp:uid',
+ resourceVersion: '1',
+ },
+ },
})
expect(mockNetApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
name: 'rpc-egress-ctx-delta-mongodb-mcp',
@@ -185,7 +205,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('skips egress generation when server not found in cache', async () => {
const ctx = makeContext('ctx-epsilon', ['nonexistent-server'])
- await reconciler.reconcileContext(ctx)
+ await reconciler.reconcileContext(ctx, { isCurrent: () => true })
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
expect(rpcProxyCalls.length).toBe(0)
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 15f46834e..8519e148d 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.240",
+ "version": "0.1.239",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.240",
+ "version": "0.1.239",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 0fd114fe5..2dcb39dec 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.240",
+ "version": "0.1.239",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/__tests__/main.mcpInitialization.test.ts b/mcp-host/src/__tests__/main.mcpInitialization.test.ts
index c90bad869..02e9e9c75 100644
--- a/mcp-host/src/__tests__/main.mcpInitialization.test.ts
+++ b/mcp-host/src/__tests__/main.mcpInitialization.test.ts
@@ -22,7 +22,12 @@ function readyServer(overrides: Partial<McpServerInfo> = {}): McpServerInfo {
transport: { type: 'streamableHttp', url: 'http://secured-server.test/mcp' },
auth: { type: 'bearer', secretRef: 'secured-server-auth' },
enabled: true,
- status: { deployed: true, ready: true },
+ // Mirror the real producer contract: HCC's reconciler.getStatus() (surfaced
+ // through k8sClient.toServerInfo) always stamps `authoritative` explicitly —
+ // a ready, identity-matched snapshot carries authoritative: true. The legacy
+ // shape that omitted it is never emitted for a live server, so the default
+ // fixture must carry it too.
+ status: { deployed: true, ready: true, authoritative: true },
...overrides,
}
}
diff --git a/nginx-egress-proxy/README.md b/nginx-egress-proxy/README.md
index b4c205974..372f8e5aa 100644
--- a/nginx-egress-proxy/README.md
+++ b/nginx-egress-proxy/README.md
@@ -53,10 +53,10 @@ reference is platform-controlled:
| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | host-context-controller | `clerum/nginx-egress-proxy:0.1.0` |
| `CONTROL_API_REMOTE_MCP_EGRESS_PROXY_IMAGE` | control-api (registry install stamps `spec.image` for remote entries) | `clerum/nginx-egress-proxy:0.1.0` |
-HCC additionally canonicalizes `spec.image` of remote McpServers back to the
-platform image (`canonicalizeRemoteEgressProxyImage`) as best-effort cleanup;
-the enforcement is that Deployment rendering always stamps the configured
-platform image, so a CRD's `spec.image` is never what runs.
+HCC does not rewrite `spec.image` during reconciliation: that field remains
+desired state owned by the CRD author. Deployment rendering always stamps the
+configured platform image for remote egress proxies, so a remote CRD's
+`spec.image` is never what runs.
## Ports
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index fde59308e..38afc5704 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.64",
+ "version": "0.1.65",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.64",
+ "version": "0.1.65",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index 2c7a4f3ed..a17ac4862 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.64",
+ "version": "0.1.65",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/config.hccBaseUrl.test.ts b/rpc-proxy/src/config.hccBaseUrl.test.ts
index b1cedb606..bc3543a8d 100644
--- a/rpc-proxy/src/config.hccBaseUrl.test.ts
+++ b/rpc-proxy/src/config.hccBaseUrl.test.ts
@@ -27,16 +27,33 @@ describe('rpc-proxy HCC base URL default', () => {
)
const configmapMatch = configmap.match(/RPC_PROXY_HCC_BASE_URL:\s*"?([^"\n]+)"?/)
- it('parses both sources', () => {
+ // Derive the real endpoint from the api-gateway manifest itself, so renaming
+ // the Service or moving its port breaks this test unless the dev default and
+ // configmap are updated in lockstep — no hand-copied literal to drift.
+ const gatewayManifest = readFileSync(
+ path.join(repoRoot, 'deploy/base/control-plane/host-context-controller-api-gateway.yaml'),
+ 'utf8'
+ )
+ const serviceDoc = gatewayManifest.split(/^---$/m).find(doc => /\bkind:\s*Service\b/.test(doc))
+ const serviceName = serviceDoc?.match(/\bmetadata:[\s\S]*?\bname:\s*([\w.-]+)/)?.[1]
+ const serviceNamespace = serviceDoc?.match(/\bnamespace:\s*([\w.-]+)/)?.[1]
+ const servicePort = serviceDoc?.match(/\bports:[\s\S]*?\bport:\s*(\d+)/)?.[1]
+
+ it('parses all three sources', () => {
expect(defaultMatch?.[1]).toBeTruthy()
expect(configmapMatch?.[1]).toBeTruthy()
+ expect(serviceName).toBeTruthy()
+ expect(serviceNamespace).toBeTruthy()
+ expect(servicePort).toBeTruthy()
})
it('dev default matches the deployed configmap endpoint', () => {
expect(defaultMatch?.[1]).toBe(configmapMatch?.[1]?.trim())
})
- it('targets the api-gateway in control-plane, not a nonexistent Service', () => {
- expect(defaultMatch?.[1]).toContain('host-context-controller-api-gateway.control-plane')
+ it('targets the api-gateway Service exactly as the gateway manifest declares it', () => {
+ const url = new URL(defaultMatch![1])
+ expect(url.hostname).toBe(`${serviceName}.${serviceNamespace}.svc.cluster.local`)
+ expect(url.port).toBe(servicePort)
})
})
diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
index bc77ee0f1..ecb97cbdc 100755
--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
@@ -1,28 +1,82 @@
#!/usr/bin/env bash
-# Safety helpers for the HCC CommunicationChannel watch-recovery fault fixture.
+# Safety helpers shared by disruptive HCC fault-injection fixtures.
+
+profile_env_value() {
+ local key=$1 file=$2
+ awk -v key="$key" '
+ index($0, key "=") == 1 {
+ print substr($0, length(key) + 2)
+ exit
+ }
+ ' "$file"
+}
require_branch_owned_hcc_gate() {
- local repo_root expected_worktree_id expected_head actual_worktree_id actual_head worktree_dirty
+ local marker_namespace="${1:-control-plane}"
+ local actual_worktree_id actual_head actual_cluster_fingerprint actual_gate
+ local worktree_dirty expected_branch profile_env
+ local pre_gate_state_root cluster_fingerprint_file expected_cluster_fingerprint
is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||
die "fault injection requires a generated branch-scoped context, got '${E2E_KUBECONTEXT}'"
- repo_root="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"
- worktree_dirty="$(git -C "$repo_root" status --porcelain --untracked-files=normal)"
+ HCC_BRANCH_GATE_REPO_ROOT="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"
+ worktree_dirty="$(
+ git -C "$HCC_BRANCH_GATE_REPO_ROOT" status --porcelain --untracked-files=normal
+ )"
[ -z "$worktree_dirty" ] ||
die "worktree has uncommitted changes; commit and re-sync before runtime proof"
- expected_worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"
- expected_head="$(git -C "$repo_root" rev-parse HEAD)"
- actual_worktree_id="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \
- -o jsonpath='{.data.worktreeId}' 2>/dev/null || true)"
- actual_head="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \
- -o jsonpath='{.data.gitHead}' 2>/dev/null || true)"
+ HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID="$(
+ printf '%s' "$HCC_BRANCH_GATE_REPO_ROOT" | shasum | awk '{print $1}'
+ )"
+ HCC_BRANCH_GATE_EXPECTED_HEAD="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" rev-parse HEAD)"
+ expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"
+ [ -n "$expected_branch" ] ||
+ die "branch-owned runtime proof requires a named branch, not detached HEAD"
+ [ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ] ||
+ die "MINIKUBE_PROFILE must explicitly select context ${E2E_KUBECONTEXT}, got ${MINIKUBE_PROFILE:-missing}"
+
+ profile_env="${E2E_BRANCH_PROFILE_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/profile.env}"
+ [ -r "$profile_env" ] ||
+ die "branch-profile helper evidence is missing at ${profile_env}; run a state-writing branch-profile helper action from this worktree"
+ [ "$(profile_env_value PROFILE "$profile_env")" = "$E2E_KUBECONTEXT" ] ||
+ die "branch-profile helper output does not select context ${E2E_KUBECONTEXT}"
+ [ "$(profile_env_value REPO_DIR "$profile_env")" = "$HCC_BRANCH_GATE_REPO_ROOT" ] ||
+ die "branch-profile helper output belongs to a different worktree"
+ [ "$(profile_env_value BRANCH "$profile_env")" = "$expected_branch" ] ||
+ die "branch-profile helper output belongs to a different branch"
+ [ "$(profile_env_value DIRTY "$profile_env")" = false ] ||
+ die "branch-profile helper recorded a dirty worktree; refresh it after committing"
+
+ pre_gate_state_root="${E2E_PRE_GATE_STATE_ROOT:-${TMPDIR:-/tmp}/clerum-pre-gate-sync}"
+ cluster_fingerprint_file="${pre_gate_state_root}/${HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID}/cluster.sha"
+ [ -r "$cluster_fingerprint_file" ] ||
+ die "pre-gate fingerprint evidence is missing at ${cluster_fingerprint_file}"
+ expected_cluster_fingerprint="$(sed -n '1p' "$cluster_fingerprint_file")"
+ [[ "$expected_cluster_fingerprint" =~ ^[0-9a-f]{40}$ ]] ||
+ die "pre-gate fingerprint evidence is malformed"
+ [ -n "${E2E_EXPECTED_PRE_GATE_GATE:-}" ] ||
+ die "E2E_EXPECTED_PRE_GATE_GATE must name the gate recorded by the branch-owned pre-gate sync"
+
+ HCC_BRANCH_GATE_SYNC_MARKER="$(
+ kctl get configmap clerum-pre-gate-sync-state -n "$marker_namespace" -o json 2>/dev/null
+ )" || die "cluster has no readable pre-gate sync marker"
+ actual_worktree_id="$(jq -r '.data.worktreeId // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
+ actual_head="$(jq -r '.data.gitHead // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
+ actual_cluster_fingerprint="$(
+ jq -r '.data.clusterFingerprint // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER"
+ )"
+ actual_gate="$(jq -r '.data.gate // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
- [ "$actual_worktree_id" = "$expected_worktree_id" ] ||
+ [ "$actual_worktree_id" = "$HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID" ] ||
die "cluster ownership marker does not match this worktree; run minikube-pre-gate-sync first"
- [ "$actual_head" = "$expected_head" ] ||
- die "cluster HEAD marker ${actual_head:-missing} does not match ${expected_head}"
+ [ "$actual_head" = "$HCC_BRANCH_GATE_EXPECTED_HEAD" ] ||
+ die "cluster HEAD marker ${actual_head:-missing} does not match ${HCC_BRANCH_GATE_EXPECTED_HEAD}"
+ [ "$actual_cluster_fingerprint" = "$expected_cluster_fingerprint" ] ||
+ die "cluster fingerprint marker does not match this worktree's last pre-gate sync"
+ [ "$actual_gate" = "$E2E_EXPECTED_PRE_GATE_GATE" ] ||
+ die "cluster gate marker ${actual_gate:-missing} does not match expected ${E2E_EXPECTED_PRE_GATE_GATE}"
}
create_hcc_api_proxy() {
@@ -284,10 +338,10 @@ delete_hcc_proxy_fixture() {
}
restore_hcc_after_fault_injection() {
- local host_override port_override host_aliases desired ready
+ local host_override port_override api_cidrs_override host_aliases desired ready
kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
- KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- >/dev/null || return 1
+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1
kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=merge \
-p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1
kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null || return 1
@@ -296,10 +350,12 @@ restore_hcc_after_fault_injection() {
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"
port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"
+ api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"
host_aliases="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')"
desired="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"
ready="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.status.readyReplicas}')"
- [ -z "$host_override$port_override$host_aliases" ] && [ "$desired" = "$ready" ]
+ [ -z "$host_override$port_override$api_cidrs_override$host_aliases" ] && [ "$desired" = "$ready" ]
}
print_hcc_repair_instructions() {
@@ -308,7 +364,7 @@ HCC restoration failed. Proxy resources were retained for repair.
Context: ${E2E_KUBECONTEXT}
Proxy: ${HCC_NS}/${PROXY_NAME}
Repair commands:
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT-
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS-
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} patch deployment/${HCC_DEPLOY} --type=merge -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}'
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s
After HCC is healthy, delete ${PROXY_NAME}, ${PROXY_EGRESS_NP}, ${HCC_PROXY_NP}, and ${PROBE_EGRESS_NP} in ${HCC_NS}.
diff --git a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh b/scripts/e2e/_lib/hcc-watch-recovery-logs.sh
index b227248a6..f0857c3d5 100755
--- a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh
+++ b/scripts/e2e/_lib/hcc-watch-recovery-logs.sh
@@ -2,6 +2,18 @@
# Log capture and causal recovery-cycle assertions for the HCC watch gate.
+hcc_log_snapshot_contains() {
+ local snapshot=$1 marker=$2
+ grep -Fq -- "$marker" <<<"$snapshot"
+}
+
+hcc_initial_pass_snapshot_is_active() {
+ local snapshot=$1 start_marker=$2 complete_marker=$3 fail_marker=$4
+ hcc_log_snapshot_contains "$snapshot" "$start_marker" &&
+ ! hcc_log_snapshot_contains "$snapshot" "$complete_marker" &&
+ ! hcc_log_snapshot_contains "$snapshot" "$fail_marker"
+}
+
hcc_recovery_logs() {
[ -n "$START_TIME" ] || return 0
grep -E 'CommunicationChannel|Listing all Hosts|Reconciling [0-9]+ Host|Completed Host reconciliation|Periodic resync' \
@@ -59,7 +71,8 @@ recovery_cycle_used_fresh_host_inventory() {
hcc_recovery_logs | awk \
-v target="$cycle" -v host_namespace="$HOST_NS" -v expected="$expected_count" '
BEGIN {
- inventory_line = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"
+ channel_inventory = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"
+ host_inventory = "Reconciling " expected " Host(s) after Host watch recovery convergence"
succeeded = 0
}
/CommunicationChannel watch ended;/ {
@@ -68,7 +81,7 @@ recovery_cycle_used_fresh_host_inventory() {
if (inside) {
listed = 0
recovered = 0
- inventory_matched = 0
+ completion_mode = ""
}
next
}
@@ -78,11 +91,20 @@ recovery_cycle_used_fresh_host_inventory() {
inside && /Recovered [0-9]+ CommunicationChannel\(s\) into cache/ {
recovered = 1
}
- inside && index($0, inventory_line) > 0 {
- inventory_matched = 1
+ inside && listed && recovered && index($0, channel_inventory) > 0 {
+ completion_mode = "channel"
}
- inside && /Completed Host reconciliation after CommunicationChannel recovery$/ {
- succeeded = listed && recovered && inventory_matched
+ inside && listed && recovered && index($0, host_inventory) > 0 {
+ completion_mode = "host"
+ }
+ inside && completion_mode == "channel" &&
+ /Completed Host reconciliation after CommunicationChannel recovery$/ {
+ succeeded = 1
+ exit
+ }
+ inside && completion_mode == "host" &&
+ /Completed Host reconciliation after Host watch recovery convergence$/ {
+ succeeded = 1
exit
}
END {
@@ -90,3 +112,15 @@ recovery_cycle_used_fresh_host_inventory() {
}
'
}
+
+wait_recovery_cycle_fresh_host_inventory() {
+ local cycle=$1 expected_count=$2 timeout=$3 elapsed=0
+ while [ "$elapsed" -lt "$timeout" ]; do
+ require_hcc_recovery_log_stream || return 1
+ recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_count" && return 0
+ sleep 2
+ elapsed=$((elapsed + 2))
+ done
+ hcc_recovery_logs >&2
+ return 1
+}
diff --git a/scripts/e2e/e2e-crd-field-injection.sh b/scripts/e2e/e2e-crd-field-injection.sh
index af8a25829..d4fe358f8 100755
--- a/scripts/e2e/e2e-crd-field-injection.sh
+++ b/scripts/e2e/e2e-crd-field-injection.sh
@@ -55,6 +55,7 @@ require_safe_kube_context || exit 1
cleanup_e2e_resources
ensure_e2e_resources_absent ||
fail "Stale e2e-sec resources remain after cleanup; refusing stale-positive E2E run"
+# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.
[ "$e2e_fail" -gt 0 ] && exit 1
# ═══ Test 1: imagePullPolicy override prevention ═════════════════════
@@ -78,8 +79,11 @@ spec:
url: http://e2e-sec-pullpolicy.mcp-server.svc:3000/mcp
EOF
-log "Waiting for HCC to create Deployment..."
-sleep 15
+if wait_for_deployment "$MCP_NS" e2e-sec-pullpolicy "$TIMEOUT_POD"; then
+ ok "imagePullPolicy fixture rendered a ready Deployment"
+else
+ fail "imagePullPolicy fixture did not render a ready Deployment"
+fi
PULL_POLICY=$(kctl get deployment e2e-sec-pullpolicy -n "$MCP_NS" \
-o jsonpath='{.spec.template.spec.containers[0].imagePullPolicy}' 2>/dev/null || echo "NOT_FOUND")
@@ -237,7 +241,11 @@ spec:
value: postgres://safe
EOF
-sleep 15
+if wait_for_deployment "$MCP_NS" e2e-sec-envvars "$TIMEOUT_POD"; then
+ ok "Dangerous-env fixture rendered a ready Deployment"
+else
+ fail "Dangerous-env fixture did not render a ready Deployment"
+fi
ENV_JSON=$(kctl get deployment e2e-sec-envvars -n "$MCP_NS" \
-o jsonpath='{.spec.template.spec.containers[0].env}' 2>/dev/null || echo "[]")
@@ -276,7 +284,11 @@ spec:
- DAC_OVERRIDE
EOF
-sleep 15
+if wait_for_deployment "$MCP_NS" e2e-sec-uid70 "$TIMEOUT_POD"; then
+ ok "Non-root UID fixture rendered a ready Deployment"
+else
+ fail "Non-root UID fixture did not render a ready Deployment"
+fi
UID_70=$(kctl get deployment e2e-sec-uid70 -n "$MCP_NS" \
-o jsonpath='{.spec.template.spec.securityContext.runAsUser}' 2>/dev/null || echo "NOT_FOUND")
@@ -292,5 +304,6 @@ ensure_e2e_resources_absent && ok "All e2e-sec resources cleaned up" || fail "e2
# ═══ Summary ═════════════════════════════════════════════════════════
header "Summary"
+# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.
echo -e " ${GREEN}PASS${NC}: ${e2e_pass} ${RED}FAIL${NC}: ${e2e_fail} Total: ${e2e_total}"
[ "$e2e_fail" -gt 0 ] && exit 1 || exit 0
diff --git a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
index d1a38ed5b..480b9abd9 100755
--- a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
+++ b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
@@ -35,13 +35,15 @@ command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }
exit 1
}
kctl get nodes -o json | jq -e \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] != null)' >/dev/null || {
+ --arg context "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null || {
echo "Refusing HCC fault injection: target is not a minikube cluster." >&2
exit 1
}
HCC_NS="${HCC_NS:-control-plane}"
HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
+MCP_NS="${MCP_NS:-mcp-server}"
SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm-stateless}"
HOST_NS="${MCP_HOST_NS:-mcp-host}"
CHANNEL_NS="${CHANNELS_NS:-channels}"
@@ -61,7 +63,7 @@ HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-stream.XXXXXX")"
NON_FIXTURE_BASELINE="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-baseline.XXXXXX")"
NON_FIXTURE_AFTER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-after.XXXXXX")"
CLEANUP_KINDS="deployments,services,secrets,serviceaccounts,roles,rolebindings,networkpolicies,persistentvolumeclaims"
-START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY=""
+START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY="" ORIGINAL_REPLICAS=""
HCC_LOG_STREAM_PID=""
# Mirrors COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS in k8sClient.ts.
CC_RECOVERY_RETRY_SECONDS=5
@@ -75,6 +77,35 @@ HCC_GATE_FINALIZATION_FAILURE=""
die() { fail "$*"; exit 1; }
now_rfc3339() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
+# This gate deliberately changes only the HCC Kubernetes API path. If its
+# rollout fails before the normal recovery log stream starts, preserve the
+# bounded pod state and both container log buffers that distinguish a proxy,
+# policy, or process-startup failure. Cleanup still restores HCC afterwards.
+print_hcc_proxy_rollout_diagnostics() {
+ local pod
+
+ echo "HCC proxy rollout diagnostics (before restoration):" >&2
+ kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o json |
+ jq -r '
+ .items[] |
+ .metadata.name as $pod |
+ .status.phase as $phase |
+ .status.containerStatuses[]? |
+ "pod=\($pod) phase=\($phase // "unknown") ready=\(.ready) restarts=\(.restartCount) waiting=\(.state.waiting.reason // "none") terminated=\(.state.terminated.reason // "none") exitCode=\(.state.terminated.exitCode // "none") lastTerminated=\(.lastState.terminated.reason // "none") lastExitCode=\(.lastState.terminated.exitCode // "none")"
+ ' >&2 || true
+
+ while IFS= read -r pod; do
+ [ -n "$pod" ] || continue
+ echo "HCC proxy rollout previous logs for pod/${pod}:" >&2
+ kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \
+ --previous --tail=120 >&2 || true
+ echo "HCC proxy rollout current logs for pod/${pod}:" >&2
+ kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \
+ --tail=120 >&2 || true
+ done < <(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
+ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
+}
+
cleanup() {
local status=$? cleanup_failed=0 restore_ok=1 host remaining
set +e
@@ -126,6 +157,8 @@ cleanup() {
else
cleanup_failed=1
print_hcc_repair_instructions
+ echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}" >&2
+ echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s" >&2
fi
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
rm -f "$HCC_LOG_BUFFER" "$NON_FIXTURE_BASELINE" "$NON_FIXTURE_AFTER"
@@ -143,19 +176,35 @@ header "HCC CommunicationChannel watch recovery"
require_branch_owned_hcc_gate
acquire_hcc_watch_gate_lock || die "another HCC watch-recovery gate owns this profile"
kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" >/dev/null
+ORIGINAL_REPLICAS="$(
+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}'
+)"
+[ "$ORIGINAL_REPLICAS" = 1 ] ||
+ die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
kctl get host "$SOURCE_HOST" -n "$HOST_NS" >/dev/null
host_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"
port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"
-[ -z "$host_override$port_override" ] || die "HCC already has an explicit Kubernetes service override"
+api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"
+[ -z "$host_override$port_override$api_cidrs_override" ] ||
+ die "HCC already has an explicit Kubernetes service or API CIDR override"
[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||
die "HCC already has hostAliases; refusing a non-restorable fault injection"
HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"
+K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
+ -c host-context-controller -- printenv KUBERNETES_SERVICE_HOST)" ||
+ die "could not read the Kubernetes-injected API service IP"
+kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \
+ node -e 'const { isIP } = require("node:net"); process.exit(isIP(process.argv[1]) === 4 ? 0 : 1)' \
+ "$K8S_API_SERVICE_HOST" >/dev/null ||
+ die "Kubernetes-injected API service host is not an IPv4 address"
+K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"
log "Creating an isolated TCP proxy for the Kubernetes API"
create_hcc_api_proxy
@@ -164,19 +213,27 @@ verify_hcc_proxy_network_policy
PROXY_IP="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"
[ -n "$PROXY_IP" ] || die "proxy Service has no ClusterIP"
HCC_PATCHED=1
-hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" '{spec:{template:{spec:{
+hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" --arg api_cidr "$K8S_API_CIDR" '{spec:{template:{spec:{
hostAliases:[{ip:$ip,hostnames:["kubernetes.default.svc"]}],
containers:[{name:"host-context-controller",env:[
{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},
- {name:"KUBERNETES_SERVICE_PORT",value:"443"}
+ {name:"KUBERNETES_SERVICE_PORT",value:"443"},
+ {name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr}
]}]
}}}}')"
kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$hcc_proxy_patch" >/dev/null
-kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||
+if ! kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null; then
+ print_hcc_proxy_rollout_diagnostics
die "HCC did not become ready through the API proxy"
+fi
initial_hcc_identity="$(wait_for_hcc_identity 30)" || die "could not capture HCC pod identity"
read -r HCC_UID HCC_RESTARTS <<<"$initial_hcc_identity"
[ -n "$HCC_UID" ] && [ "$HCC_UID" != invalid ] || die "could not capture HCC pod identity"
+proxy_policy_cidrs="$(kctl get networkpolicy "allow-k8s-api-egress-${MCP_NS}" -n "$MCP_NS" -o json |
+ jq -r '[.spec.egress[0].to[]?.ipBlock.cidr] | join(",")')" ||
+ die "could not read the proxy-run API egress policy"
+[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ] ||
+ die "proxy-run API egress policy did not retain the injected Kubernetes API CIDR"
ok "HCC is healthy through the proxy (pod=${HCC_UID}, restarts=${HCC_RESTARTS})"
log "Creating one stateful control and two stateless Host fixtures"
@@ -214,14 +271,13 @@ ok "isolated Host fleet reached its baseline"
run_recovery_cycle() {
local cycle=$1 cycle_logs recovery_logs outage_started outage_seconds max_retry_failures list_pattern
- local host_list_pattern ended_before started_before recovered_before completed_before failed_before
+ local host_list_pattern ended_before started_before recovered_before failed_before
local list_before host_list_before expected_host_count
local failed_after retry_failures
cycle_logs="$(hcc_recovery_logs)"
ended_before="$(log_count_from "$cycle_logs" 'CommunicationChannel watch ended;')"
started_before="$(log_count_from "$cycle_logs" 'Starting CommunicationChannel watch')"
recovered_before="$(log_count_from "$cycle_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"
- completed_before="$(log_count_from "$cycle_logs" 'Completed Host reconciliation after CommunicationChannel recovery$')"
failed_before="$(log_count_from "$cycle_logs" 'cache recovery failed;')"
list_pattern="Listing all CommunicationChannels in namespace ${CHANNEL_NS}"
host_list_pattern="Listing all Hosts in namespace ${HOST_NS}"
@@ -265,10 +321,7 @@ run_recovery_cycle() {
max_retry_failures=$((
(outage_seconds + CC_RECOVERY_RETRY_SECONDS - 1) / CC_RECOVERY_RETRY_SECONDS + 2
))
- wait_log_count 'Completed Host reconciliation after CommunicationChannel recovery$' \
- "$((completed_before + 1))" 180 ||
- die "Host fleet did not converge after snapshot recovery in cycle ${cycle}"
- recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_host_count" ||
+ wait_recovery_cycle_fresh_host_inventory "$cycle" "$expected_host_count" 180 ||
die "cycle ${cycle} did not reconcile the fresh ${expected_host_count}-Host inventory"
recovery_logs="$(hcc_recovery_logs)"
failed_after="$(log_count_from "$recovery_logs" 'cache recovery failed;')"
@@ -316,14 +369,18 @@ ended="$(log_count_from "$final_logs" 'CommunicationChannel watch ended;')"
started="$(log_count_from "$final_logs" 'Starting CommunicationChannel watch')"
recovered="$(log_count_from "$final_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"
failed="$(log_count_from "$final_logs" 'cache recovery failed;')"
-fleet_passes="$(log_count_from "$final_logs" 'Completed Host reconciliation after CommunicationChannel')"
+fleet_passes="$(
+ log_count_from "$final_logs" \
+ 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'
+)"
[ "$ended" = 2 ] && [ "$started" = 2 ] && [ "$recovered" = 2 ] && \
[ "$failed" = "$TOTAL_RETRY_FAILURES" ] && [ "$failed" -le "$TOTAL_RETRY_FAILURE_LIMIT" ] ||
die "unexpected watch lifecycle counts: ended=${ended}, started=${started}, recovered=${recovered}, failed=${failed}"
[ "$fleet_passes" -ge 2 ] && [ "$fleet_passes" -le 4 ] ||
die "unexpected successful fleet reconcile count ${fleet_passes}; expected 2..4 passes"
-# Each recovered snapshot requires one successful pass; at most one already
-# queued fail-closed pass may also finish per cycle. The deterministic
+# Each recovered snapshot requires one causally ordered successful pass; a
+# Host-watch recovery pass may cover the same fresh CC+Host inventories. At
+# most one already queued covering pass may also finish per cycle. The deterministic
# max-concurrency=1 proof stays in k8sClient unit tests where overlapping passes
# can be barrier-controlled.
diff --git a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
new file mode 100755
index 000000000..cccb17538
--- /dev/null
+++ b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
@@ -0,0 +1,1790 @@
+#!/usr/bin/env bash
+# Proves that HCC readiness is independent from the exact McpServer ->
+# external-egress -> runtime -> Context NetworkPolicy initial convergence path.
+#
+# The gate runs only on an exact-HEAD, branch-owned Minikube profile. It routes
+# HCC DNS through an isolated in-cluster proxy that forwards every query except
+# one fixture hostname. The gate first lets HCC create a real external-egress
+# policy. A same-intent restart then proves the safety pass RETAINS that
+# identity-stable DNS-derived allow without attempting DNS and certifies global
+# readiness with the allow still present — no deny window — while the affected
+# runtime stays blocked. DNS release lets the additive lane refresh the current
+# TCP policy and runtime. A second phase changes that binding's protocol while
+# HCC is stopped and proves the divergent contract: an identity-changing
+# TCP-to-UDP replacement IS revoked before readiness. A real Context watch update
+# still converges its three
+# NetworkPolicies. The fleet is wider than the bounded worker width, covering
+# the original large-fleet symptom.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/e2e-lib.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
+
+[ -n "$E2E_KUBECONTEXT" ] || {
+ echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2
+ exit 1
+}
+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
+ echo "Refusing HCC MCP readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
+ exit 1
+}
+require_safe_kube_context
+[ "${E2E_HCC_MCP_READINESS_FAULT_INJECTION:-0}" = 1 ] || {
+ echo "Set E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC DNS fault injection." >&2
+ exit 1
+}
+command -v jq >/dev/null 2>&1 || {
+ echo "jq is required" >&2
+ exit 1
+}
+
+HCC_NS="${HCC_NS:-control-plane}"
+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
+MCP_NS="${MCP_SERVER_NS:-mcp-server}"
+HOST_NS="${MCP_HOST_NS:-mcp-host}"
+RPC_PROXY_NS="${RPC_PROXY_NS:-rpc-proxy}"
+RUN_ID="$(date +%s)-$$"
+SUITE_NAME="hcc-mcp-readiness"
+RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"
+MCP_NAME="$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}")"
+CONTEXT_NAME="$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}")"
+CONTEXT_ID="$CONTEXT_NAME"
+MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"
+TARGET_DNS="$(truncate_rfc1123 "hcc-readiness-${RUN_ID}").example.com"
+DNS_BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-dns-${RUN_ID}")"
+DNS_BLOCKER_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-policy")"
+HCC_DNS_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-from-hcc")"
+CONTEXT_POLICY="ctx-${CONTEXT_ID}-${MCP_NAME}"
+CONTEXT_EGRESS_POLICY="${CONTEXT_POLICY}-egress"
+RPC_EGRESS_POLICY="rpc-egress-${CONTEXT_ID}-${MCP_NAME}"
+MANAGED_BY_LABEL="clerum.io/managed-by"
+MANAGED_BY_VALUE="host-context-controller"
+MCP_SERVER_LABEL="clerum.io/mcpserver"
+POLICY_TYPE_LABEL="clerum.io/policy-type"
+CONTEXT_LABEL="clerum.io/context"
+EXTERNAL_EGRESS_CIDR="93.184.216.34/32"
+EXTERNAL_EGRESS_PORT=443
+INITIAL_EXTERNAL_EGRESS_PROTOCOL=TCP
+EXTERNAL_EGRESS_PROTOCOL=UDP
+ORIGINAL_REPLICAS=""
+ORIGINAL_DNS_POLICY=""
+ORIGINAL_DNS_CONFIG=""
+DNS_BLOCKER_IP=""
+DNS_RELEASE_COUNT_AT_HOLD=0
+DNS_HOLD_COUNT_AT_ARM=0
+DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=0
+UPSTREAM_DNS_IP=""
+NEW_HCC_POD=""
+HCC_UID=""
+HCC_RESTARTS=""
+HCC_PORT=""
+HCC_MUTATED=0
+DNS_BLOCKER_CREATED=0
+MCP_CREATED=0
+CONTEXT_CREATED=0
+PEER_FLEET_CREATED=0
+MCP_FLEET_NAMES=("$MCP_NAME")
+CONTEXT_FLEET_NAMES=("$CONTEXT_NAME")
+# Read and mutated by the sourced compare-and-swap lock helper.
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_ACQUIRED=0
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_NAME=""
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_UID=""
+# shellcheck disable=SC2034
+HCC_GATE_FINALIZATION_FAILURE=""
+HCC_READY_PROBE_DIAGNOSTIC=""
+BASELINE_POLICY_SNAPSHOT=""
+SAME_INTENT_EGRESS_SNAPSHOT=""
+
+# Clean-window negative markers. Consumed ONLY while the DNS fault injection is
+# NOT applied (pre-injection pre-checks, post-restore cleanup): under the
+# injection the additive NetworkPolicy lane legitimately fails and schedules
+# retries, so these negatives false-fail inside the injected window (removed
+# there by 1abcfb1/15997353/e7064a7f). Inventoried in
+# scripts/tests/test-hcc-watch-recovery-gate.sh marker_bindings.
+NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'
+NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'
+NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'
+NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '
+clean_window_checks_run=0
+
+[[ "$MCP_FLEET_SIZE" =~ ^[0-9]+$ ]] && [ "$MCP_FLEET_SIZE" -gt 10 ] &&
+ [ "$MCP_FLEET_SIZE" -le 24 ] || {
+ echo "E2E_HCC_MCP_FLEET_SIZE must be an integer from 11 through 24; got ${MCP_FLEET_SIZE}." >&2
+ exit 1
+}
+for ((index = 2; index <= MCP_FLEET_SIZE; index++)); do
+ MCP_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}-${index}")")
+ CONTEXT_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}-${index}")")
+done
+
+die() {
+ if [ -n "$NEW_HCC_POD" ]; then
+ echo "Recent HCC logs from ${NEW_HCC_POD}:" >&2
+ kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller \
+ --tail=300 >&2 || true
+ fi
+ if [ "$DNS_BLOCKER_CREATED" = 1 ]; then
+ echo "Recent DNS blocker logs:" >&2
+ kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" --tail=120 >&2 || true
+ fi
+ fail "$*"
+ exit 1
+}
+
+wait_until() {
+ local timeout=$1 description=$2
+ shift 2
+ local deadline now
+ deadline=$(( $(date +%s) + timeout ))
+ while :; do
+ "$@" && return 0
+ now="$(date +%s)"
+ [ "$now" -lt "$deadline" ] || break
+ sleep 1
+ done
+ echo "Timed out after ${timeout}s waiting for ${description}" >&2
+ return 1
+}
+
+wait_until_fast() {
+ local timeout=$1 description=$2
+ shift 2
+ local deadline now rc
+ deadline=$(( $(date +%s) + timeout ))
+ while :; do
+ "$@" && return 0
+ rc=$?
+ # rc=2 is a fatal, non-retryable ordering violation (e.g. 200 observed with
+ # a stale policy still present): stop polling immediately so a later clean
+ # poll cannot mask it. rc=1 is benign "not ready yet".
+ if [ "$rc" -eq 2 ]; then
+ echo "FATAL ordering violation while waiting for ${description}: ${HCC_READY_PROBE_DIAGNOSTIC:-unset}" >&2
+ return 2
+ fi
+ now="$(date +%s)"
+ [ "$now" -lt "$deadline" ] || break
+ sleep 0.5
+ done
+ echo "Timed out after ${timeout}s waiting for ${description}" >&2
+ return 1
+}
+
+running_hcc_pod() {
+ local rows
+ rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
+ --field-selector=status.phase=Running \
+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
+ 2>/dev/null)" || return 1
+ awk -F '\t' '$1 != "" && $4 == "" { print $1; exit }' <<<"$rows"
+}
+
+capture_hcc() {
+ NEW_HCC_POD="$(running_hcc_pod)"
+ [ -n "$NEW_HCC_POD" ]
+}
+
+hcc_pods_absent() {
+ local pods
+ pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||
+ return 1
+ [ -z "$pods" ]
+}
+
+hcc_log_contains() {
+ local marker=$1 logs
+ [ -n "$NEW_HCC_POD" ] || return 1
+ logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \
+ -c host-context-controller 2>/dev/null)" || return 1
+ grep -Fq "$marker" <<<"$logs"
+}
+
+dns_log_contains() {
+ local marker=$1 logs
+ logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
+ return 1
+ grep -Fq "$marker" <<<"$logs"
+}
+
+dns_release_count() {
+ local marker="released DNS A ${TARGET_DNS}" logs
+ logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
+ return 1
+ grep -Fc "$marker" <<<"$logs" || true
+}
+
+dns_hold_count() {
+ local marker="holding DNS A ${TARGET_DNS}" logs
+ logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
+ return 1
+ grep -Fc "$marker" <<<"$logs" || true
+}
+
+dns_hold_observed_since_arm() {
+ [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_ARM" ]
+}
+
+dns_has_not_released_since_hold() {
+ [ "$(dns_release_count)" = "$DNS_RELEASE_COUNT_AT_HOLD" ]
+}
+
+dns_released_since_hold() {
+ [ "$(dns_release_count)" -gt "$DNS_RELEASE_COUNT_AT_HOLD" ]
+}
+
+external_egress_retry_progress_is_observed() {
+ local logs marker
+ [ -n "$NEW_HCC_POD" ] || return 1
+ logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \
+ -c host-context-controller 2>/dev/null)" || return 1
+ marker="Scheduling external egress retry "
+ grep -Fq \
+ "Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};" <<<"$logs" &&
+ awk \
+ -v marker="$marker" \
+ -v server="for McpServer \"${MCP_NAME}\"" '
+ index($0, marker) && index($0, server) { found = 1 }
+ END { exit found ? 0 : 1 }
+ ' <<<"$logs"
+}
+
+dns_retry_query_observed_since_schedule() {
+ [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_RETRY_SCHEDULE" ]
+}
+
+hcc_identity_is_stable() {
+ local row uid restarts
+ row="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \
+ -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}' \
+ 2>/dev/null)" || return 1
+ read -r uid restarts <<<"$row"
+ [ "$uid" = "$HCC_UID" ] && [ "$restarts" = "$HCC_RESTARTS" ]
+}
+
+hcc_kubernetes_readiness_is_exact() {
+ [ -n "$NEW_HCC_POD" ] && [ -n "$HCC_UID" ] || return 1
+ kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" -o json 2>/dev/null |
+ jq -e --arg uid "$HCC_UID" '
+ .metadata.uid == $uid and
+ any(.status.conditions[]?; .type == "Ready" and .status == "True") and
+ any(.status.containerStatuses[]?;
+ .name == "host-context-controller" and .ready == true
+ )
+ ' >/dev/null || return 1
+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json 2>/dev/null |
+ jq -e '
+ .status.observedGeneration == .metadata.generation and
+ (.spec.replicas // 0) == 1 and
+ (.status.updatedReplicas // 0) == 1 and
+ (.status.readyReplicas // 0) == 1 and
+ (.status.availableReplicas // 0) == 1 and
+ (.status.unavailableReplicas // 0) == 0
+ ' >/dev/null
+}
+
+resource_absent() {
+ local kind=$1 name=$2 namespace=$3 found
+ found="$(
+ kctl get "$kind" "$name" -n "$namespace" --ignore-not-found -o name 2>/dev/null
+ )" || return 1
+ [ -z "$found" ]
+}
+
+fixture_mcp_runtime_absent() {
+ local external_egress_policies
+ fixture_runtime_absent || return 1
+ external_egress_policies="$(
+ kctl get networkpolicy -n "$MCP_NS" \
+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
+ -o name 2>/dev/null
+ )" || return 1
+ [ -z "$external_egress_policies" ]
+}
+
+fixture_runtime_absent() {
+ resource_absent deployment "$MCP_NAME" "$MCP_NS" &&
+ resource_absent service "$MCP_NAME" "$MCP_NS"
+}
+
+mcpserver_current_ready() {
+ local name=$1
+ kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null |
+ jq -e '
+ .metadata.generation as $generation |
+ any(
+ .status.conditions[]?;
+ .type == "Ready" and .status == "True" and
+ (.observedGeneration // -1) == $generation
+ )
+ ' >/dev/null
+}
+
+mcpserver_current_ready_absent() {
+ local name=$1 resource
+ resource="$(kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null)" ||
+ return 1
+ jq -e '
+ .metadata.generation as $generation |
+ any(
+ .status.conditions[]?;
+ .type == "Ready" and .status == "True" and
+ (.observedGeneration // -1) == $generation
+ ) | not
+ ' <<<"$resource" >/dev/null
+}
+
+fixture_runtime_and_ready_absent() {
+ fixture_runtime_absent && mcpserver_current_ready_absent "$MCP_NAME"
+}
+
+context_policies_absent() {
+ resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" &&
+ resource_absent networkpolicy "$CONTEXT_EGRESS_POLICY" "$HOST_NS" &&
+ resource_absent networkpolicy "$RPC_EGRESS_POLICY" "$RPC_PROXY_NS"
+}
+
+context_policies_converged() {
+ local context_id=${1:-$CONTEXT_ID}
+ local context_name=${2:-$CONTEXT_NAME}
+ local server=${3:-$MCP_NAME}
+ local ingress_policy="ctx-${context_id}-${server}"
+ local host_egress_policy="${ingress_policy}-egress"
+ local rpc_egress_policy="rpc-egress-${context_id}-${server}"
+ kctl get networkpolicy "$ingress_policy" -n "$MCP_NS" -o json |
+ jq -e \
+ --arg contextId "$context_id" \
+ --arg contextName "$context_name" \
+ --arg server "$server" \
+ --arg hostNs "$HOST_NS" \
+ --arg rpcNs "$RPC_PROXY_NS" \
+ --arg mcpNs "$MCP_NS" \
+ --arg managedByLabel "$MANAGED_BY_LABEL" \
+ --arg managedByValue "$MANAGED_BY_VALUE" \
+ --arg contextLabel "$CONTEXT_LABEL" \
+ --arg serverLabel "$MCP_SERVER_LABEL" \
+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" '
+ .metadata.labels[$managedByLabel] == $managedByValue and
+ .metadata.labels[$policyTypeLabel] == "context-allow" and
+ .metadata.labels[$contextLabel] == $contextId and
+ .metadata.labels[$serverLabel] == $server and
+ .spec.podSelector.matchLabels[$serverLabel] == $server and
+ .spec.policyTypes == ["Ingress"] and
+ (.spec.ingress | length) == 1 and
+ .spec.ingress[0].ports == [{port: 3000, protocol: "TCP"}] and
+ (.spec.ingress[0].from | length) == 3 and
+ any(.spec.ingress[0].from[];
+ .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $hostNs and
+ .podSelector.matchLabels[$managedByLabel] == $managedByValue and
+ .podSelector.matchLabels[$contextLabel] == $contextName
+ ) and
+ any(.spec.ingress[0].from[];
+ .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $rpcNs and
+ .podSelector.matchLabels.app == "rpc-proxy"
+ ) and
+ any(.spec.ingress[0].from[];
+ .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $mcpNs and
+ .podSelector.matchLabels.app == "mcp-proxy"
+ )
+ ' >/dev/null || return 1
+ kctl get networkpolicy "$host_egress_policy" -n "$HOST_NS" -o json |
+ jq -e \
+ --arg contextId "$context_id" \
+ --arg contextName "$context_name" \
+ --arg server "$server" \
+ --arg mcpNs "$MCP_NS" \
+ --arg managedByLabel "$MANAGED_BY_LABEL" \
+ --arg managedByValue "$MANAGED_BY_VALUE" \
+ --arg contextLabel "$CONTEXT_LABEL" \
+ --arg serverLabel "$MCP_SERVER_LABEL" \
+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" '
+ .metadata.labels[$managedByLabel] == $managedByValue and
+ .metadata.labels[$policyTypeLabel] == "context-allow" and
+ .metadata.labels[$contextLabel] == $contextId and
+ .metadata.labels[$serverLabel] == $server and
+ .spec.podSelector.matchLabels[$managedByLabel] == $managedByValue and
+ .spec.podSelector.matchLabels[$contextLabel] == $contextName and
+ .spec.policyTypes == ["Egress"] and
+ (.spec.egress | length) == 1 and
+ .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and
+ (.spec.egress[0].to | length) == 1 and
+ .spec.egress[0].to[0].namespaceSelector.matchLabels[
+ "kubernetes.io/metadata.name"
+ ] == $mcpNs and
+ .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server
+ ' >/dev/null || return 1
+ kctl get networkpolicy "$rpc_egress_policy" -n "$RPC_PROXY_NS" -o json |
+ jq -e \
+ --arg contextId "$context_id" \
+ --arg server "$server" \
+ --arg mcpNs "$MCP_NS" \
+ --arg managedByLabel "$MANAGED_BY_LABEL" \
+ --arg managedByValue "$MANAGED_BY_VALUE" \
+ --arg contextLabel "$CONTEXT_LABEL" \
+ --arg serverLabel "$MCP_SERVER_LABEL" \
+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" '
+ .metadata.labels[$managedByLabel] == $managedByValue and
+ .metadata.labels[$policyTypeLabel] == "rpc-proxy-egress" and
+ .metadata.labels[$contextLabel] == $contextId and
+ .metadata.labels[$serverLabel] == $server and
+ .spec.podSelector.matchLabels.app == "rpc-proxy" and
+ .spec.policyTypes == ["Egress"] and
+ (.spec.egress | length) == 1 and
+ .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and
+ (.spec.egress[0].to | length) == 1 and
+ .spec.egress[0].to[0].namespaceSelector.matchLabels[
+ "kubernetes.io/metadata.name"
+ ] == $mcpNs and
+ .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server
+ ' >/dev/null
+}
+
+external_egress_policy_converged_with_protocol() {
+ local protocol=$1
+ kctl get networkpolicy -n "$MCP_NS" \
+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
+ -o json |
+ jq -e \
+ --arg server "$MCP_NAME" \
+ --arg cidr "$EXTERNAL_EGRESS_CIDR" \
+ --arg managedByLabel "$MANAGED_BY_LABEL" \
+ --arg managedByValue "$MANAGED_BY_VALUE" \
+ --arg serverLabel "$MCP_SERVER_LABEL" \
+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" \
+ --arg protocol "$protocol" \
+ --argjson port "$EXTERNAL_EGRESS_PORT" '
+ (.items | length) == 1 and
+ .items[0].metadata.labels[$managedByLabel] == $managedByValue and
+ .items[0].metadata.labels[$policyTypeLabel] == "external-egress" and
+ .items[0].metadata.labels[$serverLabel] == $server and
+ .items[0].spec.podSelector.matchLabels[$serverLabel] == $server and
+ .items[0].spec.policyTypes == ["Egress"] and
+ (.items[0].spec.egress | length) == 1 and
+ .items[0].spec.egress[0].ports == [{port: $port, protocol: $protocol}] and
+ (.items[0].spec.egress[0].to | length) == 1 and
+ .items[0].spec.egress[0].to[0].ipBlock == {cidr: $cidr}
+ ' >/dev/null
+}
+
+external_egress_policy_converged() {
+ external_egress_policy_converged_with_protocol "$EXTERNAL_EGRESS_PROTOCOL"
+}
+
+external_egress_policy_absent() {
+ local policies
+ policies="$(
+ kctl get networkpolicy -n "$MCP_NS" \
+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
+ -o name 2>/dev/null
+ )" || return 1
+ [ -z "$policies" ]
+}
+
+clean_window_netpol_log_negatives_hold() {
+ # Family 1, fail-closed snapshot: read ONE authoritative log snapshot so a
+ # kubectl logs failure cannot make the negative assertions pass vacuously.
+ local pod=$1 logs
+ [ -n "$pod" ] || return 1
+ logs="$(kctl logs "pod/${pod}" -n "$HCC_NS" \
+ -c host-context-controller 2>/dev/null)" || return 1
+ ! grep -Fq "$NETPOL_INITIAL_FAILED_MARKER" <<<"$logs" &&
+ ! grep -Fq "$NETPOL_ADDITIVE_FAILED_MARKER" <<<"$logs" &&
+ ! grep -Fq "$NETPOL_RETRY_SCHEDULED_MARKER" <<<"$logs" &&
+ ! grep -Fq "${NETPOL_CONTEXT_FAILED_MARKER}${CONTEXT_NAME}:" <<<"$logs"
+}
+
+clean_window_runtime_and_ready_absent() {
+ # Family 3, clean-window form. fixture_runtime_and_ready_absent requires the
+ # McpServer object to EXIST (mcpserver_current_ready_absent fails on a missing
+ # object); in clean windows the fixture may not exist at all, which is the
+ # strongest form of absence. An API error still fails closed.
+ fixture_runtime_absent || return 1
+ if ! resource_absent mcpserver "$MCP_NAME" "$MCP_NS"; then
+ mcpserver_current_ready_absent "$MCP_NAME" || return 1
+ fi
+ external_egress_policy_absent
+}
+
+fixture_runtime_converged_with_protocol() {
+ local protocol=$1 desired available
+ desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
+ -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1
+ available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
+ -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1
+ [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1
+ kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1
+ mcpserver_current_ready "$MCP_NAME" || return 1
+ external_egress_policy_converged_with_protocol "$protocol"
+}
+
+fixture_resources_absent() {
+ fixture_mcp_runtime_absent &&
+ context_policies_absent &&
+ peer_fleet_runtime_absent &&
+ peer_fleet_policies_absent
+}
+
+fixture_inputs_absent() {
+ resource_absent context "$CONTEXT_NAME" "$MCP_NS" &&
+ resource_absent mcpserver "$MCP_NAME" "$MCP_NS" && peer_fleet_inputs_absent
+}
+
+peer_fleet_inputs_absent() {
+ local index
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ resource_absent context "${CONTEXT_FLEET_NAMES[index]}" "$MCP_NS" || return 1
+ resource_absent mcpserver "${MCP_FLEET_NAMES[index]}" "$MCP_NS" || return 1
+ done
+}
+
+peer_fleet_runtime_absent() {
+ local index server
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ server="${MCP_FLEET_NAMES[index]}"
+ resource_absent deployment "$server" "$MCP_NS" || return 1
+ resource_absent service "$server" "$MCP_NS" || return 1
+ done
+}
+
+peer_fleet_policies_absent() {
+ local index server context context_policy external_policies
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ server="${MCP_FLEET_NAMES[index]}"
+ context="${CONTEXT_FLEET_NAMES[index]}"
+ context_policy="ctx-${context}-${server}"
+ resource_absent networkpolicy "$context_policy" "$MCP_NS" || return 1
+ resource_absent networkpolicy "${context_policy}-egress" "$HOST_NS" || return 1
+ resource_absent networkpolicy "rpc-egress-${context}-${server}" "$RPC_PROXY_NS" || return 1
+ external_policies="$(
+ kctl get networkpolicy -n "$MCP_NS" \
+ -l "${MCP_SERVER_LABEL}=${server},${POLICY_TYPE_LABEL}=external-egress" \
+ -o name 2>/dev/null
+ )" || return 1
+ [ -z "$external_policies" ] || return 1
+ done
+}
+
+peer_fleet_converged() {
+ local index server context desired available
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ server="${MCP_FLEET_NAMES[index]}"
+ context="${CONTEXT_FLEET_NAMES[index]}"
+ desired="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1
+ available="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1
+ [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1
+ kctl get service "$server" -n "$MCP_NS" >/dev/null 2>&1 || return 1
+ mcpserver_current_ready "$server" || return 1
+ context_policies_converged "$context" "$context" "$server" || return 1
+ done
+}
+
+baseline_policy_snapshot() {
+ kctl get \
+ "networkpolicy/deny-all-${MCP_NS}" \
+ "networkpolicy/allow-dns-egress-${MCP_NS}" \
+ networkpolicy/allow-host-context-controller-api \
+ -n "$MCP_NS" -o json 2>/dev/null |
+ jq -ceS \
+ --arg managedBy "$MANAGED_BY_VALUE" \
+ --arg deny "deny-all-${MCP_NS}" \
+ --arg dns "allow-dns-egress-${MCP_NS}" '
+ [.items[] | {
+ name: .metadata.name,
+ managedBy: .metadata.labels["clerum.io/managed-by"],
+ policyType: .metadata.labels["clerum.io/policy-type"],
+ spec
+ }]
+ | sort_by(.name)
+ | select(length == 3)
+ | select(all(.[];
+ .managedBy == $managedBy and
+ ((.name == $deny and .policyType == "default-deny") or
+ (.name == $dns and .policyType == "infrastructure") or
+ (.name == "allow-host-context-controller-api" and .policyType == "allow-api"))))
+ '
+}
+
+baseline_policies_unchanged() {
+ local current
+ [ -n "$BASELINE_POLICY_SNAPSHOT" ] || return 1
+ current="$(baseline_policy_snapshot)" || return 1
+ [ "$current" = "$BASELINE_POLICY_SNAPSHOT" ]
+}
+
+context_watch_has_latest_spec() {
+ kctl get context "$CONTEXT_NAME" -n "$MCP_NS" -o json |
+ jq -e --arg server "$MCP_NAME" '.spec.mcpServers == [$server]' >/dev/null
+}
+
+mcp_egress_binding_is_udp() {
+ kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json |
+ jq -e --arg dns "$TARGET_DNS" '
+ (.spec.egressBindings | length) == 1 and
+ .spec.egressBindings[0].egressClass == "exact-host" and
+ .spec.egressBindings[0].dns == $dns and
+ .spec.egressBindings[0].port == 443 and
+ .spec.egressBindings[0].protocol == "UDP"
+ ' >/dev/null
+}
+
+create_peer_fleet() {
+ local index server context
+ # Arm cleanup before the first write so a partial apply cannot strand input
+ # CRDs or leave HCC deliberately scaled down during failure recovery.
+ PEER_FLEET_CREATED=1
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ server="${MCP_FLEET_NAMES[index]}"
+ context="${CONTEXT_FLEET_NAMES[index]}"
+ kctl apply -f - >/dev/null <<EOF
+apiVersion: clerum.io/v1alpha1
+kind: Context
+metadata:
+ name: ${context}
+ namespace: ${MCP_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ contextId: ${context}
+ description: Gate-owned peer Context for HCC readiness fleet coverage.
+ mcpServers: [${server}]
+---
+apiVersion: clerum.io/v1alpha1
+kind: McpServer
+metadata:
+ name: ${server}
+ namespace: ${MCP_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ contextRef: ${context}
+ description: Gate-owned peer McpServer for bounded HCC fleet coverage.
+ image: clerum/mock-mcp-server:test
+ imagePullPolicy: IfNotPresent
+ transport:
+ type: streamableHttp
+ url: http://${server}.${MCP_NS}.svc.cluster.local:3000/mcp
+ port: 3000
+EOF
+ done
+}
+
+fixture_converged() {
+ local desired available
+ desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
+ -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1
+ available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
+ -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1
+ [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1
+ kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1
+ mcpserver_current_ready "$MCP_NAME" || return 1
+ external_egress_policy_converged || return 1
+ context_policies_converged
+}
+
+probe_hcc_final_context() {
+ local probe_script
+ probe_script="$(cat <<'NODE'
+const http = require('http');
+const context = process.argv[1];
+const expected = process.argv[2];
+function get(path) {
+ return new Promise((resolve, reject) => {
+ const request = http.get({
+ host: '127.0.0.1',
+ port: Number(process.env.HCC_E2E_PORT),
+ path
+ }, response => {
+ let body = '';
+ response.setEncoding('utf8');
+ response.on('data', chunk => { body += chunk; });
+ response.on('end', () => resolve({status: response.statusCode, body}));
+ });
+ request.setTimeout(5000, () => request.destroy(new Error('timeout')));
+ request.once('error', reject);
+ });
+}
+(async () => {
+ const ready = await get('/ready');
+ const scoped = await get('/api/v1/mcpservers/context/' + encodeURIComponent(context));
+ const body = JSON.parse(scoped.body);
+ const fixture = body.servers?.find(server => server.name === expected);
+ if (ready.status !== 200 || JSON.parse(ready.body).ready !== true) {
+ throw new Error('readiness regressed: ' + ready.status + ' ' + ready.body);
+ }
+ if (scoped.status !== 200 || !fixture || fixture.status?.ready !== true) {
+ throw new Error('latest Context did not converge in discovery: ' + scoped.status + ' ' + scoped.body);
+ }
+ console.log(JSON.stringify({
+ readyStatus: ready.status,
+ scopedDiscoveryStatus: scoped.status,
+ contextRef: body.contextRef,
+ fixture: fixture.name,
+ fixtureReady: fixture.status.ready
+ }));
+})().catch(error => { console.error(error.message); process.exit(1); });
+NODE
+)"
+ kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \
+ env "HCC_E2E_PORT=${HCC_PORT}" \
+ node -e "$probe_script" "$CONTEXT_NAME" "$MCP_NAME"
+}
+
+probe_hcc_ready_pod() {
+ local pod
+ pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1
+ kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \
+ env "HCC_E2E_PORT=${HCC_PORT}" \
+ node -e '
+ const http = require("node:http");
+ const request = http.get(
+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
+ response => {
+ let body = "";
+ response.setEncoding("utf8");
+ response.on("data", chunk => { body += chunk; });
+ response.on("end", () => {
+ let parsed;
+ try { parsed = JSON.parse(body); } catch { process.exit(2); }
+ process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);
+ });
+ }
+ );
+ request.setTimeout(5000, () => request.destroy(new Error("timeout")));
+ request.once("error", () => process.exit(4));
+ ' >/dev/null 2>&1
+}
+
+probe_new_hcc_ready_endpoint() {
+ local output first_line
+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=pod-not-selected"
+ [ -n "$NEW_HCC_POD" ] || return 1
+ if output="$(
+ kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \
+ env "HCC_E2E_PORT=${HCC_PORT}" \
+ node -e '
+ const http = require("node:http");
+ let finished = false;
+ const finish = (code, category, detail = "") => {
+ if (finished) return;
+ finished = true;
+ if (category) {
+ console.error(`HCC_READY_PROBE category=${category}${detail}`);
+ }
+ process.exitCode = code;
+ };
+ const request = http.get(
+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
+ response => {
+ let body = "";
+ response.setEncoding("utf8");
+ response.on("data", chunk => { body += chunk; });
+ response.on("end", () => {
+ if (response.statusCode !== 200) {
+ finish(3, "http-status", ` status=${response.statusCode ?? "unknown"}`);
+ return;
+ }
+ let parsed;
+ try {
+ parsed = JSON.parse(body);
+ } catch {
+ finish(2, "json-invalid");
+ return;
+ }
+ if (parsed?.ready !== true) {
+ finish(3, "json-ready-false");
+ return;
+ }
+ finish(0);
+ });
+ }
+ );
+ request.setTimeout(2000, () => {
+ finish(4, "timeout");
+ request.destroy();
+ });
+ request.once("error", () => finish(5, "transport"));
+ ' 2>&1
+ )"; then
+ HCC_READY_PROBE_DIAGNOSTIC=""
+ return 0
+ fi
+ first_line="${output%%$'\n'*}"
+ case "$first_line" in
+ "HCC_READY_PROBE category="*)
+ printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s' "$first_line"
+ ;;
+ *)
+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=kubectl-exec"
+ ;;
+ esac
+ return 1
+}
+
+# Third, independent signal of the safety contract (M6): the authoritative pass
+# must actually have MEASURED and REVOKED. The unit tests assert the readiness
+# predicate over vi.fn() mocks, so a metric rename would leave them green; these
+# family-name literals live in the gate, so a rename turns the REAL gate red.
+# Pure adjudication over a /metrics scrape — unit-testable without a cluster.
+safety_pass_metrics_are_certified() {
+ local metrics=$1
+ grep -Fq -- 'clerum_hcc_networkpolicy_safety_pass_duration_seconds' <<<"$metrics" || return 1
+ # At least one policy revoked by an authoritative pass ($NF is the sample
+ # value; tolerant of extra/reordered labels).
+ awk '$0 ~ /^clerum_hcc_networkpolicy_safety_pass_policies_total\{/ && /operation="revoked"/ { if ($NF + 0 >= 1) found = 1 } END { exit(found ? 0 : 1) }' <<<"$metrics"
+}
+
+# Scrapes the running replacement pod's /metrics over the same in-pod node/http
+# channel the /ready probes use, then adjudicates. The sub-0.5s window B4 leaves
+# unobservable on /ready is covered here: the counter is monotonic.
+safety_pass_metrics_certify() {
+ local metrics
+ metrics="$(kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \
+ env "HCC_E2E_PORT=${HCC_PORT}" \
+ node -e '
+ const http = require("node:http");
+ const request = http.get(
+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/metrics"},
+ response => {
+ let body = "";
+ response.setEncoding("utf8");
+ response.on("data", chunk => { body += chunk; });
+ response.on("end", () => {
+ process.stdout.write(body);
+ process.exit(response.statusCode === 200 ? 0 : 1);
+ });
+ }
+ );
+ request.setTimeout(2000, () => { request.destroy(); process.exit(1); });
+ request.on("error", () => process.exit(1));
+ ')" || return 1
+ safety_pass_metrics_are_certified "$metrics"
+}
+
+hcc_ready_after_stale_policy_revoked() {
+ # A single poll that sees 200 WHILE the stale policy is still present is a
+ # safety-contract violation and must be FATAL, never retried: retrying lets a
+ # later poll (policy already revoked) turn the violation into a false PASS.
+ # rc=2 signals that fatal ordering violation to wait_until_fast; rc=1 is a
+ # benign "not ready yet, keep polling". The sub-0.5s window where revocation
+ # races the probe inside one poll stays unobservable on THIS signal — the
+ # third signal (M6, /metrics) covers it.
+ local policy_absent_before=0
+ external_egress_policy_absent && policy_absent_before=1
+ probe_new_hcc_ready_endpoint || return 1
+ if external_egress_policy_absent; then
+ [ "$policy_absent_before" -eq 1 ] && return 0
+ # Revocation raced the probe inside this poll: the final state is clean but
+ # this poll cannot adjudicate the order. Retry to adjudicate cleanly.
+ return 1
+ fi
+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=ready-200-with-stale-policy-present"
+ return 2
+}
+
+# Canonical, order-stable projection of the single external-egress policy for the
+# fixture server: name, ownership labels, and full spec. Used to prove a
+# same-intent restart RETAINS the identity-stable DNS allow byte-for-byte (B3
+# retain-identity-stable contract), the inverse of the divergent revoke path.
+external_egress_policy_snapshot() {
+ kctl get networkpolicy -n "$MCP_NS" \
+ -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
+ -o json 2>/dev/null |
+ jq -ceS \
+ --arg managedByLabel "$MANAGED_BY_LABEL" \
+ --arg managedByValue "$MANAGED_BY_VALUE" \
+ --arg policyTypeLabel "$POLICY_TYPE_LABEL" \
+ --arg serverLabel "$MCP_SERVER_LABEL" \
+ --arg server "$MCP_NAME" '
+ [.items[] | {
+ name: .metadata.name,
+ managedBy: .metadata.labels[$managedByLabel],
+ policyType: .metadata.labels[$policyTypeLabel],
+ serverLabel: .metadata.labels[$serverLabel],
+ spec
+ }]
+ | select(length == 1)
+ | .[0]
+ | select(.managedBy == $managedByValue
+ and .policyType == "external-egress"
+ and .serverLabel == $server)
+ '
+}
+
+# True when the external-egress policy is present and identical (spec +
+# ownership) to the snapshot captured before the same-intent restart. An API
+# error or a missing/renamed policy fails closed.
+identity_stable_policy_retained() {
+ local current
+ [ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] || return 1
+ current="$(external_egress_policy_snapshot)" || return 1
+ [ -n "$current" ] || return 1
+ [ "$current" = "$SAME_INTENT_EGRESS_SNAPSHOT" ]
+}
+
+hcc_ready_with_identity_stable_policy_retained() {
+ # INVERSE of the pre-B3 revoke-before-ready contract. B3 retains an
+ # identity-stable DNS allow across a same-intent restart with NO deny window and
+ # certifies /ready=200 WHILE that allow is still present and unchanged. A single
+ # poll that reaches /ready=200 with the identity-stable allow ABSENT from the
+ # start is the retain regression (a deny window) and must be FATAL, never
+ # retried: retrying would let a later poll — after the held-refresh timeout
+ # legitimately fail-closed deletes the allow — mask the regression as a PASS.
+ # rc=2 signals that fatal violation to wait_until_fast; rc=1 is a benign
+ # "not ready yet, keep polling".
+ local retained_before=0
+ identity_stable_policy_retained && retained_before=1
+ probe_new_hcc_ready_endpoint || return 1
+ if identity_stable_policy_retained; then
+ [ "$retained_before" -eq 1 ] && return 0
+ # The allow reappeared within this poll: adjudicate cleanly on the next poll.
+ return 1
+ fi
+ if [ "$retained_before" -eq 1 ]; then
+ # The allow was present immediately before this probe and is gone now: a
+ # legitimate fail-closed refresh timeout can race the probe inside one poll.
+ # Retry to adjudicate — a real retain regression never presents the allow at
+ # readiness, so a later poll still returns rc=2.
+ return 1
+ fi
+ HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=identity-stable-policy-revoked"
+ return 2
+}
+
+hcc_restore_is_verified() {
+ local deployment
+ deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1
+ jq -e \
+ --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \
+ --argjson replicas "${ORIGINAL_REPLICAS:-1}" '
+ .spec.replicas == $replicas and
+ (.status.readyReplicas // 0) == $replicas and
+ (.status.availableReplicas // 0) == $replicas and
+ (.spec.template.spec.dnsPolicy // "ClusterFirst") == $dnsPolicy and
+ (.spec.template.spec.dnsConfig // null) == null
+ ' <<<"$deployment" >/dev/null || return 1
+ probe_hcc_ready_pod
+}
+
+release_dns_blocker() {
+ local release_script
+ release_script="$(cat <<'NODE'
+const http = require('http');
+const request = http.request(
+ {host: '127.0.0.1', port: 8090, path: '/release', method: 'POST'},
+ response => {
+ response.resume();
+ response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));
+ }
+);
+request.setTimeout(5000, () => request.destroy(new Error('timeout')));
+request.once('error', error => { console.error(error.message); process.exit(3); });
+request.end();
+NODE
+)"
+ kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \
+ node -e "$release_script"
+}
+
+hold_dns_blocker() {
+ local hold_script
+ DNS_HOLD_COUNT_AT_ARM="$(dns_hold_count)" ||
+ return 1
+ hold_script="$(cat <<'NODE'
+const http = require('http');
+const request = http.request(
+ {host: '127.0.0.1', port: 8090, path: '/hold', method: 'POST'},
+ response => {
+ response.resume();
+ response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));
+ }
+);
+request.setTimeout(5000, () => request.destroy(new Error('timeout')));
+request.once('error', error => { console.error(error.message); process.exit(3); });
+request.end();
+NODE
+)"
+ kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \
+ node -e "$hold_script"
+}
+
+delete_fixture_runtime_leaving_dns_policy_for_safety() {
+ local failed=0
+ kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete service "$MCP_NAME" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ [ "$failed" = 0 ] && fixture_runtime_absent
+}
+
+delete_dns_blocker_fixture() {
+ local failed=0
+ kctl delete networkpolicy "$HCC_DNS_POLICY" "$DNS_BLOCKER_POLICY" -n "$HCC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ [ "$failed" = 0 ]
+}
+
+delete_residual_fixture_runtime() {
+ local failed=0
+ kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete service "$MCP_NAME" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy "$CONTEXT_EGRESS_POLICY" -n "$HOST_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy "$RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${MCP_NAME}" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ delete_peer_fleet_runtime || failed=1
+ [ "$failed" = 0 ]
+}
+
+delete_peer_fleet_runtime() {
+ local index server context context_policy failed=0
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ server="${MCP_FLEET_NAMES[index]}"
+ context="${CONTEXT_FLEET_NAMES[index]}"
+ context_policy="ctx-${context}-${server}"
+ kctl delete deployment "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \
+ >/dev/null 2>&1 || failed=1
+ kctl delete service "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \
+ >/dev/null 2>&1 || failed=1
+ kctl delete configmap "${server}-nginx-conf" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \
+ >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${server}" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy "${context_policy}-egress" -n "$HOST_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete networkpolicy "rpc-egress-${context}-${server}" -n "$RPC_PROXY_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ done
+ [ "$failed" = 0 ]
+}
+
+delete_peer_fleet_inputs() {
+ local index failed=0
+ for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
+ kctl delete context "${CONTEXT_FLEET_NAMES[index]}" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete mcpserver "${MCP_FLEET_NAMES[index]}" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ done
+ [ "$failed" = 0 ]
+}
+
+suite_resources_absent() {
+ local namespaced cluster_crds
+ namespaced="$(kctl get deployment,service,networkpolicy -A \
+ -l "e2e.clerum.io/suite=${SUITE_NAME}" -o name 2>/dev/null)" || return 1
+ cluster_crds="$(
+ kctl get mcpserver,context -A -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ -o name 2>/dev/null
+ )" || return 1
+ [ -z "$namespaced$cluster_crds" ]
+}
+
+print_repair_instructions() {
+ cat >&2 <<EOF
+HCC MCP readiness gate cleanup could not restore a verified clean state.
+Context: ${E2E_KUBECONTEXT}
+HCC: ${HCC_NS}/${HCC_DEPLOY}
+DNS blocker: ${HCC_NS}/${DNS_BLOCKER_NAME}
+Fixtures: ${MCP_NS}/${MCP_NAME}, ${MCP_NS}/${CONTEXT_NAME}
+
+Inspect before changing anything:
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
+ kubectl --context=${E2E_KUBECONTEXT} get deployment,service,networkpolicy -A -l e2e.clerum.io/suite=${SUITE_NAME}
+ kubectl --context=${E2E_KUBECONTEXT} get mcpserver,context -A -l e2e.clerum.io/suite=${SUITE_NAME}
+
+Do not remove the HCC gate lock until HCC health and fixture absence are verified.
+
+Restore the original HCC replica count before verifying rollout and /ready:
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=240s
+EOF
+}
+
+cleanup() {
+ local status=$? cleanup_failed=0 restore_ok=1 fixture_inputs_removed=1 restore_patch
+ local clean_window_pod
+ trap - EXIT
+ set +e
+
+ if [ "$HCC_MUTATED" = 1 ]; then
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1 ||
+ restore_ok=0
+ if [ "$restore_ok" = 1 ]; then
+ wait_until 120 "HCC pods to stop before fixture cleanup" \
+ hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
+ fi
+ fi
+
+ if [ "$CONTEXT_CREATED" = 1 ]; then
+ kctl delete context "$CONTEXT_NAME" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||
+ {
+ cleanup_failed=1
+ fixture_inputs_removed=0
+ }
+ fi
+ if [ "$MCP_CREATED" = 1 ]; then
+ kctl delete mcpserver "$MCP_NAME" -n "$MCP_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||
+ {
+ cleanup_failed=1
+ fixture_inputs_removed=0
+ }
+ fi
+ if [ "$PEER_FLEET_CREATED" = 1 ]; then
+ delete_peer_fleet_inputs || {
+ cleanup_failed=1
+ fixture_inputs_removed=0
+ }
+ fi
+ if ! wait_until 60 "gate-owned Context and McpServer inputs to disappear" \
+ fixture_inputs_absent >/dev/null 2>&1; then
+ cleanup_failed=1
+ fixture_inputs_removed=0
+ fi
+
+ if [ "$HCC_MUTATED" = 1 ]; then
+ if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then
+ restore_patch="$(jq -cn --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \
+ '{spec:{template:{spec:{dnsPolicy:$dnsPolicy,dnsConfig:null}}}}')"
+ kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \
+ -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ --timeout=240s >/dev/null 2>&1 || restore_ok=0
+ if [ "$restore_ok" = 1 ]; then
+ wait_until 60 "restored HCC DNS, replicas, Ready pod, and /ready endpoint" \
+ hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0
+ fi
+ else
+ restore_ok=0
+ fi
+ fi
+
+ if [ "$restore_ok" = 1 ]; then
+ if ! wait_until 180 "HCC to remove all MCP/Context fixture runtime resources" \
+ fixture_resources_absent >/dev/null 2>&1; then
+ cleanup_failed=1
+ delete_residual_fixture_runtime || cleanup_failed=1
+ fi
+ if [ "$DNS_BLOCKER_CREATED" = 1 ]; then
+ delete_dns_blocker_fixture || cleanup_failed=1
+ fi
+ wait_until 60 "all HCC MCP readiness suite resources to disappear" \
+ suite_resources_absent >/dev/null 2>&1 || cleanup_failed=1
+
+ # Clean window B: DNS restored and fixture inputs gone. The restored pod is
+ # new (scale-0 preceded the restore), so its log is scoped to this window.
+ clean_window_pod="$(running_hcc_pod)"
+ if [ -n "$clean_window_pod" ] &&
+ clean_window_netpol_log_negatives_hold "$clean_window_pod"; then
+ clean_window_checks_run=$((clean_window_checks_run + 1))
+ else
+ fail "post-restore clean-window NetworkPolicy log negatives did not hold"
+ cleanup_failed=1
+ fi
+ if fixture_mcp_runtime_absent; then
+ clean_window_checks_run=$((clean_window_checks_run + 1))
+ else
+ fail "post-restore fixture runtime or external-egress policy survived"
+ cleanup_failed=1
+ fi
+ if clean_window_runtime_and_ready_absent; then
+ clean_window_checks_run=$((clean_window_checks_run + 1))
+ else
+ fail "post-restore McpServer Ready status or external-egress policy survived"
+ cleanup_failed=1
+ fi
+ else
+ cleanup_failed=1
+ fi
+
+ if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then
+ cleanup_failed=1
+ fi
+
+ if [ "$status" = 0 ]; then
+ [ "${clean_window_checks_run:-0}" -gt 0 ] ||
+ die "no clean-window negative checks ran (clean_window_checks_run=0)"
+ fi
+
+ if [ "$cleanup_failed" -ne 0 ]; then
+ echo "HCC MCP readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2
+ [ "$restore_ok" = 1 ] || print_repair_instructions
+ [ "$status" = 0 ] && status=1
+ fi
+ if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
+ header "HCC McpServer/Context readiness gate passed"
+ echo "$final_probe"
+ fi
+ exit "$status"
+}
+trap cleanup EXIT
+
+header "HCC readiness during initial McpServer and Context convergence"
+
+require_branch_owned_hcc_gate "$HCC_NS"
+kctl get nodes -o json |
+ jq -e --arg context "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
+ die "target context is not a minikube cluster"
+suite_resources_absent ||
+ die "stale ${SUITE_NAME} resources exist; inspect them before starting a new fault injection"
+acquire_hcc_watch_gate_lock ||
+ die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
+ok "branch helper, profile, exact HEAD/fingerprint/gate, and exclusive HCC lock verified"
+
+# Clean window A: the DNS fault injection is not applied yet. The branch-deployed
+# HCC must show a clean NetworkPolicy lane and zero fixture residue before the
+# gate mutates anything.
+clean_window_pod="$(running_hcc_pod)" && [ -n "$clean_window_pod" ] ||
+ die "no Running HCC pod before fault injection"
+clean_window_netpol_log_negatives_hold "$clean_window_pod" ||
+ die "pre-injection HCC logs already contain NetworkPolicy-lane failure or retry markers"
+clean_window_checks_run=$((clean_window_checks_run + 1))
+fixture_mcp_runtime_absent ||
+ die "pre-injection fixture runtime or external-egress policy already exists"
+clean_window_checks_run=$((clean_window_checks_run + 1))
+clean_window_runtime_and_ready_absent ||
+ die "pre-injection McpServer Ready status or external-egress policy already exists"
+clean_window_checks_run=$((clean_window_checks_run + 1))
+ok "all three clean-window negative families hold before DNS fault injection"
+
+ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.replicas}')"
+[ "$ORIGINAL_REPLICAS" = 1 ] ||
+ die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
+ORIGINAL_DNS_POLICY="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.dnsPolicy}')"
+ORIGINAL_DNS_POLICY="${ORIGINAL_DNS_POLICY:-ClusterFirst}"
+[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] ||
+ die "HCC already has non-default dnsPolicy ${ORIGINAL_DNS_POLICY}; refusing a non-restorable fixture"
+ORIGINAL_DNS_CONFIG="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |
+ jq -c '.spec.template.spec.dnsConfig // null')"
+[ "$ORIGINAL_DNS_CONFIG" = null ] ||
+ die "HCC already has dnsConfig ${ORIGINAL_DNS_CONFIG}; refusing a non-restorable fixture"
+HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
+[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"
+HCC_PORT="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json | jq -r '
+ .spec.template.spec.containers[]? |
+ select(.name == "host-context-controller") as $container |
+ (
+ [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //
+ [$container.ports[]? | select(.name == "http") | .containerPort][0] //
+ empty
+ )
+')"
+[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||
+ die "could not resolve a valid HCC HTTP port from the Deployment"
+UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"
+[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-system/kube-dns ClusterIP"
+running_dns_pods="$(kctl get pods -n kube-system -l k8s-app=kube-dns \
+ --field-selector=status.phase=Running -o name)"
+[ -n "$running_dns_pods" ] ||
+ die "no Running kube-dns pod is available"
+
+DNS_BLOCKER_CREATED=1
+kctl apply -f - >/dev/null <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ${DNS_BLOCKER_NAME}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: ${DNS_BLOCKER_NAME}
+ template:
+ metadata:
+ labels:
+ app: ${DNS_BLOCKER_NAME}
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+ spec:
+ automountServiceAccountToken: false
+ terminationGracePeriodSeconds: 1
+ containers:
+ - name: dns-blocker
+ image: ${HCC_IMAGE}
+ imagePullPolicy: IfNotPresent
+ command: [node, -e]
+ args:
+ - >-
+ const dgram=require('node:dgram'),http=require('node:http'),
+ net=require('node:net');
+ const target=process.env.TARGET_DNS,upstream=process.env.UPSTREAM_DNS;
+ const udp=dgram.createSocket('udp4');let released=process.env.START_RELEASED==='1';const held=[];
+ function question(message){let offset=12,labels=[];while(offset<message.length){
+ const length=message[offset++];if(length===0)break;if(offset+length>message.length)
+ return null;labels.push(message.subarray(offset,offset+length).toString('ascii'));
+ offset+=length;}if(offset+4>message.length)return null;return
+ {name:labels.join('.'),type:message.readUInt16BE(offset),end:offset+4};}
+ function answer(message,parsed){const header=Buffer.from(message.subarray(0,12));
+ header.writeUInt16BE(0x8180,2);header.writeUInt16BE(1,6);
+ header.writeUInt16BE(0,8);header.writeUInt16BE(0,10);
+ const record=Buffer.from([0xc0,0x0c,0x00,0x01,0x00,0x01,0x00,0x00,
+ 0x00,0x1e,0x00,0x04,93,184,216,34]);
+ return Buffer.concat([header,message.subarray(12,parsed.end),record]);}
+ function tcpFrame(message){const frame=Buffer.alloc(message.length+2);
+ frame.writeUInt16BE(message.length,0);message.copy(frame,2);return frame;}
+ function respond(entry){const response=answer(entry.message,entry.parsed);
+ if(entry.transport==='udp'){udp.send(response,entry.rinfo.port,
+ entry.rinfo.address);console.log('released DNS A '+target+' via UDP');return;}
+ if(!entry.socket.destroyed){entry.socket.write(tcpFrame(response));
+ console.log('released DNS A '+target+' via TCP');}}
+ function forwardUdp(message,rinfo){const relay=dgram.createSocket('udp4');
+ const timer=setTimeout(()=>relay.close(),5000);relay.once('message',response=>{
+ clearTimeout(timer);udp.send(response,rinfo.port,rinfo.address);relay.close();});
+ relay.once('error',()=>{clearTimeout(timer);relay.close();});
+ relay.send(message,53,upstream);}
+ function forwardTcp(message,socket){const relay=net.createConnection(
+ {host:upstream,port:53},()=>relay.write(tcpFrame(message)));
+ let pending=Buffer.alloc(0);relay.setTimeout(5000,()=>relay.destroy());
+ relay.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);
+ if(pending.length<2)return;const length=pending.readUInt16BE(0);
+ if(pending.length<length+2)return;if(!socket.destroyed)
+ socket.write(pending.subarray(0,length+2));relay.destroy();});
+ relay.once('error',()=>relay.destroy());}
+ function handle(message,entry){const parsed=question(message);
+ if(parsed&&parsed.name===target&&parsed.type===1){entry.message=message;
+ entry.parsed=parsed;if(released)respond(entry);else{held.push(entry);
+ console.log('holding DNS A '+target+' via '+entry.transport.toUpperCase());}
+ return;}if(entry.transport==='udp')forwardUdp(message,entry.rinfo);
+ else forwardTcp(message,entry.socket);}
+ let listeners=0;function listening(){listeners+=1;if(listeners===2)
+ console.log('dns hold proxy ready for '+target+' over UDP and TCP');}
+ udp.on('message',(message,rinfo)=>handle(message,{transport:'udp',rinfo}));
+ udp.bind(8053,'0.0.0.0',listening);
+ const tcp=net.createServer(socket=>{let pending=Buffer.alloc(0);
+ socket.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);
+ while(pending.length>=2){const length=pending.readUInt16BE(0);
+ if(pending.length<length+2)return;const message=pending.subarray(2,length+2);
+ pending=pending.subarray(length+2);handle(message,{transport:'tcp',socket});}});
+ });tcp.listen(8053,'0.0.0.0',listening);
+ http.createServer((request,response)=>{if(request.method==='POST'&&
+ request.url==='/release'){released=true;while(held.length)respond(held.shift());
+ response.writeHead(200);response.end('released');return;}if(request.method==='POST'&&
+ request.url==='/hold'){released=false;response.writeHead(200);response.end('holding');return;}
+ response.writeHead(404);response.end();}).listen(8090,'127.0.0.1');
+ env:
+ - name: TARGET_DNS
+ value: ${TARGET_DNS}
+ - name: START_RELEASED
+ value: "1"
+ - name: UPSTREAM_DNS
+ value: ${UPSTREAM_DNS_IP}
+ resources:
+ requests: {cpu: 5m, memory: 20Mi}
+ limits: {cpu: 75m, memory: 64Mi}
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities: {drop: [ALL]}
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+ runAsUser: 1000
+ runAsGroup: 1000
+ seccompProfile: {type: RuntimeDefault}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ${DNS_BLOCKER_NAME}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ selector:
+ app: ${DNS_BLOCKER_NAME}
+ ports:
+ - name: dns-udp
+ protocol: UDP
+ port: 53
+ targetPort: 8053
+ - name: dns-tcp
+ protocol: TCP
+ port: 53
+ targetPort: 8053
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${DNS_BLOCKER_POLICY}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ podSelector:
+ matchLabels:
+ app: ${DNS_BLOCKER_NAME}
+ policyTypes: [Ingress, Egress]
+ ingress:
+ - from:
+ - podSelector:
+ matchLabels:
+ app: ${HCC_DEPLOY}
+ ports:
+ - {protocol: UDP, port: 8053}
+ - {protocol: UDP, port: 53}
+ - {protocol: TCP, port: 8053}
+ - {protocol: TCP, port: 53}
+ egress:
+ - to:
+ - namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: kube-system
+ podSelector:
+ matchLabels:
+ k8s-app: kube-dns
+ ports:
+ - {protocol: UDP, port: 53}
+ - {protocol: TCP, port: 53}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${HCC_DNS_POLICY}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ podSelector:
+ matchLabels:
+ app: ${HCC_DEPLOY}
+ policyTypes: [Egress]
+ egress:
+ - to:
+ - podSelector:
+ matchLabels:
+ app: ${DNS_BLOCKER_NAME}
+ ports:
+ - {protocol: UDP, port: 53}
+ - {protocol: UDP, port: 8053}
+ - {protocol: TCP, port: 53}
+ - {protocol: TCP, port: 8053}
+EOF
+kctl rollout status deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
+ --timeout=90s >/dev/null || die "DNS blocker did not become ready"
+wait_until 30 "DNS blocker process to listen" \
+ dns_log_contains "dns hold proxy ready for ${TARGET_DNS}" ||
+ die "DNS blocker process did not start"
+DNS_BLOCKER_IP="$(kctl get service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
+ -o jsonpath='{.spec.clusterIP}')"
+[ -n "$DNS_BLOCKER_IP" ] || die "DNS blocker Service has no ClusterIP"
+ok "isolated DNS hold proxy is ready and forwards non-fixture UDP/TCP queries to CoreDNS"
+
+[ "${clean_window_checks_run:-0}" -ge 3 ] ||
+ die "pre-injection clean-window negatives did not run before the DNS mutation"
+
+HCC_MUTATED=1
+dns_patch="$(jq -cn --arg nameserver "$DNS_BLOCKER_IP" '
+ {spec:{template:{spec:{
+ dnsPolicy:"None",
+ dnsConfig:{
+ nameservers:[$nameserver],
+ options:[
+ {name:"ndots",value:"1"},
+ {name:"timeout",value:"30"},
+ {name:"attempts",value:"1"}
+ ]
+ }
+ }}}}
+')"
+kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \
+ -p "$dns_patch" >/dev/null
+kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||
+ die "HCC did not become Ready with the deterministic DNS proxy"
+wait_until 60 "HCC /ready endpoint with the released DNS proxy" probe_hcc_ready_pod ||
+ die "HCC readiness was unavailable before the stale-policy fixture"
+
+CONTEXT_CREATED=1
+kctl apply -f - >/dev/null <<EOF
+apiVersion: clerum.io/v1alpha1
+kind: Context
+metadata:
+ name: ${CONTEXT_NAME}
+ namespace: ${MCP_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ contextId: ${CONTEXT_ID}
+ description: E2E HCC readiness Context updated while initial MCP convergence is blocked.
+ mcpServers: []
+EOF
+
+MCP_CREATED=1
+kctl apply -f - >/dev/null <<EOF
+apiVersion: clerum.io/v1alpha1
+kind: McpServer
+metadata:
+ name: ${MCP_NAME}
+ namespace: ${MCP_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_LABEL}"
+spec:
+ contextRef: ${CONTEXT_NAME}
+ description: E2E MCP fixture for initial fleet readiness.
+ image: clerum/mock-mcp-server:test
+ imagePullPolicy: IfNotPresent
+ transport:
+ type: streamableHttp
+ url: http://${MCP_NAME}.${MCP_NS}.svc.cluster.local:3000/mcp
+ port: 3000
+ egressBindings:
+ - dns: ${TARGET_DNS}
+ port: 443
+ protocol: ${INITIAL_EXTERNAL_EGRESS_PROTOCOL}
+EOF
+
+wait_until 180 "HCC to create the real initial TCP external-egress policy" \
+ external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
+ die "HCC never created the initial external-egress policy for the stale-policy fixture"
+ok "HCC created a real initial TCP external-egress policy"
+BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)" ||
+ die "could not capture canonical baseline NetworkPolicy ownership and specs"
+
+# Same-intent safety phase: restart with the exact same desired binding. The
+# safety pass must RETAIN the identity-stable DNS-derived policy without resolving
+# DNS (B3 retain-identity-stable contract), certify global readiness with that
+# allow still present — no deny window — and leave the affected runtime blocked
+# for the additive lane. Retention-at-readiness is adjudicated promptly: a
+# held-refresh timeout later fail-closed deletes the retained allow, so the proof
+# must land before any refresh timeout.
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
+wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||
+ die "existing HCC pod did not stop"
+delete_fixture_runtime_leaving_dns_policy_for_safety ||
+ die "could not remove the initial MCP runtime while retaining its external-egress policy"
+external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
+ die "the initial TCP policy disappeared before the same-intent safety pass"
+SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)" ||
+ die "could not capture the identity-stable external-egress policy snapshot"
+[ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] ||
+ die "the identity-stable external-egress policy snapshot was empty before the same-intent restart"
+create_peer_fleet || die "could not create the gate-owned MCP/Context peer fleet"
+ok "created ${#MCP_FLEET_NAMES[@]} gate-owned MCP/Context fixtures before fleet bootstrap"
+
+hold_dns_blocker || die "could not hold the next fixture DNS query"
+DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||
+ die "could not capture the DNS release baseline before the same-intent restart"
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
+wait_until 120 "same-intent HCC pod to enter Running" capture_hcc ||
+ die "same-intent HCC pod did not start"
+identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \
+ -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"
+read -r HCC_UID HCC_RESTARTS <<<"$identity"
+[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||
+ die "could not capture same-intent HCC pod identity"
+# Adjudicate retention-at-readiness FIRST — at the earliest possible moment —
+# because the certifying safety pass retains WITHOUT resolving DNS, so /ready=200
+# and the retained allow flip together. The separate additive refresh reaches the
+# DNS hold and, on its timeout, fail-closed deletes the retained allow; latching
+# the retained+Ready observation here beats that timeout.
+wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained" \
+ hcc_ready_with_identity_stable_policy_retained ||
+ die "HCC did not certify readiness with the same-intent identity-stable DNS policy retained (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
+# Retention-at-readiness is latched. Corroborate promptly — present, unchanged,
+# under a still-held DNS, with no restart and the baseline intact — before any
+# held-refresh timeout can fail-closed delete the retained allow.
+dns_has_not_released_since_hold ||
+ die "the DNS blocker released before the same-intent retention assertion"
+identity_stable_policy_retained ||
+ die "the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)"
+fixture_runtime_and_ready_absent ||
+ die "the affected runtime or Ready status escaped the held same-intent DNS boundary"
+hcc_identity_is_stable ||
+ die "HCC restarted while the same-intent identity-stable DNS policy was retained"
+baseline_policies_unchanged ||
+ die "baseline safety NetworkPolicy ownership or specs changed during same-intent retention"
+wait_until 30 "Kubernetes to corroborate same-intent HCC readiness" \
+ hcc_kubernetes_readiness_is_exact ||
+ die "Kubernetes did not corroborate same-intent HCC readiness"
+wait_until 120 "the same-intent external-egress refresh to reach the DNS hold" \
+ dns_hold_observed_since_arm ||
+ die "the same-intent external-egress refresh never reached the DNS hold"
+ok "same-intent safety retained the identity-stable DNS policy while HCC stayed Ready and runtime stayed blocked"
+
+release_dns_blocker || die "could not release the same-intent DNS query"
+wait_until 30 "DNS blocker to answer the same-intent query" \
+ dns_released_since_hold ||
+ die "DNS blocker did not answer the same-intent query"
+wait_until 120 "current TCP external-egress policy after same-intent DNS release" \
+ external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
+ die "the current TCP external-egress policy did not converge after DNS release"
+wait_until 180 "same-intent runtime convergence after DNS release" \
+ fixture_runtime_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
+ die "same-intent runtime did not converge after DNS release"
+
+# Security phase: a protocol change invalidates the current TCP policy. The old
+# TCP allow must be revoked even when DNS prevents constructing the UDP policy.
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
+wait_until 120 "same-intent HCC pod to stop" hcc_pods_absent ||
+ die "same-intent HCC pod did not stop"
+delete_fixture_runtime_leaving_dns_policy_for_safety ||
+ die "could not remove the same-intent runtime before the divergent revision"
+external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
+ die "the current TCP policy disappeared before the divergent safety pass"
+hold_dns_blocker || die "could not hold the divergent fixture DNS query"
+DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||
+ die "could not capture the DNS release baseline before the divergent revision"
+egress_patch="$(jq -cn --arg dns "$TARGET_DNS" \
+ '{spec:{egressBindings:[{dns:$dns,port:443,protocol:"UDP"}]}}')"
+kctl patch mcpserver "$MCP_NAME" -n "$MCP_NS" --type=merge \
+ -p "$egress_patch" >/dev/null
+wait_until 60 "McpServer API to expose the revised UDP external-egress binding" \
+ mcp_egress_binding_is_udp ||
+ die "McpServer external-egress revision did not persist"
+
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
+wait_until 120 "replacement HCC pod to enter Running" capture_hcc ||
+ die "replacement HCC pod did not start"
+identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \
+ -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"
+read -r HCC_UID HCC_RESTARTS <<<"$identity"
+[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||
+ die "could not capture replacement HCC pod identity"
+
+wait_until 120 "fixture external-egress DNS query to reach the hold proxy" \
+ dns_hold_observed_since_arm ||
+ die "NetworkPolicy safety convergence never reached the divergent DNS hold"
+wait_until_fast 120 "HCC readiness after revoking the divergent stale policy" \
+ hcc_ready_after_stale_policy_revoked ||
+ die "HCC did not become globally ready after revoking the divergent stale policy (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
+wait_until 30 "Kubernetes to corroborate HCC readiness during divergent DNS hold" \
+ hcc_kubernetes_readiness_is_exact ||
+ die "Kubernetes did not corroborate HCC readiness during divergent DNS hold"
+dns_log_contains "holding DNS A ${TARGET_DNS}" ||
+ die "DNS hold ended before the fail-closed revocation assertion"
+if ! dns_has_not_released_since_hold; then
+ die "DNS blocker released before the fail-closed revocation assertion"
+fi
+fixture_runtime_and_ready_absent ||
+ die "fixture runtime or Ready status appeared while divergent DNS remained held"
+external_egress_policy_absent ||
+ die "a divergent external-egress policy survived the fail-closed revocation"
+context_policies_absent ||
+ die "Context policy existed before the Context watch update"
+baseline_policies_unchanged ||
+ die "baseline safety NetworkPolicy ownership or specs changed during safety convergence"
+hcc_identity_is_stable ||
+ die "HCC restarted while the MCP external-egress query was held"
+wait_until_fast 30 "authoritative safety pass metrics to certify the revocation" \
+ safety_pass_metrics_certify ||
+ die "safety_pass_* metrics did not corroborate the stale-policy revocation (families renamed or pass never measured?)"
+ok "/ready is 200 while the divergent stale policy and affected runtime remain absent"
+ok "safety_pass_* metrics corroborate an authoritative pass revoked the stale policy"
+ok "baseline NetworkPolicies remain present throughout fail-closed safety convergence"
+
+initial_empty_context_log="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []"
+wait_until 120 "initial NetworkPolicy pass to reconcile the fixture's empty Context snapshot" \
+ hcc_log_contains "$initial_empty_context_log" ||
+ die "initial NetworkPolicy pass never reconciled the fixture's empty Context"
+wait_until 120 "a real divergent external-egress retry to be scheduled" \
+ external_egress_retry_progress_is_observed ||
+ die "the held divergent refresh did not schedule a real retry"
+DNS_HOLD_COUNT_AT_RETRY_SCHEDULE="$(dns_hold_count)" ||
+ die "could not snapshot DNS hold evidence after the divergent retry was scheduled"
+wait_until 180 "the scheduled divergent external-egress retry to execute" \
+ dns_retry_query_observed_since_schedule ||
+ die "the scheduled divergent external-egress retry did not execute"
+context_policies_absent ||
+ die "Context policy appeared before the isolated MODIFIED event"
+
+context_patch="$(jq -cn --arg server "$MCP_NAME" '{spec:{mcpServers:[$server]}}')"
+kctl patch context "$CONTEXT_NAME" -n "$MCP_NS" --type=merge \
+ -p "$context_patch" >/dev/null
+wait_until 60 "Context API to expose the latest server membership" context_watch_has_latest_spec ||
+ die "Context update did not persist"
+wait_until 60 "HCC Context watch to observe the update during the blocked fleet pass" \
+ hcc_log_contains "Context watch event: MODIFIED for ${CONTEXT_NAME}" ||
+ die "HCC did not observe the real Context update while initial MCP convergence was blocked"
+wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold" \
+ context_policies_converged ||
+ die "Context policies remained blocked behind the held McpServer convergence"
+fixture_runtime_and_ready_absent ||
+ die "McpServer runtime or Ready status escaped the deterministic DNS hold boundary"
+external_egress_policy_absent ||
+ die "a divergent external-egress policy reappeared while DNS remained held"
+dns_log_contains "holding DNS A ${TARGET_DNS}" ||
+ die "DNS hold evidence disappeared before Context policy convergence"
+if ! dns_has_not_released_since_hold; then
+ die "DNS blocker released the McpServer query before the Context isolation assertion"
+fi
+probe_new_hcc_ready_endpoint ||
+ die "HCC lost global readiness while the affected runtime remained safely blocked (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
+hcc_kubernetes_readiness_is_exact ||
+ die "Kubernetes lost HCC readiness during isolated Context convergence"
+hcc_identity_is_stable ||
+ die "HCC restarted while Context policies converged around the held MCP entity"
+ok "the MODIFIED handler, not the initial empty-Context pass, converged all three policies"
+ok "global readiness and Context progress continued while the affected runtime stayed blocked"
+
+release_dns_blocker ||
+ die "could not release the fixture DNS query"
+wait_until 30 "DNS blocker to answer the held query" \
+ dns_released_since_hold ||
+ die "DNS blocker did not answer the held query"
+wait_until 120 "current UDP external-egress policy after DNS release" \
+ external_egress_policy_converged_with_protocol UDP ||
+ die "the current UDP external-egress policy did not converge after DNS release"
+probe_new_hcc_ready_endpoint ||
+ die "HCC lost readiness while the released UDP policy converged (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
+wait_until 300 "McpServer runtime, status, and latest Context NetworkPolicies to converge" \
+ fixture_converged ||
+ die "McpServer/Context/NetworkPolicy convergence did not complete after DNS release"
+wait_until 300 "all peer MCP/Context runtimes to converge after releasing the held primary egress" \
+ peer_fleet_converged ||
+ die "peer MCP/Context fleet did not converge after the held primary egress was released"
+hcc_identity_is_stable ||
+ die "HCC restarted instead of completing the released background convergence"
+final_probe="$(probe_hcc_final_context)" ||
+ die "HCC final Context-scoped discovery did not expose the converged fixture"
+echo "$final_probe" | jq -e \
+ --arg fixture "$MCP_NAME" --arg context "$CONTEXT_NAME" '
+ .readyStatus == 200 and .scopedDiscoveryStatus == 200 and
+ .fixture == $fixture and .fixtureReady == true and .contextRef == $context
+ ' >/dev/null ||
+ die "HCC final probe returned an invalid result: ${final_probe}"
+baseline_policies_unchanged ||
+ die "baseline safety NetworkPolicy ownership or specs changed before final convergence"
+ok "DNS release created the current UDP policy and affected runtime without gating HCC readiness"
+ok "the full worker-width-crossing peer fleet converged after releasing the held primary egress"
+ok "Context ingress, mcp-host egress, and rpc-proxy egress policies reflect the latest watch state"
+ok "HCC stayed Ready through divergent revocation while the affected runtime remained fail-closed"
+
+header "HCC McpServer/Context assertions complete; restoring branch-owned runtime"
diff --git a/scripts/e2e/e2e-hcc-readiness-bootstrap.sh b/scripts/e2e/e2e-hcc-readiness-bootstrap.sh
new file mode 100755
index 000000000..eb485a590
--- /dev/null
+++ b/scripts/e2e/e2e-hcc-readiness-bootstrap.sh
@@ -0,0 +1,785 @@
+#!/usr/bin/env bash
+# Proves that HCC exposes its real readiness and discovery endpoints while the
+# initial Host fleet reconciliation is still demonstrably in progress.
+#
+# The gate temporarily routes HCC's control-api token issuance to an isolated
+# in-cluster HTTP blocker, creates a gate-owned fleet while HCC is stopped, and
+# restarts HCC. There are more fixture Hosts than two complete bounded worker
+# waves, so the initial pass remains deterministically active even though each
+# individual held token request has its own client timeout. No real Host or
+# runtime-token Secret is modified.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+source "${SCRIPT_DIR}/e2e-lib.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
+
+[ -n "$E2E_KUBECONTEXT" ] || {
+ echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2
+ exit 1
+}
+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
+ echo "Refusing HCC readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
+ exit 1
+}
+require_safe_kube_context
+[ "${E2E_HCC_READINESS_FAULT_INJECTION:-0}" = 1 ] || {
+ echo "Set E2E_HCC_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC fault injection." >&2
+ exit 1
+}
+command -v jq >/dev/null 2>&1 || {
+ echo "jq is required" >&2
+ exit 1
+}
+
+HCC_NS="${HCC_NS:-control-plane}"
+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
+HCC_GATEWAY_DEPLOY="${HCC_GATEWAY_DEPLOY:-host-context-controller-api-gateway}"
+HOST_NS="${MCP_HOST_NS:-mcp-host}"
+MCP_NS="${MCP_SERVER_NS:-mcp-server}"
+TOKEN_REQUEST_PATH="/api/v1/auth/mcp-host/${HOST_NS}/standalone/tokens"
+TOKEN_REQUEST_LOG_PREFIX="holding-token POST ${TOKEN_REQUEST_PATH} host="
+RUN_ID="$(date +%s)-$$"
+BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-readiness-blocker-${RUN_ID}")"
+BLOCKER_EGRESS_POLICY_NAME="$(truncate_rfc1123 "${BLOCKER_NAME}-egress")"
+SUITE_NAME="hcc-readiness-bootstrap"
+FIXTURE_HOST_PREFIX="$(truncate_rfc1123 "e2e-hcc-ready-${RUN_ID}")"
+FIXTURE_CONTEXT="$(truncate_rfc1123 "e2e-hcc-ready-context-${RUN_ID}")"
+FIXTURE_SECRET="$(truncate_rfc1123 "e2e-hcc-ready-llm-${RUN_ID}")"
+START_MARKER='Starting initial Host background convergence'
+COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'
+FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'
+ORIGINAL_CONTROL_API_BASE_URL=""
+ORIGINAL_REPLICAS=""
+HCC_PORT=""
+HOST_RECONCILE_CONCURRENCY=""
+FIXTURE_HOST_COUNT=0
+FIXTURE_HOST_NAMES=()
+HCC_MUTATED=0
+BLOCKER_CREATED=0
+FIXTURES_CREATED=0
+# Read and mutated by the sourced compare-and-swap lock helper.
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_ACQUIRED=0
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_NAME=""
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_UID=""
+# shellcheck disable=SC2034
+HCC_GATE_FINALIZATION_FAILURE=""
+
+die() {
+ if [ -n "${new_hcc_pod:-}" ]; then
+ echo "Recent HCC logs from ${new_hcc_pod}:" >&2
+ kctl logs "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller --tail=300 >&2 || true
+ fi
+ fail "$*"
+ exit 1
+}
+
+wait_until() {
+ local timeout=$1 description=$2
+ shift 2
+ local deadline now
+ deadline=$(( $(date +%s) + timeout ))
+ while :; do
+ "$@" && return 0
+ now="$(date +%s)"
+ [ "$now" -lt "$deadline" ] || break
+ sleep 1
+ done
+ echo "Timed out after ${timeout}s waiting for ${description}" >&2
+ return 1
+}
+
+running_hcc_pod() {
+ local rows
+ rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
+ --field-selector=status.phase=Running \
+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
+ 2>/dev/null)" || return 1
+ awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"
+}
+
+hcc_pods_absent() {
+ local pods
+ pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||
+ return 1
+ [ -z "$pods" ]
+}
+
+hcc_logs() {
+ local pod=$1
+ kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller 2>/dev/null
+}
+
+hcc_log_contains() {
+ local pod=$1 marker=$2 logs
+ logs="$(hcc_logs "$pod")" || return 1
+ hcc_log_snapshot_contains "$logs" "$marker"
+}
+
+blocker_holds_token_request() {
+ local logs
+ logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1
+ hcc_log_snapshot_contains "$logs" \
+ "${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-"
+}
+
+blocker_fixture_request_count() {
+ local logs
+ logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1
+ awk -v prefix="${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-" '
+ index($0, prefix) {
+ host = substr($0, index($0, "host=") + length("host="))
+ sub(/[[:space:]].*$/, "", host)
+ if (!(host in seen)) {
+ seen[host] = 1
+ count += 1
+ }
+ }
+ END { print count + 0 }
+ ' <<<"$logs"
+}
+
+blocker_observed_more_than_one_worker_wave() {
+ local count
+ count="$(blocker_fixture_request_count)" || return 1
+ [ "$count" -gt "$HOST_RECONCILE_CONCURRENCY" ]
+}
+
+initial_host_pass_is_active() {
+ local pod=$1 logs
+ logs="$(hcc_logs "$pod")" || return 1
+ hcc_initial_pass_snapshot_is_active \
+ "$logs" "$START_MARKER" "$COMPLETE_MARKER" "$FAIL_MARKER"
+}
+
+control_api_service_excludes_blocker() {
+ kctl get endpointslice -n "$HCC_NS" -l 'kubernetes.io/service-name=control-api' -o json |
+ jq -e --arg blocker "$BLOCKER_NAME" \
+ 'all(.items[]?.endpoints[]?; .targetRef.name != $blocker)' >/dev/null
+}
+
+readiness_blocker_suite_absent() {
+ local resources
+ resources="$(
+ kctl get deployment,service,networkpolicy -n "$HCC_NS" \
+ -l 'e2e.clerum.io/hcc-readiness' -o name 2>/dev/null
+ )" || return 1
+ [ -z "$resources" ]
+}
+
+probe_hcc_ready_pod() {
+ local pod
+ pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1
+ kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \
+ env "HCC_E2E_PORT=${HCC_PORT}" \
+ node -e '
+ const http = require("node:http");
+ const request = http.get(
+ {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
+ response => {
+ let body = "";
+ response.setEncoding("utf8");
+ response.on("data", chunk => { body += chunk; });
+ response.on("end", () => {
+ let parsed;
+ try { parsed = JSON.parse(body); } catch { process.exit(2); }
+ process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);
+ });
+ }
+ );
+ request.setTimeout(5000, () => request.destroy(new Error("timeout")));
+ request.once("error", () => process.exit(4));
+ ' >/dev/null 2>&1
+}
+
+running_hcc_gateway_pod() {
+ ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"
+}
+
+hcc_gateway_deployment_ready() {
+ kctl get deployment "$HCC_GATEWAY_DEPLOY" -n "$HCC_NS" -o json |
+ jq -e '
+ .spec.replicas == 1 and
+ .status.observedGeneration == .metadata.generation and
+ (.status.updatedReplicas // 0) == 1 and
+ (.status.readyReplicas // 0) == 1 and
+ (.status.availableReplicas // 0) == 1 and
+ (.status.unavailableReplicas // 0) == 0
+ ' >/dev/null
+}
+
+hcc_gateway_local_health_ok() {
+ local pod
+ pod="$(running_hcc_gateway_pod)" || return 1
+ kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
+ wget -qO- http://127.0.0.1:8081/health |
+ jq -e '.status == "ok"' >/dev/null
+}
+
+hcc_gateway_ready_proxy_unavailable() {
+ local pod output status
+ pod="$(running_hcc_gateway_pod)" || return 1
+ output="$(
+ kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
+ wget -T 70 -t 1 -O /dev/null -S http://127.0.0.1:8081/ready 2>&1
+ )" && return 1
+ status="$(
+ awk '
+ /^[[:space:]]*HTTP\/[0-9.]+[[:space:]]+[0-9][0-9][0-9]([[:space:]]|$)/ {
+ code = $2
+ }
+ END {
+ if (code != "") print code
+ }
+ ' <<<"$output"
+ )"
+ case "$status" in
+ 502|503|504) return 0 ;;
+ *) return 1 ;;
+ esac
+}
+
+hcc_gateway_remains_ready_without_hcc() {
+ local duration=$1 deadline
+ deadline=$(( $(date +%s) + duration ))
+ while [ "$(date +%s)" -lt "$deadline" ]; do
+ hcc_gateway_deployment_ready &&
+ hcc_gateway_local_health_ok &&
+ hcc_gateway_ready_proxy_unavailable &&
+ hcc_gateway_deployment_ready &&
+ hcc_gateway_local_health_ok ||
+ return 1
+ sleep 1
+ done
+}
+
+hcc_gateway_ready_proxy_recovers() {
+ local pod
+ pod="$(running_hcc_gateway_pod)" || return 1
+ kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
+ wget -qO- http://127.0.0.1:8081/ready |
+ jq -e '.ready == true and .status == "ready"' >/dev/null
+}
+
+fixture_inputs_absent() {
+ local hosts contexts secrets
+ hosts="$(kctl get host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ -o name 2>/dev/null)" || return 1
+ contexts="$(kctl get context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ -o name 2>/dev/null)" || return 1
+ secrets="$(kctl get secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ -o name 2>/dev/null)" || return 1
+ [ -z "${hosts}${contexts}${secrets}" ]
+}
+
+fixture_runtime_absent() {
+ local host resources
+ [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0
+ for host in "${FIXTURE_HOST_NAMES[@]}"; do
+ resources="$(
+ kctl get deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \
+ -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \
+ -o name 2>/dev/null
+ )" || return 1
+ [ -z "$resources" ] || return 1
+ done
+}
+
+fixture_resources_absent() {
+ fixture_inputs_absent && fixture_runtime_absent
+}
+
+hcc_restore_is_verified() {
+ local deployment
+ deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1
+ jq -e \
+ --arg controlApiBaseUrl "$ORIGINAL_CONTROL_API_BASE_URL" \
+ --argjson replicas "${ORIGINAL_REPLICAS:-1}" '
+ .spec.replicas == $replicas and
+ .status.observedGeneration == .metadata.generation and
+ (.status.updatedReplicas // 0) == $replicas and
+ (.status.readyReplicas // 0) == $replicas and
+ (.status.availableReplicas // 0) == $replicas and
+ (.status.unavailableReplicas // 0) == 0 and
+ any(.spec.template.spec.containers[]?;
+ .name == "host-context-controller" and
+ any(.env[]?;
+ .name == "CONTROL_API_BASE_URL" and .value == $controlApiBaseUrl
+ )
+ )
+ ' <<<"$deployment" >/dev/null || return 1
+ probe_hcc_ready_pod && fixture_resources_absent
+}
+
+delete_blocker_fixture() {
+ local failed=0
+ kctl delete networkpolicy "$BLOCKER_EGRESS_POLICY_NAME" "$BLOCKER_NAME" -n "$HCC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete service "$BLOCKER_NAME" -n "$HCC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete deployment "$BLOCKER_NAME" -n "$HCC_NS" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ [ "$failed" = 0 ]
+}
+
+create_host_fixtures() {
+ local index host
+ FIXTURES_CREATED=1
+
+ kctl apply -f - >/dev/null <<EOF || return 1
+apiVersion: v1
+kind: Secret
+metadata:
+ name: ${FIXTURE_SECRET}
+ namespace: ${HOST_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_ID}"
+type: Opaque
+stringData:
+ # Test-only placeholder: Host secret validation checks existence, while token
+ # issuance is intentionally held before any mcp-host Deployment is created.
+ OPENAI_API_KEY: "e2e-readiness-not-a-real-key"
+---
+apiVersion: clerum.io/v1alpha1
+kind: Context
+metadata:
+ name: ${FIXTURE_CONTEXT}
+ namespace: ${MCP_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_ID}"
+spec:
+ contextId: ${FIXTURE_CONTEXT}
+ description: Gate-owned Context for HCC Host bootstrap readiness.
+ mcpServers: []
+EOF
+
+ for ((index = 1; index <= FIXTURE_HOST_COUNT; index++)); do
+ host="$(printf '%s-%02d' "$FIXTURE_HOST_PREFIX" "$index")"
+ FIXTURE_HOST_NAMES+=("$host")
+ kctl apply -f - >/dev/null <<EOF || return 1
+apiVersion: clerum.io/v1alpha1
+kind: Host
+metadata:
+ name: ${host}
+ namespace: ${HOST_NS}
+ labels:
+ e2e.clerum.io/suite: ${SUITE_NAME}
+ e2e.clerum.io/run: "${RUN_ID}"
+spec:
+ host: ${host}
+ contextRef: ${FIXTURE_CONTEXT}
+ secretRef: ${FIXTURE_SECRET}
+ model:
+ provider: openai
+ name: e2e-readiness
+EOF
+ done
+}
+
+delete_host_fixtures() {
+ local failed=0 host
+ kctl delete host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
+ kctl delete context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ kctl delete secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
+ --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"
+ for host in "${FIXTURE_HOST_NAMES[@]}"; do
+ kctl delete \
+ deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \
+ -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \
+ --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
+ done
+ [ "$failed" = 0 ] || return 1
+ wait_until 60 "all gate-owned Host inputs and managed runtime resources to disappear" \
+ fixture_resources_absent
+}
+
+print_repair_instructions() {
+ cat >&2 <<EOF
+HCC readiness bootstrap cleanup could not restore a verified clean state.
+Context: ${E2E_KUBECONTEXT}
+HCC: ${HCC_NS}/${HCC_DEPLOY}
+Fixture Hosts: ${HOST_NS}/${FIXTURE_HOST_PREFIX}-*
+Fixture Context: ${MCP_NS}/${FIXTURE_CONTEXT}
+Fixture LLM Secret: ${HOST_NS}/${FIXTURE_SECRET}
+Retained blocker: ${HCC_NS}/${BLOCKER_NAME}
+
+Inspect before changing anything:
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HOST_NS} get host,secret -l e2e.clerum.io/suite=${SUITE_NAME}
+ kubectl --context=${E2E_KUBECONTEXT} -n ${MCP_NS} get context -l e2e.clerum.io/suite=${SUITE_NAME}
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/hcc-readiness
+
+Do not remove the blocker or HCC gate lock until HCC configuration, rollout, Pod readiness,
+/ready, and fixture absence are verified.
+
+Restore the original HCC replica count before verifying rollout and /ready:
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s
+EOF
+}
+
+cleanup() {
+ local status=$? cleanup_failed=0 restore_ok=1 hcc_stopped=1
+ trap - EXIT
+ set +e
+
+ # Remove every gate-owned input while HCC is stopped. Restarting HCC with a
+ # partially deleted fixture fleet would let the real control-api path mutate
+ # those leftovers and make cleanup non-deterministic.
+ if [ "$HCC_MUTATED" = 1 ]; then
+ hcc_stopped=0
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 \
+ >/dev/null 2>&1 || restore_ok=0
+ if [ "$restore_ok" = 1 ]; then
+ wait_until 120 "HCC pods to stop before fixture deletion" \
+ hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
+ fi
+ [ "$restore_ok" = 1 ] && hcc_stopped=1
+ fi
+
+ if [ "$FIXTURES_CREATED" = 1 ]; then
+ if [ "$hcc_stopped" = 1 ]; then
+ delete_host_fixtures >/dev/null 2>&1 || restore_ok=0
+ else
+ restore_ok=0
+ fi
+ fi
+
+ if [ "$HCC_MUTATED" = 1 ] && [ "$restore_ok" = 1 ] &&
+ [ -n "$ORIGINAL_CONTROL_API_BASE_URL" ]; then
+ kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
+ "CONTROL_API_BASE_URL=${ORIGINAL_CONTROL_API_BASE_URL}" >/dev/null 2>&1 ||
+ restore_ok=0
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 ||
+ restore_ok=0
+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s \
+ >/dev/null 2>&1 || restore_ok=0
+ if [ "$restore_ok" = 1 ]; then
+ wait_until 240 \
+ "restored HCC config, rollout, Ready pod, /ready, and fixture absence" \
+ hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0
+ fi
+ fi
+
+ if [ "$restore_ok" = 1 ]; then
+ if [ "$BLOCKER_CREATED" = 1 ]; then
+ delete_blocker_fixture || cleanup_failed=1
+ fi
+ wait_until 60 "all HCC readiness blocker resources to disappear" \
+ readiness_blocker_suite_absent >/dev/null 2>&1 || cleanup_failed=1
+ else
+ cleanup_failed=1
+ fi
+
+ if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then
+ cleanup_failed=1
+ fi
+
+ if [ "$cleanup_failed" -ne 0 ]; then
+ echo "HCC readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2
+ [ "$restore_ok" = 1 ] || print_repair_instructions
+ [ "$status" = 0 ] && status=1
+ fi
+ if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
+ header "HCC readiness bootstrap gate passed"
+ fi
+ exit "$status"
+}
+trap cleanup EXIT
+
+header "HCC readiness during initial Host fleet reconciliation"
+
+require_branch_owned_hcc_gate "$HCC_NS"
+ok "branch helper, profile, exact HEAD/fingerprint/gate verified"
+
+kctl get nodes -o json |
+ jq -e --arg context "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
+ die "target context is not a minikube cluster"
+readiness_blocker_suite_absent ||
+ die "stale HCC readiness blocker resources exist; inspect them before fault injection"
+fixture_inputs_absent ||
+ die "stale ${SUITE_NAME} Host, Context, or Secret fixtures exist; inspect them before fault injection"
+acquire_hcc_watch_gate_lock ||
+ die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
+ok "exclusive HCC fault-injection lock acquired"
+
+HCC_DEPLOYMENT_JSON="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" ||
+ die "could not read HCC Deployment"
+ORIGINAL_REPLICAS="$(jq -r '.spec.replicas // empty' <<<"$HCC_DEPLOYMENT_JSON")"
+[ "$ORIGINAL_REPLICAS" = 1 ] ||
+ die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
+ORIGINAL_CONTROL_API_BASE_URL="$(jq -r '
+ first(
+ .spec.template.spec.containers[]? |
+ select(.name == "host-context-controller") |
+ .env[]? |
+ select(.name == "CONTROL_API_BASE_URL") |
+ .value
+ ) // empty
+' <<<"$HCC_DEPLOYMENT_JSON")"
+[ -n "$ORIGINAL_CONTROL_API_BASE_URL" ] ||
+ die "HCC CONTROL_API_BASE_URL is missing"
+HCC_PORT="$(jq -r '
+ .spec.template.spec.containers[]? |
+ select(.name == "host-context-controller") as $container |
+ (
+ [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //
+ [$container.ports[]? | select(.name == "http") | .containerPort][0] //
+ empty
+ )
+' <<<"$HCC_DEPLOYMENT_JSON")"
+[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||
+ die "could not resolve a valid HCC HTTP port from the Deployment"
+HOST_RECONCILE_CONCURRENCY="$(jq -r '
+ first(
+ .spec.template.spec.containers[]? |
+ select(.name == "host-context-controller") |
+ .env[]? |
+ select(.name == "HCC_HOST_FULL_RECONCILE_CONCURRENCY") |
+ .value
+ ) // "2"
+' <<<"$HCC_DEPLOYMENT_JSON")"
+[[ "$HOST_RECONCILE_CONCURRENCY" =~ ^[1-8]$ ]] ||
+ die "HCC_HOST_FULL_RECONCILE_CONCURRENCY must be 1..8, found ${HOST_RECONCILE_CONCURRENCY}"
+FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))
+HCC_IMAGE="$(jq -r '
+ first(
+ .spec.template.spec.containers[]? |
+ select(.name == "host-context-controller") |
+ .image
+ ) // empty
+' <<<"$HCC_DEPLOYMENT_JSON")"
+[ -n "$HCC_IMAGE" ] || die "could not resolve the HCC image"
+ok "planned ${FIXTURE_HOST_COUNT} gate-owned Hosts for concurrency ${HOST_RECONCILE_CONCURRENCY}"
+
+BLOCKER_CREATED=1
+kctl apply -f - >/dev/null <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ${BLOCKER_NAME}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+ template:
+ metadata:
+ labels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+ spec:
+ automountServiceAccountToken: false
+ terminationGracePeriodSeconds: 1
+ containers:
+ - name: blocker
+ image: ${HCC_IMAGE}
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: EXPECTED_TOKEN_PATH
+ value: ${TOKEN_REQUEST_PATH}
+ command: [node, -e]
+ args:
+ - >-
+ require('http').createServer((request,response) => {
+ if(request.method!=='POST'||request.url!==process.env.EXPECTED_TOKEN_PATH){
+ console.log('rejecting '+request.method+' '+request.url);
+ request.resume();
+ request.on('end',()=>{response.statusCode=404;response.end()});
+ return;
+ }
+ let body='';
+ request.setEncoding('utf8');
+ request.on('data',chunk=>{if(body.length<65536)body+=chunk;});
+ request.on('end',()=>{
+ let host='unknown';
+ try{const parsed=JSON.parse(body);if(typeof parsed.host==='string')
+ host=parsed.host;}catch{}
+ console.log('holding-token '+request.method+' '+request.url+' host='+host);
+ });
+ }).listen(8090, '0.0.0.0');
+ readinessProbe:
+ tcpSocket: {port: 8090}
+ periodSeconds: 1
+ resources:
+ requests: {cpu: 5m, memory: 16Mi}
+ limits: {cpu: 50m, memory: 48Mi}
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities: {drop: [ALL]}
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+ runAsUser: 1000
+ runAsGroup: 1000
+ seccompProfile: {type: RuntimeDefault}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ${BLOCKER_NAME}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+spec:
+ selector:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+ ports:
+ - name: http
+ port: 8090
+ targetPort: 8090
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${BLOCKER_EGRESS_POLICY_NAME}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+spec:
+ podSelector:
+ matchLabels:
+ app: ${HCC_DEPLOY}
+ policyTypes: [Egress]
+ egress:
+ - to:
+ - podSelector:
+ matchLabels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+ ports:
+ - protocol: TCP
+ port: 8090
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${BLOCKER_NAME}
+ namespace: ${HCC_NS}
+ labels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+spec:
+ podSelector:
+ matchLabels:
+ e2e.clerum.io/hcc-readiness: "${RUN_ID}"
+ policyTypes: [Ingress]
+ ingress:
+ - from:
+ - podSelector:
+ matchLabels:
+ app: ${HCC_DEPLOY}
+ ports:
+ - protocol: TCP
+ port: 8090
+EOF
+kctl rollout status deployment "$BLOCKER_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||
+ die "isolated token-issuance blocker did not become ready"
+control_api_service_excludes_blocker ||
+ die "isolated blocker was selected by the real control-api Service"
+ok "isolated in-cluster blocker is ready"
+
+HCC_MUTATED=1
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
+wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||
+ die "existing HCC pod did not stop"
+hcc_gateway_remains_ready_without_hcc 20 ||
+ die "HCC API gateway did not remain Ready on local /health while proxied /ready was unavailable"
+ok "API gateway remained Ready for a complete probe-failure window while /ready reflected HCC unavailability"
+create_host_fixtures ||
+ die "could not create the gate-owned Host fleet while HCC was stopped"
+ok "created ${FIXTURE_HOST_COUNT} gate-owned Hosts without modifying any real Host or Secret"
+kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
+ "CONTROL_API_BASE_URL=http://${BLOCKER_NAME}.${HCC_NS}.svc.cluster.local:8090" >/dev/null
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
+
+new_hcc_pod=""
+capture_running_hcc() {
+ new_hcc_pod="$(running_hcc_pod)"
+ [ -n "$new_hcc_pod" ]
+}
+wait_until 120 "replacement HCC pod to enter Running" capture_running_hcc ||
+ die "replacement HCC pod did not start"
+wait_until 60 "initial Host reconciliation to start" \
+ hcc_log_contains "$new_hcc_pod" "$START_MARKER" ||
+ die "HCC never started the initial Host reconciliation"
+wait_until 120 "a gate-owned Host token request to reach the blocker" blocker_holds_token_request ||
+ die "initial Host reconciliation never reached the deterministic blocker"
+wait_until 180 "more than one bounded Host worker wave to reach the blocker" \
+ blocker_observed_more_than_one_worker_wave ||
+ die "fixture fleet did not prove progress beyond the first bounded Host worker wave"
+initial_host_pass_is_active "$new_hcc_pod" ||
+ die "initial Host reconciliation failed or completed before the readiness assertion"
+ok "initial Host reconciliation is active beyond one bounded worker wave"
+
+probe_script="$(cat <<'NODE'
+const http = require('http');
+function get(path) {
+ return new Promise((resolve, reject) => {
+ const request = http.get(
+ {host: '127.0.0.1', port: Number(process.env.HCC_E2E_PORT), path},
+ response => {
+ let body = '';
+ response.setEncoding('utf8');
+ response.on('data', chunk => { body += chunk; });
+ response.on('end', () => resolve({status: response.statusCode, body}));
+ }
+ );
+ request.setTimeout(5000, () => request.destroy(new Error('timeout')));
+ request.once('error', reject);
+ });
+}
+(async () => {
+ const ready = await get('/ready');
+ const readyBody = JSON.parse(ready.body);
+ if (ready.status !== 200 || readyBody.status !== 'ready' || readyBody.ready !== true) {
+ throw new Error('unexpected readiness response: ' + ready.status + ' ' + ready.body);
+ }
+ const discovery = await get('/api/v1/mcpservers');
+ if (discovery.status !== 200) {
+ throw new Error('unexpected discovery response: ' + discovery.status);
+ }
+ const discoveryBody = JSON.parse(discovery.body);
+ if (!Array.isArray(discoveryBody.servers) || discoveryBody.contextRef !== '*') {
+ throw new Error('discovery response does not match the live API contract');
+ }
+ console.log(JSON.stringify({readyStatus: ready.status, discoveryStatus: discovery.status,
+ discoveredServers: discoveryBody.servers.length}));
+})().catch(error => { console.error(error.message); process.exit(1); });
+NODE
+)"
+probe_result="$(kctl exec "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller -- \
+ env "HCC_E2E_PORT=${HCC_PORT}" node -e "$probe_script")" ||
+ die "real HCC readiness/discovery HTTP probe failed while the fleet pass was active"
+echo "$probe_result" | jq -e \
+ '.readyStatus == 200 and .discoveryStatus == 200 and (.discoveredServers | type == "number")' \
+ >/dev/null ||
+ die "HCC HTTP probe returned an invalid result: ${probe_result}"
+hcc_gateway_ready_proxy_recovers ||
+ die "HCC API gateway /ready did not recover after HCC became ready"
+initial_host_pass_is_active "$new_hcc_pod" ||
+ die "initial Host reconciliation failed or completed before the HTTP assertions finished"
+ok "/ready returned 200 while the initial Host fleet pass remained active"
+ok "API gateway /ready recovered without changing its local readiness"
+ok "/api/v1/mcpservers returned a valid live discovery response during that same pass"
+
+header "HCC readiness assertions passed; restoring branch-owned runtime"
+echo "$probe_result"
diff --git a/scripts/e2e/e2e-host-storm-gate.sh b/scripts/e2e/e2e-host-storm-gate.sh
index 095bb8c93..1015dd276 100755
--- a/scripts/e2e/e2e-host-storm-gate.sh
+++ b/scripts/e2e/e2e-host-storm-gate.sh
@@ -26,16 +26,14 @@
# construction, so A1 is primarily a RECORDING of that evidence — it
# can only fail if the harness itself degenerates into sequential
# phases; the load-bearing falsifiable claims are A2-A8.
-# A2 urgent queue-wait p95 <= 5s DURING the storm window, computed from
-# the clerum_hcc_host_reconcile_queue_wait_seconds{lane="urgent"}
-# histogram buckets (§14.1) across two counter windows: pre-kill pod
-# delta (scrape B - scrape A) plus post-kill pod absolute (scrape C1,
-# taken IMMEDIATELY after created-Host convergence so post-storm quiet
-# samples cannot dilute the p95 — a fresh process starts at zero, and
-# the C1 window includes replacement-pod recovery traffic). Samples in
-# flight between scrape B and process death are unrecoverable (counter
-# reset) and are recorded as a documented measurement gap, never
-# silently backfilled.
+# A2 urgent queue-wait coverage has two independent same-pod windows:
+# (a) the supported wake window immediately before the HCC kill requires
+# every completed urgent sample <= 5s, after proving the wake reconcile
+# finished and the urgent lane returned idle; and
+# (b) ten dedicated Host ADDED events issued only after the replacement
+# HCC completed LIST→WATCH recovery use a D - C1 p95 delta. A slow wake
+# can therefore never be diluted by later fast post-recovery probes, and
+# neither window crosses the intentional process restart.
# A3 no starvation: EVERY created Host's Deployment materializes within
# STORM_MATERIALIZATION_BUDGET_S (20s, §14.2 "New Host
# materialization") of that Host's OWN metadata.creationTimestamp,
@@ -59,13 +57,16 @@
# All-zero is legitimate (deletes fully handled by the direct
# watch-DELETE path, which by design does not touch this counter) —
# A5 remains the falsifiable cleanup proof.
-# A7 fleet coalescing held (one active + one trailing slot), from
-# clerum_hcc_host_fleet_requests_total (scrape C2): failed == 0 in
-# both windows; on the post-kill pod started >= 1 (the recovery
-# convergence pass ran), coalesced >= 1 (the recovery-triggered
-# request landed against the active initial pass), and trailing <=
-# coalesced (every trailing drain corresponds to at least one request
-# that was queued behind the single pending slot).
+# A7 recovered fleet convergence is bounded, from
+# clerum_hcc_host_fleet_requests_total (scrape C2) plus the replacement
+# pod's explicit initial-fleet COMPLETED log: failed == 0 in both windows,
+# the exact replacement pod completes its startup fleet pass, and
+# trailing <= coalesced. The pod identity, completion marker, and A4's
+# successful recovery histogram form the causal fence; started is not
+# capped because an independent request may legitimately begin after the
+# recovery pass settles. Requiring a coalesced request here would invent
+# a second overlapping signal; the concurrent-request coalescing contract
+# is covered deterministically by hostFleetScheduler.test.ts.
# A8 wake ingredient integrity: the woken Host leaves state=suspended and
# reaches a Ready pod within STORM_WAKE_READY_BUDGET_S of the wake
# POST. This is an ingredient-integrity bound (a silently no-op wake
@@ -97,23 +98,26 @@
#
# ----------------------------------------------------------------------
# INPUTS (environment; fail-loud if missing/invalid)
-# CONTEXT kubectl context. REFUSED unless it matches
-# ^clerum-(test|codex-|detached-|claude-) — a local minikube
-# profile. Any GKE / *prod* context is rejected outright.
+# KUBECONTEXT / E2E_K8S_CONTEXT
+# Exact generated branch-owned minikube context.
+# MINIKUBE_PROFILE
+# Must equal that context.
+# E2E_EXPECTED_PRE_GATE_GATE
+# Exact gate recorded by the branch-owned pre-gate sync.
+# E2E_HCC_HOST_STORM_FAULT_INJECTION=1
+# Explicit acknowledgement of the disruptive HCC pod restart.
#
# OPTIONAL INPUTS
# E2E_STORM_WAKE_HOST_REF suspended-wake target (default chatllm-stateless,
# the seeded stateless Host — it must exist with
# spec.lifecycle.stateless=true and be associated to the E2E
# user; seed via scripts/e2e/seed-stateless-host.sh).
-# EXTERNAL_REST_API_BASE_URL default http://127.0.0.1:8091
-# RPC_PROXY_BASE_URL default http://127.0.0.1:8094
# E2E_DEV_LOGIN_EMAIL default [email protected]
-# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-admin123!}
+# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-changeme123!}
# STORM_CONTEXT_REF Context CR fixtures reference (default context1)
# STORM_MODEL_PROVIDER / STORM_MODEL_NAME fixture model (default zai/glm-5.1)
-# HCC_METRICS_LOCAL_PORT ephemeral scrape port (default 18082)
-# KEEP_FIXTURES set to 1 to skip teardown (diagnosis only)
+# CLERUM_PROFILE_PORTS_ENV optional explicit path to the selected branch
+# profile's generated ports.env.
#
# Prereqs (each a HARD FAIL with the concrete reason):
# - branch HCC image deployed (clerum_hcc_host_delete_cleanup_total
@@ -122,7 +126,10 @@
# - external-rest-api + rpc-proxy reachable (make minikube-pf-all)
#
# Usage:
-# CONTEXT=clerum-test bash scripts/e2e/e2e-host-storm-gate.sh
+# MINIKUBE_PROFILE=<profile> KUBECONTEXT=<profile> \
+# E2E_EXPECTED_PRE_GATE_GATE=<gate> \
+# E2E_HCC_HOST_STORM_FAULT_INJECTION=1 \
+# bash scripts/e2e/e2e-host-storm-gate.sh
#
# EXIT CODES
# 0 every storm assertion passed and teardown left zero fixtures
@@ -131,6 +138,17 @@
# ======================================================================
set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/e2e-lib.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
+# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
+
# ─── Constants mirrored from host-context-controller/src/{constants,config}.ts ──
readonly MANAGED_BY_LABEL='clerum.io/managed-by'
readonly MANAGED_BY_VALUE='host-context-controller'
@@ -144,6 +162,10 @@ readonly CONTROL_NS='control-plane'
readonly HCC_DEPLOY='host-context-controller'
readonly HCC_PORT='8081' # containerPort http (deploy/base/control-plane)
readonly HCC_METRICS_PATH='/metrics' # server.ts GET /metrics
+readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'
+readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'
+# Shared HCC ownership/lock helpers use HCC_NS.
+HCC_NS="$CONTROL_NS"
# Owned-child kinds + namespaces of a per-Host bundle (mirrors
# measure-host-bundle-reconcile.sh — includes per-Host NetworkPolicies).
@@ -156,6 +178,7 @@ readonly BUNDLE_NAMESPACES="${MCP_HOST_NS} ${CHANNELS_NS} ${RPC_PROXY_NS}"
# (`QUEUE_WAIT_METRIC="$QUEUE_WAIT_METRIC" python3 …`) would fail as a readonly
# reassignment under `set -e` — export once here and drop the inline prefixes.
readonly QUEUE_WAIT_METRIC='clerum_hcc_host_reconcile_queue_wait_seconds'
+readonly IN_FLIGHT_METRIC='clerum_hcc_host_reconcile_in_flight'
readonly WATCH_RECOVERY_METRIC='clerum_hcc_host_watch_recovery_seconds'
readonly FLEET_METRIC='clerum_hcc_host_fleet_requests_total'
readonly DELETE_CLEANUP_METRIC='clerum_hcc_host_delete_cleanup_total'
@@ -164,11 +187,20 @@ export QUEUE_WAIT_METRIC WATCH_RECOVERY_METRIC FLEET_METRIC DELETE_CLEANUP_METRI
# ─── Storm shape + budgets (each constant states its contract) ────────
# Addendum 5.1: ">=10" created fixture Hosts, mixed stateful/stateless.
readonly STORM_CREATE_COUNT=10
+# A2's direct-watch probe is deliberately separate from the restart window:
+# a cold-start LIST legitimately converges objects on the fleet lane. These
+# ten ADDED events are emitted only once the replacement's LIST→WATCH recovery
+# has succeeded, so their queue histogram is an unambiguous urgent-lane SLO.
+readonly STORM_URGENT_PROBE_COUNT=10
# Addendum 5.1: ">=3" pre-created fixture Hosts deleted mid-storm.
readonly STORM_DELETE_COUNT=3
# §14.2 "urgent new/wake queue wait": local hard gate <= 5s (histogram
# bucket boundary le="5" exists in RECONCILE_LATENCY_BUCKETS).
readonly STORM_URGENT_P95_BUDGET_S=5
+# The queue value itself is adjudicated against the 5s hard gate. This larger
+# deadline bounds collection of the completed reconcile evidence and produces
+# a loud operational failure if the wake never settles or telemetry is absent.
+readonly STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S=60
# §14.2 "New Host materialization": Host CR -> Deployment <= 20s.
readonly STORM_MATERIALIZATION_BUDGET_S=20
# §14.2 "Watch recovery": recovery requested -> WATCH installed <= 15s
@@ -240,26 +272,44 @@ print_results() {
log "fairness and convergence under realistic concurrency, not API degradation."
}
-# ─── kubectl wrapper — EVERY call carries --context (CLAUDE.md mandate) ──
-kctl() { kubectl --context "$CONTEXT" "$@"; }
-
# ─── Portable time helper (python3; mirrors measure script) ───────────
now_ms() { python3 -c 'import time;print(int(time.time()*1000))'; }
# ─── Strict local-profile context guard (fail-loud) ───────────────────
require_storm_context() {
- [[ -n "${CONTEXT:-}" ]] || die "CONTEXT is required (a local minikube profile context)." 2
- case "$CONTEXT" in
- *gke_*|*prod*)
- die "Refusing: CONTEXT='${CONTEXT}' looks like a GKE/production context. Local profiles only." 2 ;;
- esac
- if [[ ! "$CONTEXT" =~ ^clerum-(test|codex-|detached-|claude-) ]]; then
- die "Refusing: CONTEXT='${CONTEXT}' is not an allowed local profile. Must match ^clerum-(test|codex-|detached-|claude-)." 2
- fi
+ [[ -n "${E2E_KUBECONTEXT:-}" ]] ||
+ die "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." 2
+ is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||
+ die "Refusing Host storm fault injection on non-branch context '${E2E_KUBECONTEXT}'." 2
+ require_safe_kube_context ||
+ die "context '${E2E_KUBECONTEXT}' is not an allowed local E2E context." 2
if ! kctl get nodes -o name >/dev/null 2>&1; then
- die "CONTEXT='${CONTEXT}' is not reachable (kubectl get nodes failed). Start the profile and its port-forwards." 1
+ die "context '${E2E_KUBECONTEXT}' is not reachable (kubectl get nodes failed)." 1
fi
- log "context OK: ${CONTEXT}"
+ kctl get nodes -o json | jq -e --arg context "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
+ die "target context is not the exact selected minikube profile '${E2E_KUBECONTEXT}'." 2
+ log "branch-owned minikube context OK: ${E2E_KUBECONTEXT}"
+}
+
+load_branch_profile_endpoints() {
+ local ports_env external_url rpc_url
+ ports_env="${CLERUM_PROFILE_PORTS_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/ports.env}"
+ [[ -r "$ports_env" ]] ||
+ die "branch profile ports evidence is missing at ${ports_env}; refresh the branch-profile helper." 2
+ external_url="$(profile_env_value EXTERNAL_REST_API_BASE_URL "$ports_env")"
+ [[ -n "$external_url" ]] ||
+ external_url="$(profile_env_value EXTERNAL_REST_API_URL "$ports_env")"
+ rpc_url="$(profile_env_value RPC_PROXY_BASE_URL "$ports_env")"
+ [[ -n "$rpc_url" ]] ||
+ rpc_url="$(profile_env_value RPC_PROXY_URL "$ports_env")"
+ [[ "$external_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||
+ die "branch ports.env has no valid loopback External REST URL." 2
+ [[ "$rpc_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||
+ die "branch ports.env has no valid loopback RPC proxy URL." 2
+ EXT_BASE="$external_url"
+ RPC_BASE="$rpc_url"
+ log "using branch-profile endpoints from ${ports_env}"
}
require_deps() {
@@ -295,11 +345,12 @@ RUN_ID=''; WORK_DIR=''
FIXTURE_SECRET=''
STORM_CREATED_HOSTS=()
STORM_DELETE_HOSTS=()
+STORM_URGENT_PROBE_HOSTS=()
WAKE_HOST_REF="${E2E_STORM_WAKE_HOST_REF:-chatllm-stateless}"
-EXT_BASE="${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}"
-RPC_BASE="${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}"
+EXT_BASE=''
+RPC_BASE=''
DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:[email protected]}"
-DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-admin123!}}"
+DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"
STORM_CTX_REF="${STORM_CONTEXT_REF:-context1}"
STORM_PROVIDER="${STORM_MODEL_PROVIDER:-zai}"
STORM_MODEL="${STORM_MODEL_NAME:-glm-5.1}"
@@ -311,6 +362,14 @@ T_CREATE_FIRST_MS=''; T_DELETE_MS=''; T_WAKE_MS=''; T_KILL_MS=''; T_CREATE_LAST_
PF_PID=''
HCC_ENV_SAVED=''
TEARDOWN_DONE=0
+TEARDOWN_CLEANUP_FAILED=0
+HCC_RESTORE_OK=1
+STORM_FIXTURES_CREATED=0
+# Read and mutated by the sourced compare-and-swap lock helper.
+HCC_GATE_LOCK_ACQUIRED=0
+HCC_GATE_LOCK_NAME=''
+HCC_GATE_LOCK_UID=''
+HCC_GATE_FINALIZATION_FAILURE=''
cleanup_portforward() {
if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then
@@ -340,10 +399,16 @@ restore_hcc_env() {
_hcc_restore_args
[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }
log "Restoring HCC cadence env on deployment/${HCC_DEPLOY}"
- kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1 || \
- warn "failed to restore HCC env (manual check advised)"
- kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1 || \
+ if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then
+ warn "failed to restore HCC env (manual check required)"
+ HCC_RESTORE_OK=0
+ return 1
+ fi
+ if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then
warn "HCC rollout did not settle after env restore"
+ HCC_RESTORE_OK=0
+ return 1
+ fi
HCC_ENV_SAVED=''
}
@@ -358,10 +423,12 @@ restore_hcc_env_strict() {
[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }
log "Restoring HCC cadence env on deployment/${HCC_DEPLOY} (strict)"
if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then
+ HCC_RESTORE_OK=0
fail "cadence restore FAILED — the profile is left with test cadences on deployment/${HCC_DEPLOY} (the EXIT trap retries best-effort; verify manually)"
return 0
fi
if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then
+ HCC_RESTORE_OK=0
fail "HCC rollout did not settle after the cadence restore — verify deployment/${HCC_DEPLOY} manually"
return 0
fi
@@ -371,10 +438,11 @@ restore_hcc_env_strict() {
# shellcheck disable=SC2329 # invoked via the EXIT trap below
cleanup_on_exit() {
- local status=$?
+ local status=$? cleanup_failed=0
set +e
cleanup_portforward
- if [[ "$TEARDOWN_DONE" != '1' && -n "$RUN_ID" && "${KEEP_FIXTURES:-}" != '1' ]]; then
+ if [[ "$TEARDOWN_DONE" != '1' && "$STORM_FIXTURES_CREATED" == '1' && -n "$RUN_ID" ]]; then
+ cleanup_failed=1
local sel="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"
warn "abnormal exit — best-effort deletion of storm fixtures (${sel})"
kctl -n "$MCP_HOST_NS" delete host -l "$sel" --wait=false >/dev/null 2>&1 || \
@@ -382,7 +450,10 @@ cleanup_on_exit() {
kctl -n "$MCP_HOST_NS" delete secret -l "$sel" --wait=false >/dev/null 2>&1 || \
warn "best-effort fixture Secret deletion failed (manual check advised)"
fi
- restore_hcc_env
+ restore_hcc_env || HCC_RESTORE_OK=0
+ if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$HCC_RESTORE_OK"; then
+ status=1
+ fi
exit "$status"
}
trap cleanup_on_exit EXIT
@@ -405,6 +476,9 @@ init_run() {
for i in $(seq 1 "$STORM_DELETE_COUNT"); do
STORM_DELETE_HOSTS+=("$(printf 's791-%s-d%d' "$RUN_ID" "$i")")
done
+ for i in $(seq 1 "$STORM_URGENT_PROBE_COUNT"); do
+ STORM_URGENT_PROBE_HOSTS+=("$(printf 's791-%s-u%02d' "$RUN_ID" "$i")")
+ done
log "run-id ${RUN_ID}; work dir ${WORK_DIR}"
log "create fixtures: ${STORM_CREATED_HOSTS[*]}"
log "delete fixtures: ${STORM_DELETE_HOSTS[*]}"
@@ -531,14 +605,30 @@ storm_diagnostics() {
# different pod (e.g. a terminating one during a rollout) than the one the
# identity guard checked; cross-pod counter deltas would false-PASS A4.
scrape_hcc_metrics() {
- local out="$1" pod="$2" lport="${HCC_METRICS_LOCAL_PORT:-18082}"
- local url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"
- kctl -n "$CONTROL_NS" port-forward "pod/${pod}" "${lport}:${HCC_PORT}" >"${WORK_DIR}/pf.log" 2>&1 &
+ local out="$1" pod="$2" lport='' url pf_log
+ pf_log="${WORK_DIR}/pf-${pod}.log"
+ # Let kubectl allocate a process-owned loopback port. A fixed port is
+ # machine-global and can silently collide with another branch/profile gate.
+ kctl -n "$CONTROL_NS" port-forward --address=127.0.0.1 \
+ "pod/${pod}" ":${HCC_PORT}" >"$pf_log" 2>&1 &
PF_PID=$!
# shellcheck disable=SC2329 # invoked by name via wait_until
+ _pf_ready_probe() {
+ kill -0 "$PF_PID" 2>/dev/null || return 1
+ lport="$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> .*/\1/p' "$pf_log" | head -n1)"
+ [[ "$lport" =~ ^[0-9]+$ ]]
+ }
+ # shellcheck disable=SC2329 # invoked by name via wait_until
+ _pf_diag() { { echo '----- port-forward log -----'; cat "$pf_log" 2>/dev/null; } >&2; }
+ if ! wait_until 30 1 "random HCC metrics port-forward" _pf_ready_probe _pf_diag; then
+ cleanup_portforward
+ return 1
+ fi
+ url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"
+ # shellcheck disable=SC2329 # invoked by name via wait_until
_scrape_probe() { curl -fsS --max-time 3 "$url" -o "$out" 2>/dev/null; }
# shellcheck disable=SC2329 # invoked by name via wait_until
- _scrape_diag() { { echo '----- port-forward log -----'; cat "${WORK_DIR}/pf.log" 2>/dev/null; } >&2; }
+ _scrape_diag() { _pf_diag; }
if ! wait_until 30 2 "HCC metrics scrape -> ${out}" _scrape_probe _scrape_diag; then
cleanup_portforward
return 1
@@ -551,6 +641,23 @@ wake_lifecycle_state() {
kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true
}
+# shellcheck disable=SC2329 # invoked from the wait_until callback below
+wake_handled_generation() {
+ kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" \
+ -o jsonpath='{.status.lifecycle.wakeHandledGeneration}' 2>/dev/null || true
+}
+
+# Correlate the public wake request with HCC's durable lifecycle status. This
+# predicate is deliberately pure so the static gate test can exercise the
+# same ordering contract used by the cluster E2E.
+# shellcheck disable=SC2329 # invoked from the wait_until callback below
+wake_generation_is_handled() {
+ local requested="$1" handled="$2"
+ [[ "$requested" =~ ^[1-9][0-9]*$ ]] || return 1
+ [[ "$handled" =~ ^[0-9]+$ ]] || return 1
+ (( handled >= requested ))
+}
+
# shellcheck disable=SC2329 # invoked from _wake_converged_probe (runs via wait_until)
wake_ready_pod_exists() {
kctl -n "$MCP_HOST_NS" get pods -l "app=${WAKE_HOST_REF}" \
@@ -589,19 +696,31 @@ mint_rpc_token() {
# The wake route is the user-facing rpc-proxy endpoint Desktop uses;
# rpc-proxy owns the control-api hop. Never kubectl scale.
-WAKE_STATUS=''; WAKE_BODY=''
+WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''
post_wake() {
- local resp
+ local resp response_status response_generation
+ WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''
mint_rpc_token || return 1
resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST \
"${RPC_BASE}/api/v1/rpc/hosts/${WAKE_HOST_REF}/wake" \
-H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json') || {
echo "wake POST failed at ${RPC_BASE}" >&2; return 1; }
WAKE_STATUS="$(echo "$resp" | tail -n1)"; WAKE_BODY="$(echo "$resp" | sed '$d')"
- case "$WAKE_STATUS" in
- 200|202|204) return 0 ;;
- *) echo "wake POST returned HTTP ${WAKE_STATUS}: ${WAKE_BODY}" >&2; return 1 ;;
- esac
+ [[ "$WAKE_STATUS" == '202' ]] || {
+ echo "wake POST returned HTTP ${WAKE_STATUS}, expected 202: ${WAKE_BODY}" >&2
+ return 1
+ }
+ response_status="$(jq -r '.status // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"
+ response_generation="$(jq -r '.wakeGeneration // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"
+ [[ "$response_status" == 'wake-requested' ]] || {
+ echo "wake POST returned unexpected status '${response_status:-<missing>}': ${WAKE_BODY}" >&2
+ return 1
+ }
+ [[ "$response_generation" =~ ^[1-9][0-9]*$ ]] || {
+ echo "wake POST returned invalid wakeGeneration '${response_generation:-<missing>}': ${WAKE_BODY}" >&2
+ return 1
+ }
+ WAKE_GENERATION="$response_generation"
}
# ─── HCC cadence shortening (only when the wake target must suspend) ──
@@ -654,8 +773,15 @@ _wake_target_diag() {
require_deps
require_storm_context
init_run
+[[ "${KEEP_FIXTURES:-0}" != '1' ]] ||
+ die "KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result; refusing before mutation." 2
+[[ "${E2E_HCC_HOST_STORM_FAULT_INJECTION:-0}" == '1' ]] ||
+ die "Set E2E_HCC_HOST_STORM_FAULT_INJECTION=1 to acknowledge the temporary HCC pod restart." 2
+require_branch_owned_hcc_gate "$CONTROL_NS"
+load_branch_profile_endpoints
header "Prerequisites"
+ok "branch helper, profile, exact HEAD/fingerprint/gate, and minikube node label verified"
if kctl get deployment "$HCC_DEPLOY" -n "$CONTROL_NS" >/dev/null 2>&1; then
ok "HCC deployment present in ${CONTROL_NS}"
else
@@ -695,12 +821,18 @@ else
fail "cannot mint RPC token for hostRef=${WAKE_HOST_REF} -- is ${DEV_EMAIL} associated to it? Seed: bash scripts/e2e/seed-stateless-host.sh"
print_results; exit 1
fi
+if acquire_hcc_watch_gate_lock; then
+ ok "exclusive shared HCC fault-injection lock acquired"
+else
+ die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
+fi
# ======================================================================
# Prep 1 — storm fixtures that will be DELETED mid-storm (bundles must
# exist first: deleting a bundle-less Host proves nothing)
# ======================================================================
header "Prep — pre-create ${STORM_DELETE_COUNT} delete-target fixture Hosts"
+STORM_FIXTURES_CREATED=1
ensure_fixture_secret
for h in "${STORM_DELETE_HOSTS[@]}"; do
apply_storm_host "$h" false
@@ -748,6 +880,9 @@ SCRAPE_A="${WORK_DIR}/metrics-A.prom"
SCRAPE_B="${WORK_DIR}/metrics-B.prom"
SCRAPE_C1="${WORK_DIR}/metrics-C1.prom"
SCRAPE_C2="${WORK_DIR}/metrics-C2.prom"
+SCRAPE_D="${WORK_DIR}/metrics-D.prom"
+SCRAPE_WAKE_PRE="${WORK_DIR}/metrics-wake-pre.prom"
+SCRAPE_WAKE_POST="${WORK_DIR}/metrics-wake-post.prom"
HCC_POD_A="$(hcc_pod_name)"
if [[ -z "$HCC_POD_A" ]]; then
fail "no Running HCC pod found for baseline identity"; storm_diagnostics; print_results; exit 1
@@ -781,22 +916,89 @@ for h in "${STORM_DELETE_HOSTS[@]}"; do
done
log "storm: ${STORM_DELETE_COUNT} mid-storm Host deletions issued"
+# Isolate the supported wake on the pre-kill pod. Samples from the first half
+# of Host creation may still settle inside this window; they are valid urgent
+# traffic and the hard-gate assertion below requires every one of them, not
+# only a percentile, to remain within budget.
+HCC_POD_WAKE="$(hcc_pod_name)"
+if [[ -z "$HCC_POD_WAKE" || "$HCC_POD_WAKE" != "$HCC_POD_A" ]]; then
+ fail "storm: HCC pod changed before the wake metric baseline (${HCC_POD_A} -> ${HCC_POD_WAKE:-<none>})"
+ storm_diagnostics; print_results; exit 1
+fi
+if ! scrape_hcc_metrics "$SCRAPE_WAKE_PRE" "$HCC_POD_WAKE"; then
+ fail "storm: could not capture the same-pod wake metric baseline"
+ print_results; exit 1
+fi
+
T_WAKE_MS="$(now_ms)"
if post_wake; then
- log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS})"
+ log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS}, generation ${WAKE_GENERATION})"
else
fail "storm: mid-storm wake POST failed for ${WAKE_HOST_REF}"; print_results; exit 1
fi
-# Scrape B: last look at the pre-kill pod's counters, immediately before
-# the kill (samples still in flight at process death are a recorded gap).
+# Prove that this specific supported wake reached HCC: its response generation
+# must be durably reflected in lifecycle status. Then require a new urgent
+# queue sample and an idle urgent lane on the same pod. Prometheus collectors
+# are read independently, so this deliberately uses a two-scrape fence instead
+# of assuming cross-metric atomicity: once the first scrape observes the urgent
+# lane idle, HCC's observe-before-in-flight-decrement ordering guarantees the
+# wake observation is durable and therefore visible to the second scrape.
+# shellcheck disable=SC2329 # invoked by name via wait_until
+_wake_urgent_evidence_recorded() {
+ local current_pod lifecycle_state handled_generation
+ local fence_in_flight pre_count post_count in_flight
+ current_pod="$(hcc_pod_name)"
+ [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_WAKE" ]] || return 1
+ lifecycle_state="$(wake_lifecycle_state)"
+ [[ -n "$lifecycle_state" && "$lifecycle_state" != 'suspended' ]] || return 1
+ handled_generation="$(wake_handled_generation)"
+ wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation" || return 1
+ scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1
+ fence_in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '
+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }
+ END { print total + 0 }
+ ' "$SCRAPE_WAKE_POST")"
+ awk -v active="$fence_in_flight" 'BEGIN { exit !(active == 0) }' || return 1
+
+ # The wake histogram observation precedes the decrement seen by the fence.
+ # Re-scrape so the adjudicated histogram cannot predate that observation.
+ scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1
+ pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
+ END { print total + 0 }
+ ' "$SCRAPE_WAKE_PRE")"
+ post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
+ END { print total + 0 }
+ ' "$SCRAPE_WAKE_POST")"
+ in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '
+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }
+ END { print total + 0 }
+ ' "$SCRAPE_WAKE_POST")"
+ awk -v before="$pre_count" -v after="$post_count" -v active="$in_flight" \
+ 'BEGIN { exit !((after - before) >= 1 && active == 0) }'
+}
+if wait_until "$STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S" 1 \
+ "supported wake urgent reconcile + same-pod queue evidence" \
+ _wake_urgent_evidence_recorded _wake_target_diag; then
+ ok "A2 wake input: supported wake reconcile completed and published same-pod urgent queue evidence"
+else
+ fail "A2 wake input: wake did not complete with durable same-pod urgent queue evidence within ${STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S}s"
+ print_results; exit 1
+fi
+
+# Capture A7's fleet-failure baseline immediately before the kill. The wake
+# scrape remains the A2 wake-window oracle; a separate fresh scrape prevents
+# fleet failures after wake adjudication from falling outside A7's delta.
HCC_POD_B="$(hcc_pod_name)"
if [[ "$HCC_POD_B" != "$HCC_POD_A" ]]; then
fail "storm: HCC pod changed unexpectedly before the kill (${HCC_POD_A} -> ${HCC_POD_B:-<none>}) — counter deltas would be invalid"
storm_diagnostics; print_results; exit 1
fi
if ! scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"; then
- fail "storm: could not scrape HCC metrics immediately before the kill"; print_results; exit 1
+ fail "storm: could not capture the fresh pre-kill metric baseline"
+ print_results; exit 1
fi
log "storm: pre-kill scrape B captured (pod ${HCC_POD_B})"
@@ -848,6 +1050,41 @@ else
print_results; exit 1
fi
+# A7's causal business signal is the replacement pod's own completion log for
+# its startup fleet reconciliation. A scheduled/started counter is insufficient:
+# it can increment before the pass fails, stalls, or is superseded.
+FLEET_COMPLETION_PROVEN=0
+replacement_fleet_state=''
+# shellcheck disable=SC2329 # invoked by name via wait_until
+_replacement_fleet_completed_probe() {
+ local logs current_pod
+ current_pod="$(hcc_pod_name)"
+ [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1
+ logs="$(kctl -n "$CONTROL_NS" logs "pod/${HCC_POD_C}" \
+ -c host-context-controller 2>/dev/null)" || return 1
+ if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then
+ replacement_fleet_state='failed'
+ return 0
+ fi
+ if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then
+ replacement_fleet_state='completed'
+ return 0
+ fi
+ return 1
+}
+if ! wait_until "$STORM_CONVERGENCE_DEADLINE_S" 2 \
+ "replacement HCC initial Host fleet reconciliation completion" \
+ _replacement_fleet_completed_probe storm_diagnostics; then
+ fail "A7: replacement pod did not complete its initial Host fleet reconciliation within ${STORM_CONVERGENCE_DEADLINE_S}s"
+ print_results; exit 1
+fi
+if [[ "$replacement_fleet_state" != 'completed' ]]; then
+ fail "A7: replacement pod's initial Host fleet reconciliation failed"
+ storm_diagnostics; print_results; exit 1
+fi
+FLEET_COMPLETION_PROVEN=1
+ok "A7 input: replacement pod logged completed initial Host fleet reconciliation"
+
# shellcheck disable=SC2329 # invoked by name via wait_until
_all_created_deploys_probe() {
local h
@@ -880,6 +1117,68 @@ if ! scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"; then
fi
log "scrape C1 captured (replacement pod ${HCC_POD_C})"
+# ─── A2 input: direct-watch urgent probe after replacement LIST→WATCH ──
+# The storm's second half is intentionally issued immediately after pod
+# deletion, when a cold-start LIST may legitimately own those Hosts on the
+# fleet lane. C1 proves the replacement has completed LIST→WATCH recovery;
+# issue a fresh, independent set now so every observed sample is a direct
+# watch ADDED admission on the urgent lane.
+header "A2 input — urgent direct-watch probe (${STORM_URGENT_PROBE_COUNT} Hosts)"
+for i in $(seq 1 "${#STORM_URGENT_PROBE_HOSTS[@]}"); do
+ h="${STORM_URGENT_PROBE_HOSTS[$((i - 1))]}"
+ if (( i % 2 == 0 )); then apply_storm_host "$h" true; else apply_storm_host "$h" false; fi
+done
+log "urgent probe: ${STORM_URGENT_PROBE_COUNT} Host ADDED events issued after replacement LIST→WATCH"
+
+# shellcheck disable=SC2329 # invoked by name via wait_until
+_urgent_probe_deploys_probe() {
+ local h
+ for h in "${STORM_URGENT_PROBE_HOSTS[@]}"; do
+ kctl -n "$MCP_HOST_NS" get deployment "$h" >/dev/null 2>&1 || return 1
+ done
+}
+if wait_until "$STORM_CONVERGENCE_DEADLINE_S" 3 \
+ "all ${STORM_URGENT_PROBE_COUNT} urgent-probe Hosts' Deployments to exist" \
+ _urgent_probe_deploys_probe storm_diagnostics; then
+ ok "A2 input: every direct-watch urgent-probe Host materialized"
+else
+ fail "A2 input: urgent-probe Host materialization incomplete after ${STORM_CONVERGENCE_DEADLINE_S}s"
+ print_results; exit 1
+fi
+HCC_POD_C_NOW="$(hcc_pod_name)"
+if [[ -z "$HCC_POD_C_NOW" || "$HCC_POD_C_NOW" != "$HCC_POD_C" ]]; then
+ fail "replacement HCC pod changed before urgent-probe scrape D (${HCC_POD_C} -> ${HCC_POD_C_NOW:-<none>}) — urgent counter window is invalid"
+ storm_diagnostics; print_results; exit 1
+fi
+
+# Deployment existence can precede the end of reconcileCore, where the queue
+# histogram is observed. Poll the metric itself instead of sleeping, otherwise
+# a fast API server can make this gate race its own telemetry.
+# shellcheck disable=SC2329 # invoked by name via wait_until
+_urgent_probe_metrics_recorded() {
+ local current_pod pre_count post_count
+ current_pod="$(hcc_pod_name)"
+ [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1
+ scrape_hcc_metrics "$SCRAPE_D" "$HCC_POD_C" || return 1
+ pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
+ END { print total + 0 }
+ ' "$SCRAPE_C1")"
+ post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
+ index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
+ END { print total + 0 }
+ ' "$SCRAPE_D")"
+ awk -v before="$pre_count" -v after="$post_count" -v required="$STORM_URGENT_PROBE_COUNT" \
+ 'BEGIN { exit !((after - before) >= required) }'
+}
+if ! wait_until 60 2 \
+ "at least ${STORM_URGENT_PROBE_COUNT} direct-watch urgent metric samples" \
+ _urgent_probe_metrics_recorded storm_diagnostics; then
+ fail "urgent-probe reconciles did not publish ${STORM_URGENT_PROBE_COUNT} queue samples within 60s"
+ print_results; exit 1
+fi
+log "scrape D captured (replacement pod ${HCC_POD_C}; direct-watch urgent window C1→D)"
+
# ─── A3: per-host materialization <= 20s from ITS OWN creation ───────
header "A3 — per-host materialization (budget ${STORM_MATERIALIZATION_BUDGET_S}s, §14.2)"
MAT_ROWS="${WORK_DIR}/materialization.rows"
@@ -991,13 +1290,14 @@ if ! scrape_hcc_metrics "$SCRAPE_C2" "$HCC_POD_C"; then
fi
log "scrape C2 captured (replacement pod ${HCC_POD_C}; A6/A7 counters settled post-A5)"
-# One python pass performs the four metric assertions and emits one
+# One python pass performs the five metric assertions and emits one
# VERDICT|<name>|PASS/FAIL|<detail> line each. python exits nonzero only
# when it cannot parse its inputs (a hard failure of the gate itself).
METRIC_VERDICTS="${WORK_DIR}/metric-verdicts.txt"
-if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" \
+if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" SCRAPE_D="$SCRAPE_D" \
+ SCRAPE_WAKE_PRE="$SCRAPE_WAKE_PRE" SCRAPE_WAKE_POST="$SCRAPE_WAKE_POST" \
P95_BUDGET_S="$STORM_URGENT_P95_BUDGET_S" RECOVERY_BUDGET_S="$STORM_WATCH_RECOVERY_BUDGET_S" \
- MIN_URGENT_SAMPLES="$STORM_CREATE_COUNT" \
+ MIN_URGENT_SAMPLES="$STORM_URGENT_PROBE_COUNT" FLEET_COMPLETION_PROVEN="$FLEET_COMPLETION_PROVEN" \
python3 >"$METRIC_VERDICTS" <<'PY'
import os, re, sys
from collections import defaultdict
@@ -1033,6 +1333,9 @@ a, _ = load(os.environ["SCRAPE_A"])
b, _ = load(os.environ["SCRAPE_B"])
c1, _ = load(os.environ["SCRAPE_C1"])
c2, c2_types = load(os.environ["SCRAPE_C2"])
+d, _ = load(os.environ["SCRAPE_D"])
+wake_pre, _ = load(os.environ["SCRAPE_WAKE_PRE"])
+wake_post, _ = load(os.environ["SCRAPE_WAKE_POST"])
def buckets(samples, metric, match):
"""Cumulative histogram buckets {le: count} summed over matching series."""
@@ -1057,30 +1360,60 @@ def le_key(le):
verdicts = []
-# ── A2: urgent queue-wait p95 <= budget across (B-A) + C1 ─────────────
+# ── A2a: every completed urgent reconcile in the supported wake window ─
qm = os.environ["QUEUE_WAIT_METRIC"]
budget = float(os.environ["P95_BUDGET_S"])
-window_note = "windows: pre-kill delta (B-A) + replacement pod C1, which includes replacement-pod recovery traffic"
-pre_a = buckets(a, qm, {"lane": "urgent"})
-pre_b = buckets(b, qm, {"lane": "urgent"})
-post = buckets(c1, qm, {"lane": "urgent"})
+wake_before = buckets(wake_pre, qm, {"lane": "urgent"})
+wake_after = buckets(wake_post, qm, {"lane": "urgent"})
+wake_delta = defaultdict(float)
+wake_reset = False
+for le in set(wake_before) | set(wake_after):
+ delta = wake_after.get(le, 0.0) - wake_before.get(le, 0.0)
+ if delta < 0:
+ wake_reset = True
+ wake_delta[le] = delta
+if wake_reset:
+ verdicts.append(("A2-wake-urgent-hard-gate", False,
+ "negative bucket delta in the same-pod wake window"))
+else:
+ wake_total = wake_delta.get("+Inf", 0.0)
+ wake_within = wake_delta.get(str(int(budget)), wake_delta.get(str(budget), 0.0))
+ wake_dist = " ".join(
+ f"le{le}={wake_delta[le]:.0f}" for le in sorted(wake_delta, key=le_key)
+ )
+ if wake_total < 1:
+ verdicts.append(("A2-wake-urgent-hard-gate", False,
+ "no completed urgent queue sample in the supported wake window"))
+ elif wake_within != wake_total:
+ verdicts.append(("A2-wake-urgent-hard-gate", False,
+ f"{wake_total - wake_within:.0f} of {wake_total:.0f} urgent sample(s) exceeded "
+ f"{budget:.0f}s in the supported wake window ({wake_dist})"))
+ else:
+ verdicts.append(("A2-wake-urgent-hard-gate", True,
+ f"all {wake_total:.0f} urgent sample(s) <= {budget:.0f}s in the supported "
+ f"same-pod wake window ({wake_dist})"))
+
+# ── A2b: urgent queue-wait p95 across direct-watch window D-C1 ────────
+window_note = "window: replacement-pod direct-watch probe delta (D-C1) after LIST→WATCH recovery"
+pre = buckets(c1, qm, {"lane": "urgent"})
+post = buckets(d, qm, {"lane": "urgent"})
combined = defaultdict(float)
reset = False
-for le in set(pre_a) | set(pre_b) | set(post):
- delta = pre_b.get(le, 0.0) - pre_a.get(le, 0.0)
+for le in set(pre) | set(post):
+ delta = post.get(le, 0.0) - pre.get(le, 0.0)
if delta < 0:
reset = True
- combined[le] = delta + post.get(le, 0.0)
+ combined[le] = delta
if reset:
- verdicts.append(("A2-urgent-queue-wait-p95", False,
- "negative bucket delta between scrapes A and B (unexpected counter reset)"))
+ verdicts.append(("A2-direct-watch-urgent-p95", False,
+ "negative bucket delta between scrapes C1 and D (unexpected counter reset)"))
else:
total = combined.get("+Inf", 0.0)
min_samples = float(os.environ["MIN_URGENT_SAMPLES"])
if total < min_samples:
- verdicts.append(("A2-urgent-queue-wait-p95", False,
- f"only {total:.0f} urgent samples in the storm window (need >= {min_samples:.0f} — "
- "the storm did not exercise the urgent lane, cannot claim a p95)"))
+ verdicts.append(("A2-direct-watch-urgent-p95", False,
+ f"only {total:.0f} direct-watch urgent samples (need >= {min_samples:.0f} — "
+ "the urgent probe did not exercise its declared lane)"))
else:
p95_ub = None
for le in sorted(combined, key=le_key):
@@ -1089,11 +1422,10 @@ else:
break
dist = " ".join(f"le{le}={combined[le]:.0f}" for le in sorted(combined, key=le_key))
if p95_ub is not None and le_key(p95_ub) <= budget:
- verdicts.append(("A2-urgent-queue-wait-p95", True,
- f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note}; "
- "gap: samples in flight between scrape B and process death are unrecoverable)"))
+ verdicts.append(("A2-direct-watch-urgent-p95", True,
+ f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note})"))
else:
- verdicts.append(("A2-urgent-queue-wait-p95", False,
+ verdicts.append(("A2-direct-watch-urgent-p95", False,
f"p95 bucket upper bound {p95_ub} exceeds {budget:.0f}s over {total:.0f} samples ({dist}; {window_note})"))
# ── A4: watch recovery <= budget on the replacement pod (C1) ──────────
@@ -1117,28 +1449,36 @@ else:
verdicts.append(("A4-watch-recovery", True,
f"{succ_total:.0f} recovery sample(s), all <= {bound}s (budget {rbudget}s)"))
-# ── A7: fleet coalescing held (settled window C2) ─────────────────────
+# ── A7: recovered fleet convergence remains bounded (settled C2) ─────
fm = os.environ["FLEET_METRIC"]
-pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")
-post_started = counter(c2, fm, "result", "started")
-post_coalesced = counter(c2, fm, "result", "coalesced")
-post_trailing = counter(c2, fm, "result", "trailing")
-post_failed = counter(c2, fm, "result", "failed")
-problems = []
-if pre_failed > 0 or post_failed > 0:
- problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")
-if post_started < 1:
- problems.append("no fleet pass started on the replacement pod (recovery convergence missing)")
-if post_coalesced < 1:
- problems.append("no coalesced fleet request on the replacement pod (recovery request should land against the active initial pass)")
-if post_trailing > post_coalesced:
- problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")
-detail = (f"started={post_started:.0f} coalesced={post_coalesced:.0f} "
- f"trailing={post_trailing:.0f} failed={post_failed:.0f} (pre-kill failed delta={pre_failed:.0f})")
-if problems:
- verdicts.append(("A7-fleet-coalescing", False, "; ".join(problems) + f" [{detail}]"))
+# Presence check (A6 pattern): counter() returns 0 for an absent family, so
+# without this a renamed/unregistered FLEET_METRIC makes every count 0 and A7
+# passes vacuously. The causal leg (FLEET_COMPLETION_PROVEN) alone cannot catch
+# a metric rename — the metric itself must be registered on the running image.
+if fm not in c2_types:
+ verdicts.append(("A7-fleet-recovery-bounded", False,
+ f"{fm} not registered on the running image — is the branch HCC image deployed?"))
else:
- verdicts.append(("A7-fleet-coalescing", True, detail))
+ pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")
+ post_started = counter(c2, fm, "result", "started")
+ post_coalesced = counter(c2, fm, "result", "coalesced")
+ post_trailing = counter(c2, fm, "result", "trailing")
+ post_failed = counter(c2, fm, "result", "failed")
+ problems = []
+ if pre_failed != 0 or post_failed > 0:
+ problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")
+ if os.environ.get("FLEET_COMPLETION_PROVEN") != "1":
+ problems.append("replacement-pod initial fleet completion was not causally proven")
+ if post_trailing > post_coalesced:
+ problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")
+ detail = (f"completion-proven={os.environ.get('FLEET_COMPLETION_PROVEN', 'missing')} "
+ f"started={post_started:.0f} coalesced={post_coalesced:.0f} "
+ f"trailing={post_trailing:.0f} failed={post_failed:.0f} "
+ f"(replacement pod; A4 recovery-success fenced; pre-kill failed delta={pre_failed:.0f})")
+ if problems:
+ verdicts.append(("A7-fleet-recovery-bounded", False, "; ".join(problems) + f" [{detail}]"))
+ else:
+ verdicts.append(("A7-fleet-recovery-bounded", True, detail))
# ── A6: delete-cleanup counters consistent (settled window C2) ────────
dm = os.environ["DELETE_CLEANUP_METRIC"]
@@ -1178,8 +1518,8 @@ while IFS='|' read -r tag name verdict detail; do
fail "${name}: ${detail}"
fi
done <"$METRIC_VERDICTS"
-if [[ "$VERDICT_COUNT" -ne 4 ]]; then
- fail "metric adjudication emitted ${VERDICT_COUNT}/4 verdicts — truncated output is a failure, not a pass"
+if [[ "$VERDICT_COUNT" -ne 5 ]]; then
+ fail "metric adjudication emitted ${VERDICT_COUNT}/5 verdicts — truncated output is a failure, not a pass"
fi
# ======================================================================
@@ -1187,11 +1527,7 @@ fi
# The profile and seeded data (including the wake target) stay.
# ======================================================================
header "Teardown — storm fixtures (${FIXTURE_RUN_KEY}=${RUN_ID})"
-if [[ "${KEEP_FIXTURES:-}" == '1' ]]; then
- warn "KEEP_FIXTURES=1 — leaving fixtures in place for diagnosis (run-id ${RUN_ID})"
- TEARDOWN_DONE=1
-else
- STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"
+STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"
kctl -n "$MCP_HOST_NS" delete host -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \
warn "teardown: label-selector Host deletion returned nonzero (leftover check below is authoritative)"
kctl -n "$MCP_HOST_NS" delete secret -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \
@@ -1214,6 +1550,7 @@ else
ok "teardown: all storm fixture Hosts deleted via the supported CR path"
else
fail "teardown: storm fixture Hosts still present after ${STORM_TEARDOWN_DEADLINE_S}s (run-id ${RUN_ID})"
+ TEARDOWN_CLEANUP_FAILED=1
fi
# FAIL-CLOSED (B2): an unverifiable read counts as a leftover — the teardown
@@ -1227,7 +1564,7 @@ else
teardown_converge_deadline=$((SECONDS + STORM_TEARDOWN_CONVERGE_S))
while :; do
TEARDOWN_LEFTOVER=0; teardown_orphan=0; teardown_unverifiable=0; teardown_report=''
- for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; do
+ for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_URGENT_PROBE_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; do
if ! leftovers="$(collect_children "$h")"; then
teardown_report+=" ${h}: kubectl read FAILED (unverifiable, fail-closed)"$'\n'
TEARDOWN_LEFTOVER=$((TEARDOWN_LEFTOVER + 1)); teardown_unverifiable=1
@@ -1280,14 +1617,19 @@ else
ok "teardown: zero storm-fixture leftovers across ${BUNDLE_NAMESPACES} (profile and seeded data untouched)"
else
fail "teardown: ${TEARDOWN_LEFTOVER} storm fixture subject(s) left resources behind (run-id ${RUN_ID}) — investigate before trusting results"
+ TEARDOWN_CLEANUP_FAILED=1
fi
- TEARDOWN_DONE=1
-fi
+TEARDOWN_DONE=1
# M3: on the success path a failed cadence restore must FAIL the gate
# (restore_hcc_env_strict registers fail() and leaves HCC_ENV_SAVED for
# the trap's best-effort retry); the EXIT trap keeps the lenient restore.
restore_hcc_env_strict
+if ! finalize_hcc_watch_gate_lock "$TEARDOWN_CLEANUP_FAILED" "$HCC_RESTORE_OK"; then
+ fail "shared HCC fault-injection lock was retained fail-closed; follow the printed repair instructions"
+else
+ ok "exclusive shared HCC fault-injection lock finalized"
+fi
print_results
if [[ "$g_fail" -gt 0 ]]; then
diff --git a/scripts/e2e/e2e-wrc-hcc-contracts.sh b/scripts/e2e/e2e-wrc-hcc-contracts.sh
index e0b4c5029..570f0f923 100755
--- a/scripts/e2e/e2e-wrc-hcc-contracts.sh
+++ b/scripts/e2e/e2e-wrc-hcc-contracts.sh
@@ -271,6 +271,8 @@ spec:
agent:
provider: zai
model: glm-4.7
+ triggers:
+ onDemand: {}
steps:
- id: tool-step
instruction: "Synthetic status-contract step for toolsCalled args validation."
@@ -296,7 +298,7 @@ else
fi
rm -f "$running_response_file"
-if running_status_json="$(wait_for_running_workflowrecipe_status)"; then
+if wait_for_running_workflowrecipe_status >/dev/null; then
ok "WorkflowRecipe becomes active when workflow execution is running"
else
fail "WorkflowRecipe did not become active on running workflow execution"
@@ -427,7 +429,7 @@ else
fi
rm -f "$pending_response_file"
-if pending_status_json="$(wait_for_pending_preserves_non_terminal_recipe_status)"; then
+if wait_for_pending_preserves_non_terminal_recipe_status >/dev/null; then
ok "WorkflowRecipe stays non-terminal when workflow execution is pending"
else
fail "WorkflowRecipe did not stay non-terminal on pending workflow execution"
diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
index 35a61e913..f72cbca00 100755
--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
@@ -18,6 +18,8 @@ BACKEND_ID="${E2E_BACKEND_ID:-be-${RUN_ID}}"
DENIED_POD="${E2E_DENIED_POD:-deny-${RUN_ID}}"
BACKEND_PORT="${E2E_BACKEND_PORT:-8080}"
CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"
+SOURCE_DEPLOYMENT="$SOURCE_ID"
+BACKEND_DEPLOYMENT="$BACKEND_ID"
CREATED=0
cleanup() {
@@ -28,9 +30,9 @@ cleanup() {
--ignore-not-found --wait=false >/dev/null 2>&1 || status=1
wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \
>/dev/null 2>&1 || status=1
- kctl delete deployment "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \
+ kctl delete deployment "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \
--ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- kctl delete service "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \
+ kctl delete service "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \
--ignore-not-found >/dev/null 2>&1 || status=1
kctl delete networkpolicy -n "$SANDBOX_NS" \
-l "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \
@@ -85,6 +87,21 @@ wait_internal_ready() {
return 1
}
+wait_for_workload_instance() {
+ local workload_id=$1 timeout=${2:-$TIMEOUT_POD} elapsed=0 instance
+ while [ "$elapsed" -lt "$timeout" ]; do
+ instance="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
+ -o "jsonpath={.status.workloadInstances.${workload_id}}" 2>/dev/null || true)"
+ if [ -n "$instance" ]; then
+ printf '%s\n' "$instance"
+ return 0
+ fi
+ sleep "$POLL_INTERVAL"
+ elapsed=$((elapsed + POLL_INTERVAL))
+ done
+ return 1
+}
+
policy_refs() {
kctl get networkpolicy -A -l "$1" \
-o go-template='{{range .items}}{{.metadata.namespace}}/{{.metadata.name}}{{"\n"}}{{end}}' \
@@ -214,18 +231,26 @@ fi
ok "Fixture has no egressBindings shortcut"
header "Phase 2 - WRC reconciliation"
-wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID" "$TIMEOUT_POD" && ok "Source deployment ready" || {
+SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID" "$TIMEOUT_POD")" || {
+ fail "Source workload instance was not assigned"
+ exit 1
+}
+BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID" "$TIMEOUT_POD")" || {
+ fail "Backend workload instance was not assigned"
+ exit 1
+}
+wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD" && ok "Source deployment ready" || {
fail "Source deployment not ready"
exit 1
}
-wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID" "$TIMEOUT_POD" && ok "Backend deployment ready" || {
+wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD" && ok "Backend deployment ready" || {
fail "Backend deployment not ready"
exit 1
}
wait_internal_ready "$TIMEOUT_POD"
-resolved_target="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"
-expected_target="http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"
+resolved_target="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"
+expected_target="http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"
[ "$resolved_target" = "$expected_target" ] && ok "TARGET_URL resolved to ${resolved_target}" || {
fail "TARGET_URL resolved to '${resolved_target}', expected '${expected_target}'"
exit 1
@@ -240,7 +265,7 @@ assert_policy "$egress_ref" "egress" "$SOURCE_ID" "$BACKEND_ID"
assert_policy "$ingress_ref" "ingress" "$BACKEND_ID" "$SOURCE_ID"
header "Phase 4 - Positive packet flow"
-positive_output="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- \
+positive_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \
sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"
printf "%s" "$positive_output" | grep -Fq "issue485-ok" && \
ok "WRC source reached backend through inferred internal dependency" || {
@@ -282,7 +307,7 @@ wait_for_pod "$SANDBOX_NS" "run=${DENIED_POD}" 60 && ok "Unlabeled negative pod
}
if kctl exec "$DENIED_POD" -n "$SANDBOX_NS" -- wget -qO- \
--timeout="$CONNECT_TIMEOUT" --tries=1 \
- "http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then
+ "http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then
fail "Unlabeled pod reached backend; policy is too broad or NetworkPolicy is not enforced"
exit 1
fi
diff --git a/scripts/e2e/measure-host-bundle-reconcile.sh b/scripts/e2e/measure-host-bundle-reconcile.sh
index 1eb66d287..c1f7ee5a7 100755
--- a/scripts/e2e/measure-host-bundle-reconcile.sh
+++ b/scripts/e2e/measure-host-bundle-reconcile.sh
@@ -151,9 +151,10 @@ readonly DELPASS_RESURRECTION_SETTLE_S=30 # quiet window before re-asserting
# termination grace. A child with NO deletionTimestamp never gets this
# extension (the delete was never issued → immediate fail).
readonly MEASURE_TEARDOWN_CONVERGE_S=120
-# Exact log lines emitted by host-context-controller/src/k8sClient.ts for
-# the startup full fleet pass (reason string 'initial Host reconciliation').
-readonly HCC_PASS_STARTED_MARKER='Running initial Host reconciliation'
+# The startup marker is emitted immediately before the cold-start Host
+# inventory/watch recovery. COMPLETE/FAIL below retain the precise
+# 'initial Host reconciliation' reason supplied to the fleet pass.
+readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'
readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'
readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'
@@ -964,9 +965,9 @@ phase_delete_during_pass() {
_delpass_pass_active_probe() {
local logs
logs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"
- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi
- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then pass_state='completed'; return 0; fi
- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_STARTED_MARKER"; then pass_state='active'; return 0; fi
+ if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi
+ if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then pass_state='completed'; return 0; fi
+ if grep -Fq -- "$HCC_PASS_STARTED_MARKER" <<<"$logs"; then pass_state='active'; return 0; fi
return 1
}
# shellcheck disable=SC2329 # invoked by name via wait_until
@@ -1015,7 +1016,7 @@ phase_delete_during_pass() {
fail "delete-during-pass: could not re-read HCC logs immediately after the delete — overlap cannot be verified (fail-closed)"
return 1
fi
- if printf '%s' "$post_delete_logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then
+ if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$post_delete_logs"; then
fail "delete-during-pass: pass logged COMPLETED within the probe->delete window — overlap NOT established; re-run"
return 1
fi
@@ -1027,8 +1028,8 @@ phase_delete_during_pass() {
_delpass_pass_completed_probe() {
local logs
logs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"
- if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi
- printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER" && pass_state='completed'
+ if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi
+ grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs" && pass_state='completed'
}
if ! wait_until "$DELPASS_PASS_COMPLETE_DEADLINE_S" 2 "initial full pass completion after the delete" \
_delpass_pass_completed_probe _delpass_logs_diag; then
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 0f9247a86..0186ed584 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -1012,7 +1012,7 @@ if [ "$RESET_DB" = true ]; then
$KC -n control-plane rollout status deployment/trace-maintenance-worker --timeout=180s >/dev/null
fi
if [ "$RESET_HCC_REPLICAS" -gt 0 ]; then
- $KC -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/null
+ $KC -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null
fi
ok "Control-api database and GFS roles converged after reset"
else
diff --git a/scripts/tests/test-converge-control-db-after-reset.sh b/scripts/tests/test-converge-control-db-after-reset.sh
index 71f036828..590c749c0 100755
--- a/scripts/tests/test-converge-control-db-after-reset.sh
+++ b/scripts/tests/test-converge-control-db-after-reset.sh
@@ -51,7 +51,7 @@ case "$*" in
*'rollout status deployment/workflow-recipes --timeout=180s') exit 0 ;;
*'rollout status deployment/trace-maintenance-worker --timeout=180s') exit 0 ;;
*'scale deployment/host-context-controller --replicas=4') exit 0 ;;
- *'rollout status deployment/host-context-controller --timeout=180s') exit 0 ;;
+ *'rollout status deployment/host-context-controller --timeout=900s') exit 0 ;;
*'get pods -l app=workflow-recipes -o name') printf 'pod/workflow-old\n' ;;
*'get pods -l app=trace-maintenance-worker -o name') printf 'pod/trace-old\n' ;;
*'wait --for=delete pod -l app=workflow-recipes --timeout=180s') exit 0 ;;
diff --git a/scripts/tests/test-hcc-api-gateway-readiness.sh b/scripts/tests/test-hcc-api-gateway-readiness.sh
new file mode 100755
index 000000000..ce2930e86
--- /dev/null
+++ b/scripts/tests/test-hcc-api-gateway-readiness.sh
@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+# Functions under test are evaluated from the E2E source, so ShellCheck cannot
+# see their use of the mock environment variables declared by each harness.
+# shellcheck disable=SC2034
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+DEPLOYMENT="${ROOT}/deploy/base/control-plane/host-context-controller-api-gateway.yaml"
+CONFIGMAP="${ROOT}/deploy/base/control-plane/configmaps.yaml"
+BOOTSTRAP_E2E="${ROOT}/scripts/e2e/e2e-hcc-readiness-bootstrap.sh"
+
+fail() {
+ echo "FAIL: $1" >&2
+ exit 1
+}
+
+readiness_probe="$(
+ awk '
+ /^[[:space:]]*readinessProbe:/ { in_probe = 1; next }
+ in_probe && /^[[:space:]]*resources:/ { exit }
+ in_probe { print }
+ ' "${DEPLOYMENT}"
+)"
+
+if ! grep -Eq '^[[:space:]]*path:[[:space:]]*/health[[:space:]]*$' <<<"${readiness_probe}"; then
+ fail "HCC API gateway readiness probe must use the gateway-local /health endpoint"
+fi
+echo "PASS: HCC API gateway readiness probe uses the gateway-local /health endpoint"
+
+hcc_config="$(
+ awk '
+ BEGIN { RS = "---" }
+ /name:[[:space:]]*host-context-controller-api-gateway/ { print; exit }
+ ' "${CONFIGMAP}"
+)"
+
+if [[ -z "${hcc_config}" ]]; then
+ fail "HCC API gateway nginx ConfigMap is missing"
+fi
+
+health_location="$(
+ awk '
+ /^[[:space:]]*location = \/health \{/ { in_location = 1 }
+ in_location { print }
+ in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }
+ ' <<<"${hcc_config}"
+)"
+
+if [[ "${health_location}" != *'return 200'* ]] ||
+ [[ "${health_location}" == *'proxy_pass'* ]]; then
+ fail "HCC API gateway /health must remain a local nginx health response"
+fi
+echo "PASS: HCC API gateway /health remains local to nginx"
+
+ready_location="$(
+ awk '
+ /^[[:space:]]*location = \/ready \{/ { in_location = 1 }
+ in_location { print }
+ in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }
+ ' <<<"${hcc_config}"
+)"
+
+if [[ "${ready_location}" != *'proxy_pass http://host_context_controller_upstream;'* ]]; then
+ fail "HCC API gateway /ready must continue proxying the HCC readiness contract"
+fi
+echo "PASS: HCC API gateway /ready continues proxying HCC readiness"
+
+runtime_hold_function="$(
+ sed -n '/^hcc_gateway_remains_ready_without_hcc() {$/,/^}$/p' "${BOOTSTRAP_E2E}"
+)"
+unavailable_probe_function="$(
+ sed -n '/^hcc_gateway_ready_proxy_unavailable() {$/,/^}$/p' "${BOOTSTRAP_E2E}"
+)"
+gateway_pod_function="$(
+ sed -n '/^running_hcc_gateway_pod() {$/,/^}$/p' "${BOOTSTRAP_E2E}"
+)"
+if [[ "${gateway_pod_function}" != *'ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"'* ]] ||
+ [[ "${gateway_pod_function}" == *'running_pod_name'* ]]; then
+ fail "gateway runtime proof must select a Ready pod through the shared E2E helper"
+fi
+echo "PASS: gateway runtime proof selects a Ready pod through the shared helper"
+
+if [[ "${runtime_hold_function}" != *'hcc_gateway_deployment_ready'* ]] ||
+ [[ "${runtime_hold_function}" != *'hcc_gateway_local_health_ok'* ]] ||
+ [[ "${runtime_hold_function}" != *'hcc_gateway_ready_proxy_unavailable'* ]] ||
+ ! grep -Fq 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}"; then
+ fail "bootstrap E2E must hold HCC unavailable across a full gateway probe-failure window"
+fi
+echo "PASS: bootstrap E2E proves gateway readiness remains local while HCC is unavailable"
+
+# Exercise the full conjunction with a bounded synthetic clock. Each required
+# observation must independently fail the hold; this kills a permissive
+# && -> || mutation without requiring a cluster or a real sleep.
+gateway_hold_case() (
+ local failed_observation=$1 clock_file
+ clock_file="$(mktemp "${TMPDIR:-/tmp}/hcc-gateway-clock.XXXXXX")"
+ printf '%s' 0 >"$clock_file"
+ trap 'rm -f "$clock_file"' EXIT
+ HCC_NS=control-plane
+ date() {
+ local calls
+ calls="$(<"$clock_file")"
+ calls=$((calls + 1))
+ printf '%s' "$calls" >"$clock_file"
+ [ "$calls" -le 2 ] && printf '%s\n' 0 || printf '%s\n' 1
+ }
+ sleep() { :; }
+ hcc_gateway_deployment_ready() {
+ [ "$failed_observation" != deployment ]
+ }
+ hcc_gateway_local_health_ok() {
+ [ "$failed_observation" != health ]
+ }
+ hcc_gateway_ready_proxy_unavailable() {
+ [ "$failed_observation" != proxy ]
+ }
+ eval "${runtime_hold_function}"
+ hcc_gateway_remains_ready_without_hcc 1
+)
+
+if gateway_hold_case none &&
+ ! gateway_hold_case deployment &&
+ ! gateway_hold_case health &&
+ ! gateway_hold_case proxy; then
+ echo "PASS: gateway hold requires every readiness, health, and proxy-unavailable observation"
+else
+ fail "gateway hold can accept a failed readiness, health, or proxy observation"
+fi
+
+for unavailable_status in 502 503 504; do
+ if (
+ HCC_NS=control-plane
+ running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }
+ kctl() {
+ printf ' HTTP/1.1 %s upstream unavailable\n' "${unavailable_status}"
+ return 8
+ }
+ eval "${unavailable_probe_function}"
+ hcc_gateway_ready_proxy_unavailable
+ ); then
+ echo "PASS: gateway hold accepts explicit upstream-unavailable HTTP ${unavailable_status}"
+ else
+ fail "gateway hold rejects expected upstream-unavailable HTTP ${unavailable_status}"
+ fi
+done
+
+if (
+ HCC_NS=control-plane
+ running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }
+ kctl() {
+ printf '%s\n' 'error: unable to upgrade connection: container not found'
+ return 1
+ }
+ eval "${unavailable_probe_function}"
+ ! hcc_gateway_ready_proxy_unavailable
+); then
+ echo "PASS: gateway hold rejects kubectl or container transport failure"
+else
+ fail "gateway hold can mistake kubectl or container transport failure for upstream unavailability"
+fi
+
+if (
+ HCC_NS=control-plane
+ running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }
+ kctl() {
+ printf '%s\n' 'healthy upstream response'
+ return 0
+ }
+ eval "${unavailable_probe_function}"
+ ! hcc_gateway_ready_proxy_unavailable
+); then
+ echo "PASS: gateway hold rejects a successful proxied /ready response"
+else
+ fail "gateway hold can accept a successful proxied /ready response as unavailable"
+fi
+
+hold_line="$(grep -nF 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}" | cut -d: -f1)"
+restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "${BOOTSTRAP_E2E}" | cut -d: -f1)"
+recovery_line="$(grep -nF 'hcc_gateway_ready_proxy_recovers ||' "${BOOTSTRAP_E2E}" | cut -d: -f1)"
+if [[ -z "${hold_line}" || -z "${restart_line}" || -z "${recovery_line}" ]] ||
+ (( hold_line >= restart_line || restart_line >= recovery_line )); then
+ fail "gateway unavailable/ready/recovery assertions must surround the HCC restart"
+fi
+echo "PASS: bootstrap E2E proves proxied /ready failure and recovery in causal order"
diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.sh
index 50104032d..379d362e6 100755
--- a/scripts/tests/test-hcc-watch-recovery-gate.sh
+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh
@@ -1,21 +1,39 @@
#!/usr/bin/env bash
+# Values in the eval-based fixture harness are consumed by extracted gate
+# functions, which static shell analysis cannot follow.
+# shellcheck disable=SC2034
set -u
FAIL=0
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SCRIPT_DIR="${ROOT}/scripts/e2e"
-GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"
+WATCH_GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"
+READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-readiness-bootstrap.sh"
+MCP_READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-mcp-context-readiness.sh"
+HOST_BUNDLE_MEASURE="${SCRIPT_DIR}/measure-host-bundle-reconcile.sh"
+HOST_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"
+GATES=("$WATCH_GATE" "$READINESS_GATE" "$MCP_READINESS_GATE")
LOCK_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
LOG_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"
+FIXTURE_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
+HCC_K8S_CLIENT="${ROOT}/host-context-controller/src/k8sClient.ts"
+HCC_EGRESS_COORDINATOR="$(
+ printf '%s' "${ROOT}/host-context-controller/src/externalEgressConvergenceCoordinator.ts"
+)"
+HCC_NETWORK_POLICY_RECONCILER="$(
+ printf '%s' "${ROOT}/host-context-controller/src/networkPolicyReconciler.ts"
+)"
+WATCH_GATE_TARGET="$(sed -n '/^test-e2e-hcc-communicationchannel-watch-recovery:/,/^$/p' "${ROOT}/Makefile")"
MOCK_STATE_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-lock-test.XXXXXX")"
MOCK_LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-log-test.XXXXXX")"
+MOCK_PROFILE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hcc-profile-test.XXXXXX")"
rm -f "$MOCK_STATE_FILE"
-trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"' EXIT
+trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"; rm -rf "$MOCK_PROFILE_ROOT"' EXIT
pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }
-for script in "$GATE" "$LOCK_HELPER" "$LOG_HELPER"; do
+for script in "${GATES[@]}" "$LOCK_HELPER" "$LOG_HELPER" "$FIXTURE_HELPER"; do
if bash -n "$script"; then
pass "$(basename "$script") has valid bash syntax"
else
@@ -23,12 +41,1282 @@ for script in "$GATE" "$LOCK_HELPER" "$LOG_HELPER"; do
fi
done
+# A node carrying a minikube label from another profile must not satisfy the
+# destructive-gate boundary. Each gate binds the label value to the explicit
+# context instead of accepting any non-null label.
+matching_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"branch-profile"}}}]}'
+mismatched_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"other-profile"}}}]}'
+if jq -e --arg context branch-profile \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \
+ <<<"$matching_nodes" >/dev/null &&
+ ! jq -e --arg context branch-profile \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \
+ <<<"$mismatched_nodes" >/dev/null &&
+ [ "$(grep -Fl 'minikube.k8s.io/name"] == $context' "${GATES[@]}" | wc -l | tr -d ' ')" = 3 ] &&
+ ! grep -Fq 'minikube.k8s.io/name"] != null' "${GATES[@]}"; then
+ pass "all destructive HCC gates bind the minikube node label to the explicit profile"
+else
+ fail "an HCC gate can accept a minikube node owned by another profile"
+fi
+
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if [[ "$WATCH_GATE_TARGET" == *'E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"'* ]] &&
+ [[ "$WATCH_GATE_TARGET" == *'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT)'* ]]; then
+ pass "CommunicationChannel gate target propagates exact pre-gate and profile ownership"
+else
+ fail "CommunicationChannel gate target omits exact pre-gate or profile ownership"
+fi
+
+proxy_rollout_function="$(
+ sed -n '/^print_hcc_proxy_rollout_diagnostics() {$/,/^}$/p' "$WATCH_GATE"
+)"
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if [[ "$proxy_rollout_function" == *'kctl get pods'* ]] &&
+ [[ "$proxy_rollout_function" == *'--previous --tail=120'* ]] &&
+ [[ "$proxy_rollout_function" == *'status.containerStatuses'* ]] &&
+ [[ "$proxy_rollout_function" == *'// "unknown"'* ]] &&
+ [[ "$proxy_rollout_function" != *'\\"unknown\\"'* ]] &&
+ grep -Fq 'if ! kctl rollout status deployment "$HCC_DEPLOY"' "$WATCH_GATE" &&
+ [ "$(grep -Fc 'print_hcc_proxy_rollout_diagnostics' "$WATCH_GATE")" = 2 ]; then
+ pass "proxy rollout failure emits bounded HCC pod and previous-container diagnostics"
+else
+ fail "proxy rollout failure can discard the distinguishing HCC startup evidence"
+fi
+
+if (
+ HCC_NS=control-plane
+ HCC_DEPLOY='host-context-controller'
+ kctl() {
+ if [ "$1" = get ] && [ "$2" = pods ] && [[ "$*" == *"-o jsonpath="* ]]; then
+ printf '%s\n' hcc-pod
+ elif [ "$1" = get ] && [ "$2" = pods ]; then
+ printf '%s\n' '{"items":[{"metadata":{"name":"hcc-pod"},"status":{"phase":"Pending","containerStatuses":[{"name":"host-context-controller","ready":false,"restartCount":2,"state":{"waiting":{"reason":"CrashLoopBackOff"}},"lastState":{}}]}}]}'
+ elif [ "$1" = logs ]; then
+ printf '%s\n' 'bounded diagnostic log'
+ else
+ return 1
+ fi
+ }
+ eval "$proxy_rollout_function"
+ output="$(print_hcc_proxy_rollout_diagnostics 2>&1)"
+ [[ "$output" == *"pod=hcc-pod phase=Pending"* ]] &&
+ [[ "$output" != *"pod=null"* ]] &&
+ [[ "$output" != *"phase=null"* ]]
+); then
+ pass "proxy rollout diagnostics retain the parent pod identity and phase"
+else
+ fail "proxy rollout diagnostics lose pod metadata inside container status iteration"
+fi
+
+# The proxy changes only the HCC client's target hostname. The HCC's
+# allow-k8s-api-egress policies must retain the Kubernetes-injected API IP,
+# which is the explicit fail-closed configuration seam for CIDR policy data.
+# shellcheck disable=SC2016
+if grep -Fq 'K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY"' "$WATCH_GATE" &&
+ grep -Fq 'K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"' "$WATCH_GATE" &&
+ grep -Fq -- '--arg api_cidr "$K8S_API_CIDR"' "$WATCH_GATE" &&
+ grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr' "$WATCH_GATE" &&
+ grep -Fq 'allow-k8s-api-egress-${MCP_NS}' "$WATCH_GATE" &&
+ grep -Fq '[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ]' "$WATCH_GATE" &&
+ grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS-' "$FIXTURE_HELPER"; then
+ pass "proxy fault injection preserves an explicit valid API egress CIDR and restores it"
+else
+ fail "proxy fault injection can turn a DNS hostname into an invalid API egress CIDR"
+fi
+
+if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' "$READINESS_GATE" &&
+ [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$READINESS_GATE")" = 1 ] &&
+ ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$READINESS_GATE" &&
+ ! grep -Fq 'cluster_fingerprint_file=' "$READINESS_GATE" &&
+ ! grep -Fq 'profile_env=' "$READINESS_GATE"; then
+ pass "readiness gate delegates ownership, HEAD, profile, fingerprint, and gate proof once"
+else
+ fail "readiness gate duplicates, bypasses, or splits the shared branch-owned proof"
+fi
+
+bootstrap_repair_function="$(
+ sed -n '/^print_repair_instructions() {$/,/^}$/p' "$READINESS_GATE"
+)"
+mcp_repair_function="$(
+ sed -n '/^print_repair_instructions() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+# Literal source-code assertions: every fail-closed repair path must restore
+# the captured replica count before treating rollout health as repair proof.
+# shellcheck disable=SC2016
+if [[ "$bootstrap_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&
+ [[ "$bootstrap_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=180s'* ]] &&
+ [[ "$mcp_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&
+ [[ "$mcp_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=240s'* ]] &&
+ grep -Fq 'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}' "$WATCH_GATE" &&
+ grep -Fq 'rollout status deployment/${HCC_DEPLOY} --timeout=180s' "$WATCH_GATE"; then
+ pass "all HCC repair instructions restore the original replica count and verify rollout"
+else
+ fail "an HCC repair path can leave the controller scaled to zero"
+fi
+
+fixture_gate_function="$(sed -n '/^require_branch_owned_hcc_gate() {$/,/^}$/p' "$FIXTURE_HELPER")"
+marker_get_count="$(
+ grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' <<<"$fixture_gate_function"
+)"
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if [ "$marker_get_count" = 1 ] &&
+ [[ "$fixture_gate_function" == *'-o json'* ]] &&
+ [[ "$fixture_gate_function" == *'.data.worktreeId'* ]] &&
+ [[ "$fixture_gate_function" == *'.data.gitHead'* ]] &&
+ [[ "$fixture_gate_function" == *'<<<"$HCC_BRANCH_GATE_SYNC_MARKER"'* ]]; then
+ pass "shared branch gate derives ownership and HEAD from one exported ConfigMap JSON snapshot"
+else
+ fail "shared branch gate can split ownership and HEAD across marker reads"
+fi
+
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if [[ "$fixture_gate_function" == *'expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"'* ]] &&
+ [[ "$fixture_gate_function" == *'[ -n "$expected_branch" ]'* ]] &&
+ [[ "$fixture_gate_function" == *'[ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ]'* ]] &&
+ [[ "$fixture_gate_function" == *'profile_env="${E2E_BRANCH_PROFILE_ENV:-'* ]] &&
+ [[ "$fixture_gate_function" == *'profile_env_value PROFILE'* ]] &&
+ [[ "$fixture_gate_function" == *'profile_env_value REPO_DIR'* ]] &&
+ [[ "$fixture_gate_function" == *'profile_env_value BRANCH'* ]] &&
+ [[ "$fixture_gate_function" != *'profile_env_value SHA_SHORT'* ]] &&
+ [[ "$fixture_gate_function" == *'profile_env_value DIRTY'* ]] &&
+ [[ "$fixture_gate_function" == *'cluster_fingerprint_file='* ]] &&
+ [[ "$fixture_gate_function" == *'.data.clusterFingerprint'* ]] &&
+ [[ "$fixture_gate_function" == *'.data.gate'* ]] &&
+ [[ "$fixture_gate_function" == *'E2E_EXPECTED_PRE_GATE_GATE'* ]]; then
+ pass "shared branch gate owns the complete profile, fingerprint, and expected-gate proof"
+else
+ fail "shared branch gate omits a required profile, fingerprint, or expected-gate boundary"
+fi
+
+for gate in "${GATES[@]}"; do
+ marker_reads="$(grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' "$gate")"
+ if [ "$marker_reads" = 0 ] &&
+ ! grep -Fq 'profile_env=' "$gate" &&
+ ! grep -Fq 'cluster_fingerprint_file=' "$gate" &&
+ grep -Fq 'require_branch_owned_hcc_gate' "$gate"; then
+ pass "$(basename "$gate") delegates its branch-owned proof to the shared authority"
+ else
+ fail "$(basename "$gate") duplicates or bypasses the shared branch-owned proof"
+ fi
+done
+
+run_branch_gate_case() (
+ local mode=$1
+ local repo_root="${MOCK_PROFILE_ROOT}/repo"
+ local context="clerum-codex-profile-test-1234abcd"
+ local head="1111111111111111111111111111111111111111"
+ local short_head="11111111"
+ local branch="codex/profile-test"
+ local fingerprint="2222222222222222222222222222222222222222"
+ local marker_fingerprint="$fingerprint"
+ local expected_gate="full"
+ local marker_gate="$expected_gate"
+ local minikube_profile="$context"
+ local profile="$context"
+ local profile_repo="$repo_root"
+ local profile_branch="$branch"
+ local profile_sha="$short_head"
+ local profile_dirty=false
+ local mock_worktree_dirty=""
+ local worktree_id
+ worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"
+
+ case "$mode" in
+ success) ;;
+ detached) branch="" ;;
+ minikube-profile-missing) minikube_profile="" ;;
+ minikube-profile) minikube_profile="clerum-other-profile-1234abcd" ;;
+ profile-name) profile="clerum-other-profile-1234abcd" ;;
+ profile-repo) profile_repo="${repo_root}-other" ;;
+ profile-branch) profile_branch="codex/other" ;;
+ reused-profile) profile_sha="aaaaaaaa" ;;
+ profile-dirty) profile_dirty=true ;;
+ dirty-worktree) mock_worktree_dirty=" M changed.ts" ;;
+ fingerprint) marker_fingerprint="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;;
+ gate) marker_gate="targeted" ;;
+ *) exit 98 ;;
+ esac
+
+ mkdir -p "${MOCK_PROFILE_ROOT}/state/${worktree_id}"
+ printf '%s\n' "$fingerprint" >"${MOCK_PROFILE_ROOT}/state/${worktree_id}/cluster.sha"
+ printf '%s\n' \
+ "PROFILE=${profile}" \
+ "REPO_DIR=${profile_repo}" \
+ "BRANCH=${profile_branch}" \
+ "SHA_SHORT=${profile_sha}" \
+ "DIRTY=${profile_dirty}" >"${MOCK_PROFILE_ROOT}/profile.env"
+
+ E2E_KUBECONTEXT="$context"
+ MINIKUBE_PROFILE="$minikube_profile"
+ E2E_BRANCH_PROFILE_ENV="${MOCK_PROFILE_ROOT}/profile.env"
+ E2E_PRE_GATE_STATE_ROOT="${MOCK_PROFILE_ROOT}/state"
+ E2E_EXPECTED_PRE_GATE_GATE="$expected_gate"
+ HCC_NS=control-plane
+ SCRIPT_DIR="${ROOT}/scripts/e2e"
+ MOCK_MARKER="$(
+ jq -cn --arg worktreeId "$worktree_id" --arg gitHead "$head" \
+ --arg clusterFingerprint "$marker_fingerprint" --arg gate "$marker_gate" \
+ '{data:{worktreeId:$worktreeId,gitHead:$gitHead,
+ clusterFingerprint:$clusterFingerprint,gate:$gate}}'
+ )"
+
+ is_branch_scoped_e2e_context() { return 0; }
+ die() { exit 97; }
+ git() {
+ local command=$3
+ shift 3
+ case "$command $*" in
+ "rev-parse --show-toplevel") printf '%s\n' "$repo_root" ;;
+ "status --porcelain --untracked-files=normal") printf '%s\n' "$mock_worktree_dirty" ;;
+ "rev-parse HEAD") printf '%s\n' "$head" ;;
+ "rev-parse --short=8 HEAD") printf '%s\n' "$short_head" ;;
+ "branch --show-current") printf '%s\n' "$branch" ;;
+ *) return 96 ;;
+ esac
+ }
+ kctl() { printf '%s\n' "$MOCK_MARKER"; }
+ # shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+ source "$FIXTURE_HELPER"
+ require_branch_owned_hcc_gate "$HCC_NS"
+)
+
+if run_branch_gate_case success; then
+ pass "shared branch gate accepts matching branch-profile evidence"
+else
+ fail "shared branch gate rejects matching branch-profile evidence"
+fi
+if run_branch_gate_case reused-profile; then
+ pass "shared branch gate accepts a reused same-branch profile after an exact-head pre-gate sync"
+else
+ fail "shared branch gate rejects a reused same-branch profile despite exact-head pre-gate evidence"
+fi
+for case_name in detached minikube-profile-missing minikube-profile profile-name profile-repo profile-branch \
+ profile-dirty dirty-worktree fingerprint gate; do
+ if run_branch_gate_case "$case_name" >/dev/null 2>&1; then
+ fail "shared branch gate accepted mismatched ${case_name} evidence"
+ else
+ pass "shared branch gate rejects mismatched ${case_name} evidence"
+ fi
+done
+
+retry_progress_function="$(
+ sed -n '/^external_egress_retry_progress_is_observed() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+retry_query_function="$(
+ sed -n '/^dns_retry_query_observed_since_schedule() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ CONTEXT_NAME=e2e-context
+ MOCK_HCC_LOG='
+[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout
+[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms
+'
+ kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }
+ eval "$retry_progress_function"
+ external_egress_retry_progress_is_observed
+); then
+ pass "DNS failure is attributed to the dedicated external-egress retry lane"
+else
+ fail "the injected DNS hold cannot prove dedicated external-egress retry progress"
+fi
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ CONTEXT_NAME=e2e-context
+ MOCK_HCC_LOG='
+[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout
+[K8s] Scheduling external egress retry 1/3 for McpServer "different-server" in 5000ms
+[K8s] unrelated message for McpServer "e2e-held-server"
+'
+ kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }
+ eval "$retry_progress_function"
+ ! external_egress_retry_progress_is_observed
+); then
+ pass "external-egress retry evidence cannot combine a peer retry with the fixture server marker"
+else
+ fail "external-egress retry evidence can be assembled from different log lines or servers"
+fi
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ CONTEXT_NAME=e2e-context
+ MOCK_HCC_LOG='
+[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout
+[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms
+'
+ for ((line = 0; line < 4096; line++)); do
+ MOCK_HCC_LOG+=$'\n[K8s] unrelated high-volume log line'
+ done
+ kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }
+ eval "$retry_progress_function"
+ external_egress_retry_progress_is_observed
+); then
+ pass "an early matching retry line survives a high-volume log buffer without SIGPIPE ambiguity"
+else
+ fail "a valid early retry marker is lost in a high-volume log buffer"
+fi
+if (
+ DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3
+ dns_hold_count() { printf '%s\n' 4; }
+ eval "$retry_query_function"
+ dns_retry_query_observed_since_schedule
+); then
+ pass "a post-schedule DNS query proves the dedicated retry executed"
+else
+ fail "a fresh DNS query after scheduling does not prove retry execution"
+fi
+if (
+ DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3
+ dns_hold_count() { printf '%s\n' 3; }
+ eval "$retry_query_function"
+ ! dns_retry_query_observed_since_schedule
+); then
+ pass "a pre-schedule DNS query cannot satisfy retry execution"
+else
+ fail "stale DNS evidence can masquerade as retry execution"
+fi
+if grep -Fq '{name:"timeout",value:"30"}' "$MCP_READINESS_GATE" &&
+ grep -Fq '{name:"attempts",value:"1"}' "$MCP_READINESS_GATE"; then
+ pass "the DNS fixture disables resolver retransmits that could counterfeit retry execution"
+else
+ fail "the DNS fixture can mistake a resolver retransmit for a dedicated retry"
+fi
+
+# Full marker inventory: every runtime log marker any HCC gate consumes, paired
+# with the producer statement that emits it. Both sides are matched as fixed
+# source text, never as a rendered string: a template literal is pinned by its
+# fixed prefix plus the variable that supplies the dynamic identity, so a
+# marker that only ever exists rendered (`${config.hostNamespace}`,
+# `${reason}`, `${type}`, `${snapshot.channels.length}`) still resolves without
+# reporting false drift. Matching the consumer side too means a rename on
+# either side breaks the pair instead of silently disarming a live gate: the
+# marker no longer matches any log line, the count stays zero, and the gate
+# passes on evidence it never observed.
+# Entries are flat 4-tuples: consumer file, consumer text, producer file,
+# producer text.
+# shellcheck disable=SC2016
+marker_bindings=(
+ "$WATCH_GATE" 'CommunicationChannel watch ended;'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'
+
+ "$WATCH_GATE" 'Starting CommunicationChannel watch'
+ "$HCC_K8S_CLIENT" '[K8s] Starting CommunicationChannel watch'
+
+ "$WATCH_GATE" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'
+ "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'
+
+ "$WATCH_GATE" 'cache recovery failed;'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; stateless lifecycle remains held active'
+
+ "$WATCH_GATE" 'Listing all CommunicationChannels in namespace ${CHANNEL_NS}'
+ "$HCC_K8S_CLIENT" 'Listing all CommunicationChannels in namespace ${config.channelsNamespace}'
+
+ "$WATCH_GATE" 'Listing all Hosts in namespace ${HOST_NS}'
+ "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'
+
+ "$WATCH_GATE" 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'
+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
+
+ "$WATCH_GATE" 'CommunicationChannel watch event: DELETED for ${FIXTURE_CHANNEL}'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'
+
+ "$WATCH_GATE" 'CommunicationChannel watch event: ADDED for ${FIXTURE_CHANNEL}'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'
+
+ "$LOG_HELPER" 'CommunicationChannel watch ended;'
+ "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'
+
+ "$LOG_HELPER" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'
+ "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'
+
+ "$LOG_HELPER" 'Listing all Hosts in namespace '
+ "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'
+
+ "$LOG_HELPER" 'Host(s) for lifecycle after CommunicationChannel recovery'
+ "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) for lifecycle after ${reason}'
+
+ "$LOG_HELPER" 'Host(s) after Host watch recovery convergence'
+ "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) after ${reason}'
+
+ "$LOG_HELPER" 'Completed Host reconciliation after CommunicationChannel recovery$'
+ "$HCC_K8S_CLIENT" "requestHostFleetReconcile('CommunicationChannel recovery'"
+
+ "$LOG_HELPER" 'Completed Host reconciliation after Host watch recovery convergence$'
+ "$HCC_K8S_CLIENT" "convergenceReason = 'Host watch recovery convergence'"
+
+ "$LOG_HELPER" 'Periodic resync'
+ "$HCC_K8S_CLIENT" "requestHostFleetReconcile('Periodic resync'"
+
+ "$READINESS_GATE" "START_MARKER='Starting initial Host background convergence'"
+ "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'
+
+ "$READINESS_GATE" "COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'"
+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
+
+ "$READINESS_GATE" "FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'"
+ "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'
+
+ "$READINESS_GATE" 'after initial Host reconciliation'
+ "$HCC_K8S_CLIENT" "'initial Host reconciliation',"
+
+ "$HOST_BUNDLE_MEASURE" "HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'"
+ "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'
+
+ "$HOST_BUNDLE_MEASURE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"
+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
+
+ "$HOST_BUNDLE_MEASURE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"
+ "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'
+
+ "$HOST_STORM_GATE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"
+ "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
+
+ "$HOST_STORM_GATE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"
+ "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'
+
+ "$MCP_READINESS_GATE" 'Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};'
+ "$HCC_EGRESS_COORDINATOR" 'Initial external egress reconciliation failed for ${key};'
+
+ "$MCP_READINESS_GATE" 'Scheduling external egress retry '
+ "$HCC_EGRESS_COORDINATOR" 'Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}'
+
+ "$MCP_READINESS_GATE" 'for McpServer \"${MCP_NAME}\"'
+ "$HCC_EGRESS_COORDINATOR" 'for McpServer "${this.retryIntents.get(key)!.server.name}"'
+
+ "$MCP_READINESS_GATE" 'Context watch event: MODIFIED for ${CONTEXT_NAME}'
+ "$HCC_K8S_CLIENT" 'Context watch event: ${type} for ${context.name}'
+
+ "$MCP_READINESS_GATE" '[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []'
+ "$HCC_NETWORK_POLICY_RECONCILER" 'Reconciling context "${contextId}" — allowed servers: [${allowedServers.join('
+
+ "$MCP_READINESS_GATE" "NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'"
+ "$HCC_K8S_CLIENT" 'Initial NetworkPolicy background reconciliation failed:'
+
+ "$MCP_READINESS_GATE" "NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'"
+ "$HCC_K8S_CLIENT" 'Initial NetworkPolicy post-certification additive reconciliation failed:'
+
+ "$MCP_READINESS_GATE" "NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'"
+ "$HCC_K8S_CLIENT" 'Scheduling initial ${lane} background convergence retry'
+
+ "$MCP_READINESS_GATE" "NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '"
+ "$HCC_K8S_CLIENT" 'NetworkPolicy reconciliation failed for context ${context.name}:'
+)
+marker_drift=""
+marker_binding_count=0
+for ((marker_index = 0; marker_index < ${#marker_bindings[@]}; marker_index += 4)); do
+ consumer_file="${marker_bindings[marker_index]}"
+ consumer_marker="${marker_bindings[marker_index + 1]}"
+ producer_file="${marker_bindings[marker_index + 2]}"
+ producer_marker="${marker_bindings[marker_index + 3]}"
+ marker_binding_count=$((marker_binding_count + 1))
+ # `grep -Fq -- ""` matches every file, so an empty marker would disarm this
+ # pair while still reporting a pass. Refuse it instead of searching for it.
+ if [ -z "$consumer_marker" ] || [ -z "$producer_marker" ]; then
+ marker_drift+=" marker pair at index ${marker_index} has an empty marker"$'\n'
+ continue
+ fi
+ if ! grep -Fq -- "$consumer_marker" "$consumer_file"; then
+ marker_drift+=" consumer $(basename "$consumer_file") no longer reads: ${consumer_marker}"$'\n'
+ fi
+ if ! grep -Fq -- "$producer_marker" "$producer_file"; then
+ marker_drift+=" producer $(basename "$producer_file") no longer emits: ${producer_marker}"$'\n'
+ fi
+done
+# The stride loop consumes 4 elements per pair, so an array whose length is not
+# a multiple of 4 silently shifts every entry after the edit and leaves the tail
+# unset. Assert the arity explicitly rather than relying on the pair count to
+# happen to notice.
+if [ $((${#marker_bindings[@]} % 4)) -eq 0 ] &&
+ [ "$marker_binding_count" -eq 35 ] &&
+ [ -z "$marker_drift" ]; then
+ pass "all ${marker_binding_count} consumed HCC log markers stay bound to their runtime producers"
+else
+ printf '%s' "$marker_drift" >&2
+ fail "a consumed HCC log marker drifted from its runtime producer (${marker_binding_count} pairs checked)"
+fi
+
+# Completeness fence: every static single-quoted *_MARKER defined in the e2e
+# gates MUST be inventoried above as a consumer_marker, so a NEW marker cannot be
+# born outside the drift fence (the pinned count only catches removals). The scan
+# covers both the gates themselves and the _lib/ helpers they source — a marker
+# born in a sourced helper is the same escape class. Scoped to single-quoted
+# literals — a dynamic id like C_MARKER="refusal-c-${RUN_ID}" (double-quoted, a
+# per-run test id, not a cross-file HCC log marker) is not the class this fence
+# covers and is excluded by the quote style, not a hard-coded allow-list.
+marker_is_inventoried() {
+ local needle="$1" i
+ for ((i = 1; i < ${#marker_bindings[@]}; i += 4)); do
+ [ "${marker_bindings[i]}" = "$needle" ] && return 0
+ done
+ return 1
+}
+uninventoried_markers=""
+while IFS= read -r marker_def; do
+ [ -n "$marker_def" ] || continue
+ marker_is_inventoried "$marker_def" ||
+ uninventoried_markers+=" ${marker_def}"$'\n'
+done < <(
+ grep -rhoE "^[[:space:]]*(readonly[[:space:]]+)?[A-Z_]+_MARKER='[^']*'" \
+ "${SCRIPT_DIR}"/*.sh "${SCRIPT_DIR}"/_lib/*.sh |
+ sed -E "s/^[[:space:]]*(readonly[[:space:]]+)?//"
+)
+if [ -z "$uninventoried_markers" ]; then
+ pass "every single-quoted *_MARKER defined in the e2e gates is inventoried in the drift fence"
+else
+ printf '%s' "$uninventoried_markers" >&2
+ fail "an e2e gate defines a *_MARKER absent from marker_bindings (drift fence would miss it)"
+fi
+
+fixture_mcp_runtime_absent_function="$(
+ sed -n '/^fixture_mcp_runtime_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+hcc_kubernetes_readiness_function="$(
+ sed -n '/^hcc_kubernetes_readiness_is_exact() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+external_policy_protocol_function="$(
+ sed -n '/^external_egress_policy_converged_with_protocol() {$/,/^}$/p' \
+ "$MCP_READINESS_GATE"
+)"
+hcc_ready_after_revoke_function="$(
+ sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+ready_probe_function="$(
+ sed -n '/^probe_new_hcc_ready_endpoint() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+
+run_ready_probe_failure_case() (
+ local expected=$1
+ MOCK_READY_PROBE_OUTPUT=$2
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ HCC_PORT=8080
+ HCC_READY_PROBE_DIAGNOSTIC=""
+ kctl() {
+ printf '%s\n' "$MOCK_READY_PROBE_OUTPUT"
+ return 1
+ }
+ eval "$ready_probe_function"
+ ! probe_new_hcc_ready_endpoint &&
+ [ "$HCC_READY_PROBE_DIAGNOSTIC" = "HCC_READY_PROBE category=${expected}" ]
+)
+
+if run_ready_probe_failure_case timeout 'HCC_READY_PROBE category=timeout' &&
+ run_ready_probe_failure_case transport 'HCC_READY_PROBE category=transport' &&
+ run_ready_probe_failure_case 'http-status status=503' \
+ 'HCC_READY_PROBE category=http-status status=503' &&
+ run_ready_probe_failure_case json-ready-false \
+ 'HCC_READY_PROBE category=json-ready-false'; then
+ pass "ready probe preserves distinct timeout, transport, HTTP, and ready=false diagnostics"
+else
+ fail "ready probe collapses a failure boundary into an indiagnostic boolean"
+fi
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ HCC_PORT=8080
+ HCC_READY_PROBE_DIAGNOSTIC="stale"
+ kctl() { return 0; }
+ eval "$ready_probe_function"
+ probe_new_hcc_ready_endpoint && [ -z "$HCC_READY_PROBE_DIAGNOSTIC" ]
+); then
+ pass "successful ready probe clears stale failure diagnostics"
+else
+ fail "successful ready probe retains a misleading prior diagnostic"
+fi
+
+# H6 gate-fix mutation (E4): the clean-window NetworkPolicy negative must FAIL
+# against an injection-window log (moving it inside the injected window would
+# false-fail — the scoping is load-bearing), PASS against a clean log, and fail
+# closed when kubectl errors. The gate itself can't run (minikube churn), so
+# this statically pins the predicate.
+clean_window_netpol_negatives_function="$(
+ sed -n '/^clean_window_netpol_log_negatives_hold() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+clean_window_marker_defs="$(grep -E "^NETPOL_[A-Z_]+_MARKER='" "$MCP_READINESS_GATE")"
+run_clean_window_negative_case() (
+ local mode=$1
+ CONTEXT_NAME=alpha
+ HCC_NS=control-plane
+ eval "$clean_window_marker_defs"
+ case $mode in
+ injected)
+ kctl() {
+ printf '%s\n' \
+ '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed: dns timeout' \
+ '[K8s] Scheduling initial NetworkPolicy background convergence retry 1 in 5000ms'
+ } ;;
+ clean) kctl() { printf '%s\n' '[K8s] NetworkPolicy convergence certified'; } ;;
+ api-error) kctl() { return 1; } ;;
+ esac
+ eval "$clean_window_netpol_negatives_function"
+ clean_window_netpol_log_negatives_hold hcc-pod
+)
+if ! run_clean_window_negative_case injected &&
+ run_clean_window_negative_case clean &&
+ ! run_clean_window_negative_case api-error; then
+ pass "clean-window NetworkPolicy negative fails inside the injected window, passes clean, and fails closed"
+else
+ fail "clean-window NetworkPolicy negative does not distinguish injected-window failures or fails open"
+fi
+
+# Static ordering fence: a clean-window negative invocation must exist in the
+# MAIN BODY (after `trap cleanup EXIT`) and before the DNS mutation. cleanup()
+# is defined earlier in the file than the mutation but runs on EXIT, so a plain
+# "first line < mutation line" test would be satisfied by the cleanup-window
+# invocation alone; excluding everything at or before the trap line pins the
+# pre-injection (window A) invocation specifically.
+trap_line="$(grep -n '^trap cleanup EXIT' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"
+dns_mutation_line="$(grep -n '^HCC_MUTATED=1' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"
+body_clean_window_negative_line="$(
+ grep -n 'clean_window_netpol_log_negatives_hold "' "$MCP_READINESS_GATE" |
+ awk -F: -v t="${trap_line:-0}" -v m="${dns_mutation_line:-0}" \
+ '$1 > t && $1 < m { print $1; exit }'
+)"
+if [ -n "$body_clean_window_negative_line" ]; then
+ pass "clean-window NetworkPolicy negative runs in the body before the DNS fault injection"
+else
+ fail "clean-window NetworkPolicy negative is missing from the pre-injection body window"
+fi
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_NS=control-plane
+ HCC_PORT=8080
+ HCC_READY_PROBE_DIAGNOSTIC=""
+ kctl() {
+ printf '%s\n' 'untrusted kubectl output containing super-secret material'
+ return 1
+ }
+ eval "$ready_probe_function"
+ ! probe_new_hcc_ready_endpoint &&
+ [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=kubectl-exec' ]
+); then
+ pass "ready probe replaces untrusted exec output with a bounded sanitized category"
+else
+ fail "ready probe can leak untrusted exec output into diagnostics"
+fi
+# Literal source-code assertions guarantee one request and bounded categories,
+# without logging the response body, headers, URL, or transport error text.
+if [ "$(grep -Fc 'kctl exec' <<<"$ready_probe_function")" = 1 ] &&
+ [[ "$ready_probe_function" == *'finish(4, "timeout")'* ]] &&
+ [[ "$ready_probe_function" == *'finish(5, "transport")'* ]] &&
+ [[ "$ready_probe_function" == *'finish(3, "http-status"'* ]] &&
+ [[ "$ready_probe_function" == *'finish(3, "json-ready-false")'* ]] &&
+ [[ "$ready_probe_function" == *"printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s'"* ]] &&
+ [[ "$ready_probe_function" != *'console.error(body'* ]] &&
+ [[ "$ready_probe_function" != *'error.message'* ]]; then
+ pass "ready probe uses one request and emits only bounded non-secret failure categories"
+else
+ fail "ready probe duplicates requests or exposes an unbounded diagnostic surface"
+fi
+
+if (
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ MCP_SERVER_LABEL=clerum.io/mcp-server
+ POLICY_TYPE_LABEL=clerum.io/policy-type
+ fixture_runtime_absent() { return 0; }
+ kctl() { return 0; }
+ eval "$fixture_mcp_runtime_absent_function"
+ fixture_mcp_runtime_absent
+); then
+ pass "fixture absence predicate accepts an absent runtime and external-egress policy"
+else
+ fail "fixture absence predicate rejects an actually absent runtime"
+fi
+if (
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ MCP_SERVER_LABEL=clerum.io/mcp-server
+ POLICY_TYPE_LABEL=clerum.io/policy-type
+ fixture_runtime_absent() { return 0; }
+ kctl() { printf '%s\n' 'networkpolicy.networking.k8s.io/stale-policy'; }
+ eval "$fixture_mcp_runtime_absent_function"
+ ! fixture_mcp_runtime_absent
+); then
+ pass "fixture absence predicate rejects a surviving external-egress policy"
+else
+ fail "fixture absence predicate can hide a surviving external-egress policy"
+fi
+if (
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ MCP_SERVER_LABEL=clerum.io/mcp-server
+ POLICY_TYPE_LABEL=clerum.io/policy-type
+ fixture_runtime_absent() { return 1; }
+ kctl() { return 0; }
+ eval "$fixture_mcp_runtime_absent_function"
+ ! fixture_mcp_runtime_absent
+); then
+ pass "fixture absence predicate rejects a surviving MCP runtime"
+else
+ fail "fixture absence predicate can hide a surviving MCP runtime"
+fi
+
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_UID=pod-uid
+ HCC_NS=control-plane
+ HCC_DEPLOY='host-context-controller'
+ kctl() {
+ case "$2" in
+ pod)
+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
+ ;;
+ deployment)
+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ eval "$hcc_kubernetes_readiness_function"
+ hcc_kubernetes_readiness_is_exact
+); then
+ pass "exact Kubernetes readiness accepts one current Ready HCC identity"
+else
+ fail "exact Kubernetes readiness rejects current Ready HCC evidence"
+fi
+# Kubelet reports an indeterminate pod as Ready=Unknown, not Ready=False. Only
+# an Unknown fixture separates the exact `== "True"` readiness contract from a
+# weakened `!= "False"` one, so this red case pins the exact comparison instead
+# of the single status value both spellings already reject. Every other field is
+# healthy, which leaves the Ready condition as the only reason to refuse.
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_UID=pod-uid
+ HCC_NS=control-plane
+ HCC_DEPLOY='host-context-controller'
+ kctl() {
+ case "$2" in
+ pod)
+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
+ ;;
+ deployment)
+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ eval "$hcc_kubernetes_readiness_function"
+ ! hcc_kubernetes_readiness_is_exact
+); then
+ pass "exact Kubernetes readiness rejects a current pod whose Ready condition is Unknown"
+else
+ fail "exact Kubernetes readiness can accept an indeterminate HCC pod"
+fi
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_UID=pod-uid
+ HCC_NS=control-plane
+ HCC_DEPLOY='host-context-controller'
+ kctl() {
+ case "$2" in
+ pod)
+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"False"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
+ ;;
+ deployment)
+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ eval "$hcc_kubernetes_readiness_function"
+ ! hcc_kubernetes_readiness_is_exact
+); then
+ pass "exact Kubernetes readiness rejects a current pod whose Ready condition is False"
+else
+ fail "exact Kubernetes readiness can accept a non-Ready HCC pod"
+fi
+# The container-level readiness clause has its own red case: a pod whose Ready
+# condition is True while the host-context-controller container is not ready
+# must still be refused, so deleting or weakening that clause cannot pass.
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_UID=pod-uid
+ HCC_NS=control-plane
+ HCC_DEPLOY='host-context-controller'
+ kctl() {
+ case "$2" in
+ pod)
+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":false}]}}'
+ ;;
+ deployment)
+ printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ eval "$hcc_kubernetes_readiness_function"
+ ! hcc_kubernetes_readiness_is_exact
+); then
+ pass "exact Kubernetes readiness rejects a Ready pod whose HCC container is not ready"
+else
+ fail "exact Kubernetes readiness can accept an unready host-context-controller container"
+fi
+if (
+ NEW_HCC_POD=hcc-pod
+ HCC_UID=pod-uid
+ HCC_NS=control-plane
+ HCC_DEPLOY='host-context-controller'
+ kctl() {
+ case "$2" in
+ pod)
+ printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
+ ;;
+ deployment)
+ printf '%s\n' '{"metadata":{"generation":5},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ eval "$hcc_kubernetes_readiness_function"
+ ! hcc_kubernetes_readiness_is_exact
+); then
+ pass "exact Kubernetes readiness rejects a stale Deployment generation"
+else
+ fail "exact Kubernetes readiness can accept stale Deployment status"
+fi
+
+if (
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ MCP_SERVER_LABEL=clerum.io/mcp-server
+ POLICY_TYPE_LABEL=clerum.io/policy-type
+ MANAGED_BY_LABEL=app.kubernetes.io/managed-by
+ MANAGED_BY_VALUE='host-context-controller'
+ EXTERNAL_EGRESS_CIDR=203.0.113.10/32
+ EXTERNAL_EGRESS_PORT=443
+ kctl() {
+ printf '%s\n' '{"items":[{"metadata":{"labels":{"app.kubernetes.io/managed-by":"host-context-controller","clerum.io/policy-type":"external-egress","clerum.io/mcp-server":"e2e-held-server"}},"spec":{"podSelector":{"matchLabels":{"clerum.io/mcp-server":"e2e-held-server"}},"policyTypes":["Egress"],"egress":[{"ports":[{"port":443,"protocol":"UDP"}],"to":[{"ipBlock":{"cidr":"203.0.113.10/32"}}]}]}}]}'
+ }
+ eval "$external_policy_protocol_function"
+ external_egress_policy_converged_with_protocol UDP &&
+ ! external_egress_policy_converged_with_protocol TCP
+); then
+ pass "external-egress convergence binds the exact current protocol"
+else
+ fail "external-egress convergence can accept a stale protocol"
+fi
+
+mcpserver_current_ready_function="$(
+ sed -n '/^mcpserver_current_ready() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+mcpserver_current_ready_absent_function="$(
+ sed -n '/^mcpserver_current_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+fixture_runtime_and_ready_absent_function="$(
+ sed -n '/^fixture_runtime_and_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+fixture_runtime_converged_function="$(
+ sed -n '/^fixture_runtime_converged_with_protocol() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+peer_fleet_converged_function="$(
+ sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+fixture_converged_function="$(
+ sed -n '/^fixture_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+if (
+ MCP_NS=mcp-server
+ GENERATION=7
+ OBSERVED_GENERATION=7
+ kctl() {
+ printf '{"metadata":{"generation":%s},"status":{"conditions":[{"type":"Ready","status":"True","observedGeneration":%s}]}}\n' \
+ "$GENERATION" "$OBSERVED_GENERATION"
+ }
+ eval "$mcpserver_current_ready_function"
+ eval "$mcpserver_current_ready_absent_function"
+ mcpserver_current_ready current &&
+ ! mcpserver_current_ready_absent current &&
+ OBSERVED_GENERATION=6 &&
+ ! mcpserver_current_ready stale &&
+ mcpserver_current_ready_absent stale
+) &&
+ [[ "$fixture_runtime_and_ready_absent_function" == *'mcpserver_current_ready_absent "$MCP_NAME"'* ]] &&
+ [[ "$fixture_runtime_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]] &&
+ [[ "$peer_fleet_converged_function" == *'mcpserver_current_ready "$server"'* ]] &&
+ [[ "$fixture_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]]; then
+ pass "McpServer runtime predicates accept only Ready status observed for the current generation"
+else
+ fail "McpServer runtime predicates can accept stale Ready status or bypass the canonical check"
+fi
+
+if (
+ MCP_NS=mcp-server
+ MCP_NAME=e2e-held-server
+ fixture_runtime_absent() { return 0; }
+ kctl() { return 1; }
+ eval "$mcpserver_current_ready_absent_function"
+ eval "$fixture_runtime_and_ready_absent_function"
+ ! fixture_runtime_and_ready_absent
+); then
+ pass "McpServer Ready absence fails closed when the API read is unavailable"
+else
+ fail "an unavailable McpServer API read can masquerade as Ready absence"
+fi
+
+if (
+ probe_new_hcc_ready_endpoint() { return 0; }
+ external_egress_policy_absent() { return 0; }
+ eval "$hcc_ready_after_revoke_function"
+ hcc_ready_after_stale_policy_revoked
+); then
+ pass "divergent DNS hold permits global readiness only after stale-policy revocation"
+else
+ fail "divergent DNS hold rejects globally safe readiness evidence"
+fi
+if (
+ HCC_READY_PROBE_DIAGNOSTIC=""
+ probe_new_hcc_ready_endpoint() { return 0; }
+ external_egress_policy_absent() { return 1; }
+ eval "$hcc_ready_after_revoke_function"
+ rc=0
+ hcc_ready_after_stale_policy_revoked || rc=$?
+ # A 200 observed WITH the stale policy present is a fatal ordering violation
+ # (rc=2), not a benign retryable "not ready" (rc=1).
+ [ "$rc" -eq 2 ] &&
+ [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=ready-200-with-stale-policy-present' ]
+); then
+ pass "global readiness evidence flags a surviving divergent policy as a fatal rc=2 ordering violation"
+else
+ fail "global readiness can self-confirm (or only soft-fail) while a divergent policy survives"
+fi
+# The loop must PROPAGATE the fatal rc=2 immediately, not retry until a later
+# clean poll turns the violation into a false PASS (the round-2 bug).
+wait_until_fast_function="$(
+ sed -n '/^wait_until_fast() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+if (
+ HCC_READY_PROBE_DIAGNOSTIC=""
+ POLL=0
+ probe_new_hcc_ready_endpoint() { return 0; }
+ # Stale policy present for the first polls, revoked only later. The buggy
+ # retry loop would poll past the fatal first observation and PASS here.
+ external_egress_policy_absent() { POLL=$((POLL + 1)); [ "$POLL" -gt 4 ]; }
+ eval "$hcc_ready_after_revoke_function"
+ eval "$wait_until_fast_function"
+ rc=0
+ wait_until_fast 10 "rc=2 propagation" hcc_ready_after_stale_policy_revoked || rc=$?
+ # rc must be the fatal 2, and it must have stopped inside the first predicate
+ # poll (which calls external_egress_policy_absent twice) — POLL<3 proves no
+ # retry occurred.
+ [ "$rc" -eq 2 ] && [ "$POLL" -lt 3 ]
+); then
+ pass "wait_until_fast propagates a fatal rc=2 on the first poll instead of retrying to a false PASS"
+else
+ fail "wait_until_fast retried past a fatal ordering violation to a later clean poll"
+fi
+
+# M6: the third contract signal adjudicates a real /metrics scrape. Extract the
+# PURE predicate (the wrapper does kctl exec, untestable here) and feed it
+# synthetic scrapes. The family-name literals are exactly what a metric rename
+# would break — a rename leaves the vi.fn() unit tests green but must turn this
+# and the real gate red.
+safety_pass_metrics_predicate="$(
+ sed -n '/^safety_pass_metrics_are_certified() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+[ -n "$safety_pass_metrics_predicate" ] ||
+ fail "could not extract safety_pass_metrics_are_certified from the readiness gate"
+metrics_with_revocation='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1
+clerum_hcc_networkpolicy_safety_pass_policies_total{operation="listed"} 5
+clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 1'
+metrics_without_family='clerum_hcc_some_other_family_total 3'
+metrics_zero_revoked='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1
+clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 0'
+if (
+ eval "$safety_pass_metrics_predicate"
+ safety_pass_metrics_are_certified "$metrics_with_revocation"
+); then
+ pass "safety_pass metrics predicate certifies a scrape with >=1 revoked policy"
+else
+ fail "safety_pass metrics predicate rejected a valid certifying scrape"
+fi
+if (
+ eval "$safety_pass_metrics_predicate"
+ ! safety_pass_metrics_are_certified "$metrics_without_family"
+); then
+ pass "safety_pass metrics predicate rejects a scrape missing the duration family (rename guard)"
+else
+ fail "safety_pass metrics predicate passed with the safety_pass family absent"
+fi
+if (
+ eval "$safety_pass_metrics_predicate"
+ ! safety_pass_metrics_are_certified "$metrics_zero_revoked"
+); then
+ pass "safety_pass metrics predicate rejects a scrape with zero revoked policies"
+else
+ fail "safety_pass metrics predicate passed with zero policies revoked"
+fi
+if (
+ probe_new_hcc_ready_endpoint() { return 1; }
+ external_egress_policy_absent() { return 0; }
+ eval "$hcc_ready_after_revoke_function"
+ ! hcc_ready_after_stale_policy_revoked
+); then
+ pass "stale-policy revocation alone cannot substitute for global readiness"
+else
+ fail "divergent hold evidence can pass without HCC readiness"
+fi
+
+baseline_policy_snapshot_function="$(
+ sed -n '/^baseline_policy_snapshot() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+baseline_policies_unchanged_function="$(
+ sed -n '/^baseline_policies_unchanged() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+valid_baseline_policies='{"items":[
+ {"metadata":{"name":"deny-all-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"default-deny"}},"spec":{"podSelector":{},"policyTypes":["Ingress","Egress"]}},
+ {"metadata":{"name":"allow-dns-egress-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"infrastructure"}},"spec":{"podSelector":{},"policyTypes":["Egress"],"egress":[{"ports":[{"port":53,"protocol":"UDP"},{"port":53,"protocol":"TCP"}]}]}},
+ {"metadata":{"name":"allow-host-context-controller-api","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"allow-api"}},"spec":{"podSelector":{"matchLabels":{"app":"host-context-controller"}},"policyTypes":["Ingress"]}}
+]}'
+if (
+ MCP_NS=mcp-server
+ MANAGED_BY_VALUE='host-context-controller'
+ MOCK_POLICY_JSON="$valid_baseline_policies"
+ kctl() { printf '%s\n' "$MOCK_POLICY_JSON"; }
+ eval "$baseline_policy_snapshot_function"
+ eval "$baseline_policies_unchanged_function"
+ BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)"
+ baseline_policies_unchanged &&
+ MOCK_POLICY_JSON="$(jq -c '
+ .items[0].metadata.labels["clerum.io/managed-by"] = "foreign-controller"
+ ' <<<"$valid_baseline_policies")" &&
+ ! baseline_policies_unchanged &&
+ MOCK_POLICY_JSON="$(jq -c '.items[1].spec.egress[0].ports[0].port = 5353' \
+ <<<"$valid_baseline_policies")" &&
+ ! baseline_policies_unchanged
+); then
+ pass "baseline safety proof rejects ownership and spec drift under stable policy names"
+else
+ fail "baseline safety proof can pass after ownership or spec drift"
+fi
+
+peer_fleet_policies_absent_function="$(
+ sed -n '/^peer_fleet_policies_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+fixture_resources_absent_function="$(
+ sed -n '/^fixture_resources_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+delete_peer_fleet_runtime_function="$(
+ sed -n '/^delete_peer_fleet_runtime() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+if (
+ MCP_NS=mcp-server
+ HOST_NS=mcp-host
+ RPC_PROXY_NS=rpc-proxy
+ MCP_SERVER_LABEL=clerum.io/mcpserver
+ POLICY_TYPE_LABEL=clerum.io/policy-type
+ MCP_FLEET_NAMES=(primary peer)
+ CONTEXT_FLEET_NAMES=(primary-context peer-context)
+ SURVIVOR_NS=""
+ resource_absent() { [ "$3" != "$SURVIVOR_NS" ]; }
+ kctl() { return 0; }
+ eval "$peer_fleet_policies_absent_function"
+ peer_fleet_policies_absent &&
+ SURVIVOR_NS=mcp-host &&
+ ! peer_fleet_policies_absent &&
+ SURVIVOR_NS=rpc-proxy &&
+ ! peer_fleet_policies_absent
+) &&
+ [[ "$fixture_resources_absent_function" == *'peer_fleet_policies_absent'* ]] &&
+ [[ "$delete_peer_fleet_runtime_function" == *'"${context_policy}-egress"'* ]] &&
+ [[ "$delete_peer_fleet_runtime_function" == *'"rpc-egress-${context}-${server}"'* ]]; then
+ pass "peer fleet cleanup rejects surviving mcp-host or rpc-proxy policies"
+else
+ fail "peer fleet cleanup can release ownership with residual derived policies"
+fi
+
# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh
source "$LOG_HELPER"
HCC_LOG_BUFFER="$MOCK_LOG_FILE"
START_TIME=2026-07-14T00:00:00Z
HOST_NS=mcp-host
+blocker_holds_function="$(sed -n '/^blocker_holds_token_request() {$/,/^}$/p' "$READINESS_GATE")"
+blocker_count_function="$(sed -n '/^blocker_fixture_request_count() {$/,/^}$/p' "$READINESS_GATE")"
+if (
+ BLOCKER_NAME=readiness-blocker
+ HCC_NS=control-plane
+ FIXTURE_HOST_PREFIX=e2e-hcc-ready-test
+ TOKEN_REQUEST_LOG_PREFIX='holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host='
+ kctl() { cat "$MOCK_LOG_FILE"; }
+ eval "$blocker_holds_function"
+ eval "$blocker_count_function"
+ printf '%s\n' \
+ 'holding-token GET /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-01' \
+ 'holding-token POST /wrong/path host=e2e-hcc-ready-test-02' \
+ 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \
+ 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \
+ 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-04' \
+ >"$MOCK_LOG_FILE"
+ [ "$(blocker_fixture_request_count)" = 2 ] &&
+ blocker_holds_token_request
+); then
+ pass "blocker evidence counts unique fixtures only on the canonical token issuance method and path"
+else
+ fail "wrong-method, wrong-path, or duplicate blocker traffic can satisfy the fleet evidence"
+fi
+
+create_fixtures_function="$(sed -n '/^create_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"
+if (
+ RUN_ID=fixture-failure
+ SUITE_NAME=hcc-readiness-bootstrap
+ HOST_NS=mcp-host
+ MCP_NS=mcp-server
+ FIXTURE_SECRET=e2e-secret
+ FIXTURE_CONTEXT=e2e-context
+ FIXTURE_HOST_PREFIX=e2e-host
+ FIXTURE_HOST_COUNT=4
+ FIXTURE_HOST_NAMES=()
+ FIXTURES_CREATED=0
+ KCTL_APPLY_COUNT=0
+ kctl() {
+ KCTL_APPLY_COUNT=$((KCTL_APPLY_COUNT + 1))
+ cat >/dev/null
+ [ "$KCTL_APPLY_COUNT" -ne 3 ]
+ }
+ eval "$create_fixtures_function"
+ if create_host_fixtures; then
+ exit 1
+ fi
+ [ "$KCTL_APPLY_COUNT" = 3 ]
+); then
+ pass "an intermediate Host apply failure aborts fixture creation even from an OR-list"
+else
+ fail "fixture creation can continue or report success after an intermediate apply failure"
+fi
+
+bootstrap_runtime_absent_function="$(
+ sed -n '/^fixture_runtime_absent() {$/,/^}$/p' "$READINESS_GATE"
+)"
+bootstrap_delete_fixtures_function="$(
+ sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE"
+)"
+# Literal source-code assertions for macOS bash 3.2, where expanding an empty
+# array under set -u terminates the shell before cleanup can release its lock.
+# shellcheck disable=SC2016
+if [[ "$bootstrap_runtime_absent_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0'* ]] &&
+ [[ "$bootstrap_delete_fixtures_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"'* ]]; then
+ pass "bootstrap cleanup guards empty fixture arrays before bash 3.2 expansion"
+else
+ fail "bootstrap cleanup can abort on an empty fixture array before lock finalization"
+fi
+
+create_peer_fleet_function="$(
+ sed -n '/^create_peer_fleet() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+peer_flag_line="$(grep -nF ' PEER_FLEET_CREATED=1' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"
+peer_loop_line="$(grep -nF ' for ((index = 1;' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"
+if [ -n "$peer_flag_line" ] && [ -n "$peer_loop_line" ] &&
+ [ "$peer_flag_line" -lt "$peer_loop_line" ]; then
+ pass "peer fleet cleanup is armed before the first partial apply can fail"
+else
+ fail "peer fleet cleanup is armed only after all applies complete"
+fi
+
+divergent_ready_function="$(
+ sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+divergent_ready_line="$(
+ grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+divergent_context_line="$(
+ grep -nF 'wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+divergent_release_line="$(
+ grep -nF 'release_dns_blocker ||' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1
+)"
+udp_policy_line="$(
+ grep -nF 'wait_until 120 "current UDP external-egress policy after DNS release"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+# shellcheck disable=SC2016
+if [[ "$divergent_ready_function" == *'probe_new_hcc_ready_endpoint || return 1'* ]] &&
+ [[ "$divergent_ready_function" == *'external_egress_policy_absent'* ]] &&
+ [ -n "$divergent_ready_line" ] && [ -n "$divergent_context_line" ] &&
+ [ -n "$divergent_release_line" ] && [ -n "$udp_policy_line" ] &&
+ [ "$divergent_ready_line" -lt "$divergent_context_line" ] &&
+ [ "$divergent_context_line" -lt "$divergent_release_line" ] &&
+ [ "$divergent_release_line" -lt "$udp_policy_line" ] &&
+ ! grep -Fq 'probe_new_hcc_unready_endpoint' "$MCP_READINESS_GATE" &&
+ ! grep -Fq '/ready is 503' "$MCP_READINESS_GATE"; then
+ pass "divergent DNS hold proves global readiness, fail-closed runtime isolation, and post-release UDP convergence"
+else
+ fail "the MCP gate still couples global readiness to divergent DNS replacement"
+fi
+
+initial_tcp_policy_line="$(
+ grep -nF 'wait_until 180 "HCC to create the real initial TCP external-egress policy"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+# B3 retain-identity-stable contract: a same-intent restart RETAINS the
+# identity-stable DNS-derived allow (no deny window), certifies /ready with it
+# still present, keeps the affected runtime blocked, and converges the additive
+# TCP lane only AFTER the DNS hold releases — the inverse of the pre-B3
+# revoke-before-ready ordering this gate previously asserted.
+same_intent_snapshot_line="$(
+ grep -nF 'SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)"' \
+ "$MCP_READINESS_GATE" | head -1 | cut -d: -f1
+)"
+same_intent_retained_ready_line="$(
+ grep -nF 'wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+same_intent_retain_guard_line="$(
+ grep -nF 'the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+same_intent_runtime_blocked_line="$(
+ grep -nF 'the affected runtime or Ready status escaped the held same-intent DNS boundary' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+same_intent_release_line="$(
+ grep -nF 'release_dns_blocker || die "could not release the same-intent DNS query"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+same_intent_tcp_policy_line="$(
+ grep -nF 'wait_until 120 "current TCP external-egress policy after same-intent DNS release"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+divergent_revoke_line="$(
+ grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \
+ "$MCP_READINESS_GATE" | cut -d: -f1
+)"
+if [ -n "$initial_tcp_policy_line" ] && [ -n "$same_intent_snapshot_line" ] &&
+ [ -n "$same_intent_retained_ready_line" ] && [ -n "$same_intent_retain_guard_line" ] &&
+ [ -n "$same_intent_runtime_blocked_line" ] && [ -n "$same_intent_release_line" ] &&
+ [ -n "$same_intent_tcp_policy_line" ] && [ -n "$divergent_revoke_line" ] &&
+ [ "$initial_tcp_policy_line" -lt "$same_intent_snapshot_line" ] &&
+ [ "$same_intent_snapshot_line" -lt "$same_intent_retained_ready_line" ] &&
+ [ "$same_intent_retained_ready_line" -lt "$same_intent_retain_guard_line" ] &&
+ [ "$same_intent_retain_guard_line" -lt "$same_intent_runtime_blocked_line" ] &&
+ [ "$same_intent_runtime_blocked_line" -lt "$same_intent_release_line" ] &&
+ [ "$same_intent_release_line" -lt "$same_intent_tcp_policy_line" ] &&
+ [ "$same_intent_tcp_policy_line" -lt "$divergent_revoke_line" ] &&
+ grep -Fq 'hcc_ready_with_identity_stable_policy_retained' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'HCC readiness after revoking the same-intent DNS policy' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'a real same-intent external-egress retry to be scheduled' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'CERTIFIED_POLICY_IDENTITY' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'external_egress_policy_identity' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'UID/spec' "$MCP_READINESS_GATE"; then
+ pass "same-intent safety retains the identity-stable DNS policy at readiness; additive TCP convergence follows DNS release"
+else
+ fail "the MCP readiness gate no longer proves B3 retain-identity-stable (or regressed to revoke-same-intent before readiness)"
+fi
+
printf '%s\n' \
'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
@@ -41,6 +1329,30 @@ else
fail "valid LIST-to-recovery ordering was rejected"
fi
+printf '%s\n' \
+ '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] Listing all Hosts in namespace mcp-host' \
+ '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
+ '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \
+ '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"
+if recovery_cycle_used_fresh_host_inventory 1 5; then
+ pass "a causally ordered Host-watch pass may cover the recovered CC and fresh Host inventories"
+else
+ fail "a valid covering Host-watch recovery pass was rejected"
+fi
+
+printf '%s\n' \
+ '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
+ '[K8s] Listing all Hosts in namespace mcp-host' \
+ '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \
+ '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
+ '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"
+if recovery_cycle_used_fresh_host_inventory 1 5; then
+ fail "a Host-watch pass selected before CC recovery satisfied the covering assertion"
+else
+ pass "covering Host-watch recovery requires recovered CC state before pass selection"
+fi
+
printf '%s\n' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
@@ -82,19 +1394,272 @@ else
fail "the second recovery cycle was not isolated from the first"
fi
-acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock || die' "$GATE" | cut -d: -f1)"
+large_log_tail="$(awk 'BEGIN {
+ for (i = 1; i <= 20000; i++) {
+ printf "trailing-log-line-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n", i
+ }
+}')"
+large_active_snapshot="${START_MARKER:-Starting initial Host background convergence}
+${large_log_tail}"
+if hcc_log_snapshot_contains \
+ "$large_active_snapshot" 'Starting initial Host background convergence'; then
+ pass "buffered marker lookup survives a log tail larger than a pipe buffer"
+else
+ fail "large trailing logs can hide a marker from buffered lookup"
+fi
+
+if hcc_initial_pass_snapshot_is_active \
+ "$large_active_snapshot" \
+ 'Starting initial Host background convergence' \
+ 'Completed Host reconciliation after initial Host reconciliation' \
+ 'Host reconciliation after initial Host reconciliation failed'; then
+ pass "Host pass snapshot is active only with START and no terminal marker"
+else
+ fail "valid active Host pass snapshot was rejected"
+fi
+
+for terminal_marker in \
+ 'Completed Host reconciliation after initial Host reconciliation' \
+ 'Host reconciliation after initial Host reconciliation failed'; do
+ if hcc_initial_pass_snapshot_is_active \
+ "${large_active_snapshot}
+${terminal_marker}" \
+ 'Starting initial Host background convergence' \
+ 'Completed Host reconciliation after initial Host reconciliation' \
+ 'Host reconciliation after initial Host reconciliation failed'; then
+ fail "terminal Host marker '${terminal_marker}' can still satisfy the active-pass guard"
+ else
+ pass "terminal Host marker '${terminal_marker}' invalidates the active-pass guard"
+ fi
+done
+
+readiness_log_function="$(
+ sed -n '/^hcc_log_contains() {$/,/^}$/p' "$READINESS_GATE"
+)"
+running_pod_function="$(
+ sed -n '/^running_hcc_pod() {$/,/^}$/p' "$READINESS_GATE"
+)"
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if [[ "$readiness_log_function" == *'logs="$(hcc_logs "$pod")"'* ]] &&
+ [[ "$readiness_log_function" == *'hcc_log_snapshot_contains "$logs" "$marker"'* ]] &&
+ [[ "$running_pod_function" == *'rows="$(kctl get pods'* ]] &&
+ [[ "$running_pod_function" == *'<<<"$rows"'* ]]; then
+ pass "readiness log and Running-pod selection buffer producers before early-exit consumers"
+else
+ fail "readiness log or Running-pod selection reintroduced a pipefail/SIGPIPE path"
+fi
+
+active_guard_function="$(
+ sed -n '/^initial_host_pass_is_active() {$/,/^}$/p' "$READINESS_GATE"
+)"
+initial_host_startup_producer="$(
+ sed -n \
+ '/Starting initial Host background convergence/,/const resyncSec = config.hostResyncIntervalSec/p' \
+ "$HCC_K8S_CLIENT"
+)"
+# The bootstrap guard's negative COMPLETE/FAIL checks are safe only while all
+# three consumed markers remain bound to the runtime templates and the startup
+# call supplies the exact reason that renders the terminal pair.
+if grep -Fq 'Starting initial Host background convergence...' \
+ "$HCC_K8S_CLIENT" &&
+ grep -Fq 'Completed Host reconciliation after ${reason}' \
+ "$HCC_K8S_CLIENT" &&
+ grep -Fq 'Host reconciliation after ${reason} failed:' \
+ "$HCC_K8S_CLIENT" &&
+ [[ "$initial_host_startup_producer" == *'await this.recoverHostInventoryAndWatch('* ]] &&
+ [[ "$initial_host_startup_producer" == *"'initial Host reconciliation',"* ]]; then
+ pass "bootstrap START/COMPLETE/FAIL markers remain bound to the initial Host producer reason"
+else
+ fail "bootstrap Host pass markers drifted from their runtime producer or reason binding"
+fi
+
+# The destructive Host-bundle measurement uses the same cold-start transition
+# as the bootstrap readiness gate. Keep its active-pass marker tied to the
+# emitted producer rather than an obsolete reason-string log line.
+if grep -Fq "readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'" \
+ "$HOST_BUNDLE_MEASURE" &&
+ grep -Fq 'Starting initial Host background convergence...' "$HCC_K8S_CLIENT"; then
+ pass "Host-bundle measurement START marker remains bound to the cold-start producer"
+else
+ fail "Host-bundle measurement can wait for a stale HCC START marker"
+fi
+# shellcheck disable=SC2016
+probe_result_line="$(grep -nF 'probe_result="$(kctl exec' "$READINESS_GATE" | cut -d: -f1)"
+# shellcheck disable=SC2016
+final_active_guard_line="$(
+ grep -nF 'initial_host_pass_is_active "$new_hcc_pod"' "$READINESS_GATE" |
+ tail -1 |
+ cut -d: -f1
+)"
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if [[ "$active_guard_function" == *'"$COMPLETE_MARKER"'* ]] &&
+ [[ "$active_guard_function" == *'"$FAIL_MARKER"'* ]] &&
+ [ -n "$probe_result_line" ] &&
+ [ -n "$final_active_guard_line" ] &&
+ [ "$final_active_guard_line" -gt "$probe_result_line" ]; then
+ pass "final readiness assertion rejects both failed and completed Host passes"
+else
+ fail "final readiness assertion does not enforce both terminal Host markers"
+fi
+
+cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$READINESS_GATE")"
+fixture_cleanup_body="$(sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"
+# shellcheck disable=SC2016
+cleanup_stop_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0' \
+ <<<"$cleanup_body" | head -1 | cut -d: -f1)"
+cleanup_fixture_line="$(grep -nF 'delete_host_fixtures' <<<"$cleanup_body" | head -1 | cut -d: -f1)"
+# shellcheck disable=SC2016
+cleanup_restore_line="$(grep -nF 'kctl set env deployment/"$HCC_DEPLOY"' \
+ <<<"$cleanup_body" | head -1 | cut -d: -f1)"
# Literal source-code assertion.
# shellcheck disable=SC2016
-mutation_line="$(grep -nF 'kctl get deployment "$HCC_DEPLOY"' "$GATE" | head -1 | cut -d: -f1)"
+if grep -Fq 'FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))' "$READINESS_GATE" &&
+ grep -Fq 'kind: Host' "$READINESS_GATE" &&
+ grep -Fq "host='+host" "$READINESS_GATE" &&
+ grep -Fq 'blocker_fixture_request_count' "$READINESS_GATE" &&
+ [[ "$fixture_cleanup_body" == *'clerum.io/managed-by=host-context-controller,clerum.io/host=${host}'* ]] &&
+ [[ "$fixture_cleanup_body" == *'fixture_resources_absent'* ]] &&
+ ! grep -Fq 'RUNTIME_SECRET' "$READINESS_GATE" &&
+ ! grep -Fq 'HOST_REF' "$READINESS_GATE" &&
+ [ -n "$cleanup_stop_line" ] &&
+ [ -n "$cleanup_fixture_line" ] &&
+ [ -n "$cleanup_restore_line" ] &&
+ [ "$cleanup_stop_line" -lt "$cleanup_fixture_line" ] &&
+ [ "$cleanup_fixture_line" -lt "$cleanup_restore_line" ]; then
+ pass "readiness gate uses a multi-wave owned Host fleet and deletes it while HCC is stopped"
+else
+ fail "readiness gate can mutate a real Host Secret or restore HCC before fixture deletion"
+fi
+
+if [[ "$cleanup_body" == *'header "HCC readiness bootstrap gate passed"'* ]] &&
+ grep -Fq 'header "HCC readiness assertions passed; restoring branch-owned runtime"' \
+ "$READINESS_GATE" &&
+ [ "$(grep -Fc 'header "HCC readiness bootstrap gate passed"' "$READINESS_GATE")" = 1 ]; then
+ pass "readiness gate emits its final pass banner only after verified restoration and lock finalization"
+else
+ fail "readiness gate can announce a final pass before cleanup and ownership release finish"
+fi
+
+mcp_cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$MCP_READINESS_GATE")"
+# shellcheck disable=SC2016
+mcp_input_absence_line="$(
+ grep -nF 'fixture_inputs_absent' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
+)"
+# shellcheck disable=SC2016
+mcp_restore_line="$(
+ grep -nF 'restore_patch="$(jq -cn' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
+)"
+if [[ "$mcp_cleanup_body" == *'fixture_inputs_removed=0'* ]] &&
+ [[ "$mcp_cleanup_body" == *'if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then'* ]] &&
+ [ -n "$mcp_input_absence_line" ] &&
+ [ -n "$mcp_restore_line" ] &&
+ [ "$mcp_input_absence_line" -lt "$mcp_restore_line" ]; then
+ pass "MCP readiness cleanup keeps HCC stopped until fixture input deletion is verified"
+else
+ fail "MCP readiness cleanup can restart HCC after an unverified Context or McpServer deletion"
+fi
+
+mcp_finalize_line="$(
+ grep -nF 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' \
+ <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
+)"
+mcp_pass_line="$(
+ grep -nF 'header "HCC McpServer/Context readiness gate passed"' \
+ <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
+)"
+if [ "$(grep -Fc 'header "HCC McpServer/Context readiness gate passed"' \
+ "$MCP_READINESS_GATE")" = 1 ] &&
+ [ -n "$mcp_finalize_line" ] &&
+ [ -n "$mcp_pass_line" ] &&
+ [ "$mcp_finalize_line" -lt "$mcp_pass_line" ]; then
+ pass "MCP readiness gate announces PASS only after cleanup and lock finalization"
+else
+ fail "MCP readiness gate can announce PASS before cleanup and lock finalization"
+fi
+
+wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$READINESS_GATE")"
# Literal source-code assertion.
# shellcheck disable=SC2016
-if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] && [ "$acquire_line" -lt "$mutation_line" ] &&
- grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1' "$GATE"; then
- pass "gate acquires before HCC access and finalizes ownership from cleanup"
+if [[ "$wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&
+ ! grep -Fq -- '--for=delete' "$READINESS_GATE" &&
+ ! grep -Fq 'port: 8081' "$READINESS_GATE" &&
+ grep -Fq 'HCC_E2E_PORT' "$READINESS_GATE"; then
+ pass "readiness gate uses wall-clock deadlines, absence waits, and the deployed HCC port"
+else
+ fail "readiness gate retains a command-latency, zero-pod wait, or hard-coded-port edge"
+fi
+
+mcp_wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$MCP_READINESS_GATE")"
+mcp_port_probe_count="$(grep -Fc 'env "HCC_E2E_PORT=${HCC_PORT}"' "$MCP_READINESS_GATE")"
+mcp_pod_absence_count="$(grep -Fc 'hcc_pods_absent' "$MCP_READINESS_GATE")"
+# Literal source-code assertions.
+# shellcheck disable=SC2016
+if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \
+ "$MCP_READINESS_GATE" &&
+ [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$MCP_READINESS_GATE")" = 1 ] &&
+ ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'cluster_fingerprint_file=' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'profile_env=' "$MCP_READINESS_GATE" &&
+ [[ "$mcp_wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&
+ grep -Fq 'hcc_pods_absent' "$MCP_READINESS_GATE" &&
+ ! grep -Fq -- '--for=delete' "$MCP_READINESS_GATE" &&
+ ! grep -Fq 'port: 8081' "$MCP_READINESS_GATE" &&
+ [ "$mcp_port_probe_count" = 4 ] &&
+ [ "$mcp_pod_absence_count" = 4 ]; then
+ pass "MCP readiness gate shares exact-head ownership and uses bounded portable runtime probes"
+else
+ fail "MCP readiness gate duplicates ownership or retains a deadline, pod-absence, or port edge"
+fi
+
+# A single held McpServer proves the minimal failure, but this gate must also
+# cross the controller's bounded worker width with independent, gate-owned
+# McpServer/Context pairs. Keep this as a static contract so future edits do
+# not silently reduce the fleet scenario back to one fixture. The fixture
+# must exist while HCC is stopped, otherwise it would not exercise startup.
+mcp_peer_create_line="$(grep -nF 'create_peer_fleet || die' "$MCP_READINESS_GATE" | cut -d: -f1)"
+mcp_restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1)"
+peer_fleet_converged_body="$(
+ sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"
+)"
+if grep -Fq 'MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"' "$MCP_READINESS_GATE" &&
+ grep -Fq '[ "$MCP_FLEET_SIZE" -gt 10 ]' "$MCP_READINESS_GATE" &&
+ grep -Fq 'create_peer_fleet' "$MCP_READINESS_GATE" &&
+ grep -Fq 'peer_fleet_converged' "$MCP_READINESS_GATE" &&
+ [[ "$peer_fleet_converged_body" == *'mcpserver_current_ready "$server"'* ]] &&
+ [[ "$peer_fleet_converged_body" == *'context_policies_converged "$context" "$context" "$server"'* ]] &&
+ grep -Fq 'all peer MCP/Context runtimes to converge after releasing the held primary egress' "$MCP_READINESS_GATE" &&
+ [ -n "$mcp_peer_create_line" ] && [ -n "$mcp_restart_line" ] &&
+ [ "$mcp_peer_create_line" -lt "$mcp_restart_line" ]; then
+ pass "MCP readiness gate crosses the bounded worker width with gate-owned MCP and Context peers"
else
- fail "gate lock is not held across the full HCC fault-injection window"
+ fail "MCP readiness gate does not prove a fleet larger than the worker width"
fi
+for gate in "${GATES[@]}"; do
+ acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock' "$gate" | tail -1 | cut -d: -f1)"
+ case "$gate" in
+ "$WATCH_GATE")
+ # Literal source-code assertion.
+ # shellcheck disable=SC2016
+ mutation_line="$(grep -nF 'kctl patch deployment "$HCC_DEPLOY"' "$gate" | tail -1 | cut -d: -f1)"
+ ;;
+ *)
+ mutation_line="$(grep -nF 'HCC_MUTATED=1' "$gate" | tail -1 | cut -d: -f1)"
+ ;;
+ esac
+ # The grep pattern intentionally matches literal shell variable references.
+ # shellcheck disable=SC2016
+ if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] &&
+ [ "$acquire_line" -lt "$mutation_line" ] &&
+ grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' "$gate"; then
+ pass "$(basename "$gate") acquires before mutation and finalizes ownership from cleanup"
+ else
+ fail "$(basename "$gate") does not hold the shared lock across its full fault-injection window"
+ fi
+done
+
kctl() {
local action=$1 body expected_rv current_rv current_uid next_rv
case "$action" in
@@ -138,7 +1703,7 @@ truncate_rfc1123() { printf '%.63s' "$1"; }
source "$LOCK_HELPER"
HCC_NS=control-plane
-HCC_DEPLOY=host-context-controller
+HCC_DEPLOY="host-context-controller"
E2E_KUBECONTEXT=clerum-codex-lock-test-1234abcd
RUN_ID=owner-1
HCC_GATE_LOCK_ACQUIRED=0
@@ -253,12 +1818,62 @@ else
fi
unset MOCK_CREATE_WITHOUT_UID
-cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$GATE" | cut -d: -f1)"
-results_line="$(grep -nF ' print_results' "$GATE" | head -1 | cut -d: -f1)"
+cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$WATCH_GATE" | cut -d: -f1)"
+results_line="$(grep -nF ' print_results' "$WATCH_GATE" | head -1 | cut -d: -f1)"
if [ -n "$cleanup_fail_line" ] && [ -n "$results_line" ] && [ "$cleanup_fail_line" -lt "$results_line" ]; then
pass "cleanup failure is counted before the final result summary"
else
fail "cleanup can still print a false all-passed summary"
fi
+# --- H3: SIGPIPE/pipefail sweep on HCC pass-marker log probes -------------
+# A `printf '%s[\n]' "$logs" | grep -Fq "$..._MARKER"` probe is a false-green
+# trap: on a large log with the marker near the top, `grep -Fq` exits early,
+# `printf` takes SIGPIPE, and under `set -o pipefail` the whole pipeline reports
+# failure — so a PRESENT marker reads as MISSING (deterministic 3/3 repro in the
+# round-2 storm gate). The safe idiom is a here-string: no producer to signal.
+# grep -Fq -- "$MARKER" <<<"$logs"
+#
+# Scope: HCC pass-marker gates only (storm + measure). Explicitly OUT OF SCOPE
+# (tracked separately — a different failure class, or a non-HCC gate outside
+# this PR's blast radius):
+# - e2e-gke-secrets.sh:291-294 grep -oE|wc leak COUNT: reads the whole log,
+# no early-exit, so it is NOT the SIGPIPE class.
+# - e2e-plugin-workload-sdk.sh SDK markers (non-HCC).
+# - e2e-lib.sh / e2e-prod-lib.sh / e2e-channel-reader-per-host.sh (non-HCC).
+HCC_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"
+# shellcheck disable=SC2016
+sigpipe_offenders="$(
+ grep -nE 'printf[^|]*\| *grep -Fq[^<]*_MARKER' \
+ "$HCC_STORM_GATE" "$HOST_BUNDLE_MEASURE" 2>/dev/null || true
+)"
+if [ -z "$sigpipe_offenders" ]; then
+ pass "HCC pass-marker probes use here-strings, not pipefail/SIGPIPE pipes"
+else
+ fail "HCC pass-marker probe reintroduced a printf|grep -Fq SIGPIPE path:
+${sigpipe_offenders}"
+fi
+
+# Corroborating repro of the mechanism the sweep guards against. The here-string
+# form MUST find a marker on line 1 of a 1.6 MB log (hard invariant). The pipe
+# form is expected to miss it under pipefail (SIGPIPE) — that miss is the bug,
+# demonstrated, not a standing assertion, so a platform that does not reproduce
+# it is not a suite failure: the static sweep above is the real enforcement.
+h3_marker='HCC_PASS_COMPLETED: h3 repro sentinel'
+h3_bulk="$(yes 'xxxxxxxxxxxxxxxx' | head -c 1600000)"
+h3_log="${h3_marker}
+${h3_bulk}"
+if grep -Fq -- "$h3_marker" <<<"$h3_log"; then
+ pass "here-string probe finds a marker on line 1 of a 1.6 MB log"
+else
+ fail "here-string probe MISSED a present marker on a large log (regression)"
+fi
+h3_pipe_found=0
+( set -o pipefail; printf '%s\n' "$h3_log" | grep -Fq -- "$h3_marker" ) && h3_pipe_found=1
+if [ "$h3_pipe_found" -eq 0 ]; then
+ pass "pipe+pipefail probe misses the same marker (SIGPIPE repro confirmed)"
+else
+ pass "pipe probe did not SIGPIPE on this platform; static sweep still enforces the idiom"
+fi
+
exit "$FAIL"
diff --git a/scripts/tests/test-host-storm-metric-gate.sh b/scripts/tests/test-host-storm-metric-gate.sh
new file mode 100755
index 000000000..e5d994bb5
--- /dev/null
+++ b/scripts/tests/test-host-storm-metric-gate.sh
@@ -0,0 +1,296 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2016 # literal source-contract assertions below
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+GATE="${ROOT}/scripts/e2e/e2e-host-storm-gate.sh"
+WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/host-storm-metric-test.XXXXXX")"
+ADJUDICATOR="${WORK_DIR}/adjudicator.py"
+trap 'rm -rf "$WORK_DIR"' EXIT
+
+fail() {
+ echo "FAIL: $*" >&2
+ exit 1
+}
+
+extract_adjudicator() {
+ awk '
+ /python3 >"\$METRIC_VERDICTS" <<'\''PY'\''/ { capture = 1; next }
+ capture && /^PY$/ { exit }
+ capture { print }
+ ' "$GATE" >"$ADJUDICATOR"
+ [ -s "$ADJUDICATOR" ] || fail "could not extract host-storm metric adjudicator"
+}
+
+write_metrics() {
+ local path=$1 le_05=$2 le_5=$3 le_15=$4 total=$5 fleet_started=${6:-0}
+ local fleet_coalesced=${7:-0} fleet_trailing=${8:-0} fleet_failed=${9:-0}
+ cat >"$path" <<EOF
+# TYPE clerum_hcc_host_delete_cleanup_total counter
+clerum_hcc_host_delete_cleanup_total{outcome="queued"} 0
+clerum_hcc_host_delete_cleanup_total{outcome="confirmed"} 0
+clerum_hcc_host_delete_cleanup_total{outcome="completed"} 0
+clerum_hcc_host_delete_cleanup_total{outcome="retried"} 0
+clerum_hcc_host_delete_cleanup_total{outcome="superseded"} 0
+# TYPE clerum_hcc_host_fleet_requests_total counter
+clerum_hcc_host_fleet_requests_total{result="started"} ${fleet_started}
+clerum_hcc_host_fleet_requests_total{result="coalesced"} ${fleet_coalesced}
+clerum_hcc_host_fleet_requests_total{result="trailing"} ${fleet_trailing}
+clerum_hcc_host_fleet_requests_total{result="failed"} ${fleet_failed}
+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="0.5"} ${le_05}
+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="5"} ${le_5}
+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="15"} ${le_15}
+clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="+Inf"} ${total}
+EOF
+}
+
+extract_adjudicator
+
+for scrape in a b c1 c2 d wake_pre wake_post; do
+ write_metrics "${WORK_DIR}/${scrape}.prom" 0 0 0 0
+done
+
+cat >>"${WORK_DIR}/c1.prom" <<'EOF'
+clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="15"} 1
+clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="+Inf"} 1
+clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="failure",le="+Inf"} 0
+EOF
+
+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 1
+
+# The post-recovery direct-watch window is healthy.
+write_metrics "${WORK_DIR}/d.prom" 10 10 10 10
+
+# The wake window contains one sample over the 5s hard gate plus nineteen
+# fast samples. A combined p95 would be green (19/20 == 95%), but the
+# supported wake journey violated its binding urgent queue budget.
+write_metrics "${WORK_DIR}/wake_post.prom" 19 19 20 20
+
+run_adjudicator() {
+ local wake_post=$1 fleet_completion_proven=${2:-1}
+ SCRAPE_A="${WORK_DIR}/a.prom" \
+ SCRAPE_B="${WORK_DIR}/b.prom" \
+ SCRAPE_C1="${WORK_DIR}/c1.prom" \
+ SCRAPE_C2="${WORK_DIR}/c2.prom" \
+ SCRAPE_D="${WORK_DIR}/d.prom" \
+ SCRAPE_WAKE_PRE="${WORK_DIR}/wake_pre.prom" \
+ SCRAPE_WAKE_POST="$wake_post" \
+ QUEUE_WAIT_METRIC=clerum_hcc_host_reconcile_queue_wait_seconds \
+ WATCH_RECOVERY_METRIC=clerum_hcc_host_watch_recovery_seconds \
+ FLEET_METRIC=clerum_hcc_host_fleet_requests_total \
+ DELETE_CLEANUP_METRIC=clerum_hcc_host_delete_cleanup_total \
+ P95_BUDGET_S=5 \
+ RECOVERY_BUDGET_S=15 \
+ MIN_URGENT_SAMPLES=10 \
+ FLEET_COMPLETION_PROVEN="$fleet_completion_proven" \
+ python3 "$ADJUDICATOR"
+}
+
+OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_post.prom")"
+
+if ! grep -Fq 'VERDICT|A2-direct-watch-urgent-p95|PASS|' <<<"$OUTPUT" &&
+ ! grep -Fq 'VERDICT|A2-urgent-queue-wait-p95|PASS|' <<<"$OUTPUT"; then
+ fail "healthy post-recovery direct-watch window did not pass"
+fi
+grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|FAIL|' <<<"$OUTPUT" ||
+ fail "slow wake window was diluted into a false PASS"
+
+write_metrics "${WORK_DIR}/wake_healthy.prom" 20 20 20 20
+HEALTHY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
+grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|PASS|' <<<"$HEALTHY_OUTPUT" ||
+ fail "healthy wake window did not pass its hard gate"
+
+# A7 is fenced to the replacement pod's explicit initial-fleet COMPLETED log.
+# A second non-overlapping request can therefore be legitimate activity after
+# recovery; an absolute started==1 assertion must not false-red it.
+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2
+TWO_STARTS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$TWO_STARTS_OUTPUT" ||
+ fail "healthy replacement-pod recovery false-reds when a second sequential fleet request starts"
+
+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0
+NO_START_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$NO_START_OUTPUT" ||
+ fail "A7 still treats a scheduled counter as stronger than causal completion evidence"
+
+NO_COMPLETION_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom" 0)"
+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$NO_COMPLETION_OUTPUT" ||
+ fail "A7 false-passed without replacement-pod fleet completion evidence"
+
+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 0 0 1
+FAILED_PASS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$FAILED_PASS_OUTPUT" ||
+ fail "A7 false-passed a failed replacement-pod fleet pass"
+
+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 1 2
+UNBOUNDED_TRAILING_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$UNBOUNDED_TRAILING_OUTPUT" ||
+ fail "A7 false-passed trailing fleet work without a matching coalesced request"
+
+# M8: without the FLEET_METRIC family registered on the running image, counter()
+# returns 0 for every result label and A7 would pass vacuously (all-zero looks
+# like "no failures"). Strip the family's # TYPE line so it drops out of
+# c2_types (mirrors a metric rename / an image that never registered it) and
+# assert A7 FAILs on the presence check, not vacuously passes.
+write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0
+grep -v '^# TYPE clerum_hcc_host_fleet_requests_total ' "${WORK_DIR}/c2.prom" \
+ >"${WORK_DIR}/c2.nofleet.prom"
+mv "${WORK_DIR}/c2.nofleet.prom" "${WORK_DIR}/c2.prom"
+MISSING_FAMILY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
+grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$MISSING_FAMILY_OUTPUT" ||
+ fail "A7 vacuously passed with the FLEET_METRIC family absent from the scrape"
+grep -Fq 'not registered on the running image' <<<"$MISSING_FAMILY_OUTPUT" ||
+ fail "A7 failed for the wrong reason when the FLEET_METRIC family was absent"
+
+WAKE_EVIDENCE_FUNCTION="$(
+ sed -n '/^_wake_urgent_evidence_recorded() {$/,/^}$/p' "$GATE"
+)"
+WAKE_GENERATION_FUNCTION="$(
+ sed -n '/^wake_generation_is_handled() {$/,/^}$/p' "$GATE"
+)"
+POST_WAKE_FUNCTION="$(
+ sed -n '/^post_wake() {$/,/^}$/p' "$GATE"
+)"
+
+[[ -n "$WAKE_GENERATION_FUNCTION" ]] ||
+ fail "runtime wake evidence has no generation-correlation predicate"
+
+# Exercise the exact pure predicate embedded in the E2E gate. The requested
+# generation is the business identifier returned by the public wake route;
+# an older handled generation must never satisfy that request.
+eval "$WAKE_GENERATION_FUNCTION"
+if wake_generation_is_handled 7 6; then
+ fail "wake generation predicate accepted an older handled generation"
+fi
+wake_generation_is_handled 7 7 ||
+ fail "wake generation predicate rejected the exact handled generation"
+wake_generation_is_handled 7 8 ||
+ fail "wake generation predicate rejected a newer handled generation"
+if wake_generation_is_handled 0 8 || wake_generation_is_handled invalid 8 ||
+ wake_generation_is_handled 7 invalid; then
+ fail "wake generation predicate accepted malformed generation evidence"
+fi
+
+# Literal source assertions intentionally match unexpanded shell variables.
+# shellcheck disable=SC2016
+if ! grep -Fq '"$current_pod" == "$HCC_POD_WAKE"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
+ ! grep -Fq '"$lifecycle_state" != '\''suspended'\''' <<<"$WAKE_EVIDENCE_FUNCTION" ||
+ ! grep -Fq 'handled_generation="$(wake_handled_generation)"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
+ ! grep -Fq 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
+ ! grep -Fq 'metric="${IN_FLIGHT_METRIC}"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
+ ! grep -Fq '(after - before) >= 1 && active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then
+ fail "runtime wake evidence is not fenced to the same pod, requested generation, metric delta, and idle urgent lane"
+fi
+generation_check_line="$(
+ grep -nF 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' \
+ <<<"$WAKE_EVIDENCE_FUNCTION" | cut -d: -f1
+)"
+metrics_scrape_line="$(
+ grep -nF 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \
+ <<<"$WAKE_EVIDENCE_FUNCTION" | head -n1 | cut -d: -f1
+)"
+if [[ -z "$generation_check_line" || -z "$metrics_scrape_line" ]] ||
+ (( generation_check_line >= metrics_scrape_line )); then
+ fail "wake generation is not correlated before accepting shared urgent metrics"
+fi
+wake_scrape_count="$(
+ grep -Fc 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \
+ <<<"$WAKE_EVIDENCE_FUNCTION"
+)"
+if (( wake_scrape_count < 2 )) ||
+ ! grep -Fq 'fence_in_flight=' <<<"$WAKE_EVIDENCE_FUNCTION" ||
+ ! grep -Fq 'active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then
+ fail "wake evidence lacks a completed-lane fence followed by a fresh metric scrape"
+fi
+
+# shellcheck disable=SC2016
+if ! grep -Fq 'WAKE_GENERATION=' <<<"$POST_WAKE_FUNCTION" ||
+ ! grep -Fq '"$WAKE_STATUS" == '\''202'\''' <<<"$POST_WAKE_FUNCTION" ||
+ ! grep -Fq '.status // empty' <<<"$POST_WAKE_FUNCTION" ||
+ ! grep -Fq '.wakeGeneration // empty' <<<"$POST_WAKE_FUNCTION"; then
+ fail "public wake response is not validated as an accepted wake-generation contract"
+fi
+
+# A7 must start from a fresh scrape immediately before the kill. Reusing the
+# earlier wake-evidence scrape creates an unobserved interval in which a fleet
+# failure could occur and escape the pre/post recovery counter delta.
+# shellcheck disable=SC2016
+if ! grep -Fq 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" ||
+ grep -Fq 'cp "$SCRAPE_WAKE_POST" "$SCRAPE_B"' "$GATE"; then
+ fail "pre-kill scrape B is not freshly captured after wake adjudication"
+fi
+fresh_scrape_line="$(
+ grep -nF 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" | cut -d: -f1
+)"
+kill_line="$(grep -nF 'T_KILL_MS="$(now_ms)"' "$GATE" | cut -d: -f1)"
+if [[ -z "$fresh_scrape_line" || -z "$kill_line" ]] ||
+ (( fresh_scrape_line >= kill_line )); then
+ fail "fresh scrape B does not precede the HCC kill"
+fi
+
+# The destructive Host storm must use the same exact-HEAD/profile/fingerprint
+# ownership contract and exclusive compare-and-swap lock as every HCC gate.
+# shellcheck disable=SC2016
+for required in \
+ 'source "${SCRIPT_DIR}/e2e-lib.sh"' \
+ 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \
+ 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"' \
+ 'require_branch_owned_hcc_gate "$CONTROL_NS"' \
+ 'acquire_hcc_watch_gate_lock' \
+ 'finalize_hcc_watch_gate_lock'; do
+ grep -Fq "$required" "$GATE" ||
+ fail "Host storm omits shared HCC gate contract: ${required}"
+done
+grep -Fq '.metadata.labels["minikube.k8s.io/name"] == $context' "$GATE" ||
+ fail "Host storm does not bind the node's exact minikube profile label"
+grep -Fq 'MINIKUBE_PROFILE must explicitly select context' \
+ "${ROOT}/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh" ||
+ fail "shared profile ownership helper no longer binds MINIKUBE_PROFILE"
+
+# Branch-owned URLs must come from the selected profile's ports.env. Fixed
+# machine-global service or metrics ports are a cross-worktree false-green risk.
+grep -Fq 'CLERUM_PROFILE_PORTS_ENV' "$GATE" ||
+ fail "Host storm does not load branch-profile port evidence"
+grep -Fq 'profile_env_value EXTERNAL_REST_API_URL' "$GATE" ||
+ fail "Host storm does not derive External REST from branch ports.env"
+grep -Fq 'profile_env_value RPC_PROXY_URL' "$GATE" ||
+ fail "Host storm does not derive RPC proxy from branch ports.env"
+if grep -Eq '127\.0\.0\.1:(8091|8094|18082)' "$GATE"; then
+ fail "Host storm retains a fixed machine-global service or metrics port"
+fi
+grep -Fq '"pod/${pod}" ":${HCC_PORT}"' "$GATE" ||
+ fail "HCC metric scrape no longer asks kubectl for a process-owned random port"
+
+# KEEP_FIXTURES may be useful interactively, but it cannot produce PASS evidence
+# or leave a lock-free mutated profile that could later be mistaken for T2.
+keep_guard_line="$(grep -nF 'KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result' "$GATE" | cut -d: -f1)"
+branch_gate_line="$(grep -nF 'require_branch_owned_hcc_gate "$CONTROL_NS"' "$GATE" | cut -d: -f1)"
+if [[ -z "$keep_guard_line" || -z "$branch_gate_line" ]] ||
+ (( keep_guard_line >= branch_gate_line )); then
+ fail "KEEP_FIXTURES is not rejected before branch verification and mutation"
+fi
+if grep -Fq 'leaving fixtures in place' "$GATE"; then
+ fail "Host storm can still report through the fixture-retention path"
+fi
+
+# Completion must be read from the exact replacement pod and established
+# before C1/C2 metric adjudication. Merely observing `started` is not proof.
+FLEET_COMPLETION_FUNCTION="$(
+ sed -n '/^_replacement_fleet_completed_probe() {$/,/^}$/p' "$GATE"
+)"
+grep -Fq '"$current_pod" == "$HCC_POD_C"' <<<"$FLEET_COMPLETION_FUNCTION" ||
+ fail "fleet completion probe is not fenced to the replacement pod"
+grep -Fq '"pod/${HCC_POD_C}"' <<<"$FLEET_COMPLETION_FUNCTION" ||
+ fail "fleet completion probe reads a deployment aggregate instead of the replacement pod"
+grep -Fq '$HCC_PASS_COMPLETED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||
+ fail "fleet completion probe does not require the explicit COMPLETED marker"
+grep -Fq '$HCC_PASS_FAILED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||
+ fail "fleet completion probe does not fail loud on the explicit FAILED marker"
+completion_wait_line="$(grep -nF '"replacement HCC initial Host fleet reconciliation completion"' "$GATE" | cut -d: -f1)"
+c1_line="$(grep -nF 'scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"' "$GATE" | head -n1 | cut -d: -f1)"
+if [[ -z "$completion_wait_line" || -z "$c1_line" ]] ||
+ (( completion_wait_line >= c1_line )); then
+ fail "replacement fleet completion is not causally established before post-recovery metrics"
+fi
+
+echo "PASS: host-storm oracle binds exact branch/profile ownership, shared fault locking, random endpoints, wake generation, and causal fleet completion"
diff --git a/scripts/tests/test-lint-networkpolicies.sh b/scripts/tests/test-lint-networkpolicies.sh
index 2400d8517..3153dd5dd 100755
--- a/scripts/tests/test-lint-networkpolicies.sh
+++ b/scripts/tests/test-lint-networkpolicies.sh
@@ -3,7 +3,23 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
TMP_DIR="$(mktemp -d)"
-trap 'rm -rf "${TMP_DIR}"' EXIT
+
+# The minikube k8s-api-ip patch is a machine-specific, gitignored file rendered
+# at setup time. On a developer box with a live minikube it may already exist;
+# this test shadows it with a dummy render below. Save the real one (if any) and
+# restore it on EXIT so the test leaves the working tree exactly as it found it.
+MINIKUBE_API_IP_PATCH=""
+MINIKUBE_API_IP_PATCH_BACKUP=""
+MINIKUBE_API_IP_PATCH_PREEXISTING=0
+cleanup() {
+ if [[ "${MINIKUBE_API_IP_PATCH_PREEXISTING}" -eq 1 ]]; then
+ cp "${MINIKUBE_API_IP_PATCH_BACKUP}" "${MINIKUBE_API_IP_PATCH}"
+ elif [[ -n "${MINIKUBE_API_IP_PATCH:-}" && -f "${MINIKUBE_API_IP_PATCH}" ]]; then
+ rm -f "${MINIKUBE_API_IP_PATCH}"
+ fi
+ rm -rf "${TMP_DIR}"
+}
+trap cleanup EXIT
write_policy() {
local path="$1"
@@ -62,7 +78,7 @@ write_policy "${TMP_DIR}/safe-specific.yaml" ' - ports:
protocol: TCP
to:
- ipBlock:
- cidr: 203.0.113.10/32'
+ cidr: 198.51.100.10/32'
run_lint "${TMP_DIR}/safe-specific.yaml" >/dev/null
echo "PASS: lint-networkpolicies allows explicit port plus specific destination"
@@ -135,8 +151,30 @@ ${PUBLIC_EXCEPT_BLOCK}"
run_lint "${TMP_DIR}/public-with-excepts.yaml" >/dev/null
echo "PASS: lint-networkpolicies allows public-only exception set with explicit ports"
-GCP_DEV_RENDERED="${TMP_DIR}/gcp-dev-rendered.yaml"
-kubectl kustomize "${ROOT}/deploy/overlays/gcp-dev" >"${GCP_DEV_RENDERED}"
+# The minikube overlay references patches/k8s-api-ip.yaml, which is rendered
+# from a gitignored template at setup time (deploy/scripts/minikube-detect-k8s-api-ip.sh)
+# and absent on a fresh checkout — so `kubectl kustomize` would abort with a
+# "no such file" evalsymlink error before ever reaching a lint verdict. Render
+# it inline here from the template with a placeholder CIDR, exactly as the
+# single-writer CI step does (.github/workflows/ci-public.yml), so this test
+# exercises the real rendered overlay instead of failing on a missing env file.
+MINIKUBE_API_IP_TEMPLATE="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template"
+MINIKUBE_API_IP_PATCH="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml"
+if [[ ! -f "${MINIKUBE_API_IP_TEMPLATE}" ]]; then
+ echo "FAIL: missing ${MINIKUBE_API_IP_TEMPLATE}; cannot render the minikube k8s-api-ip patch" >&2
+ exit 1
+fi
+if [[ -f "${MINIKUBE_API_IP_PATCH}" ]]; then
+ # A developer's real rendered patch is present — preserve it so the dummy
+ # render below doesn't clobber it; the EXIT trap restores it verbatim.
+ MINIKUBE_API_IP_PATCH_BACKUP="${TMP_DIR}/k8s-api-ip.yaml.orig"
+ cp "${MINIKUBE_API_IP_PATCH}" "${MINIKUBE_API_IP_PATCH_BACKUP}"
+ MINIKUBE_API_IP_PATCH_PREEXISTING=1
+fi
+sed 's#__K8S_API_IP__#10.96.0.1#g' "${MINIKUBE_API_IP_TEMPLATE}" >"${MINIKUBE_API_IP_PATCH}"
+
+MINIKUBE_RENDERED="${TMP_DIR}/minikube-rendered.yaml"
+kubectl kustomize "${ROOT}/deploy/overlays/minikube" >"${MINIKUBE_RENDERED}"
RUBYOPT=--disable=gems ruby -ryaml -e '
forbidden = %w[allow-dns-egress-sandbox-ui deny-all-sandbox-ui]
required = %w[sandbox-ui-static-deny-all sandbox-ui-static-dns-egress]
@@ -151,7 +189,6 @@ RUBYOPT=--disable=gems ruby -ryaml -e '
found << name if forbidden.include?(name)
if name == "sandbox-ui-static-dns-egress"
saw_dns_peer = false
- saw_dns_ipblock = false
Array(doc.dig("spec", "egress")).each_with_index do |rule, rule_idx|
ports = Array(rule["ports"]).map { |port| [port.fetch("protocol", "TCP"), port["port"]] }.sort
expected_ports = [["TCP", 53], ["UDP", 53]]
@@ -164,27 +201,20 @@ RUBYOPT=--disable=gems ruby -ryaml -e '
peers.each_with_index do |peer, peer_idx|
ns_name = peer.dig("namespaceSelector", "matchLabels", "kubernetes.io/metadata.name")
dns_selector = peer.dig("podSelector", "matchLabels", "k8s-app")
- dns_ipblock = peer.dig("ipBlock", "cidr")
if ns_name == "kube-system" && dns_selector == "kube-dns"
saw_dns_peer = true
next
end
- if dns_ipblock == "203.0.113.10/32"
- saw_dns_ipblock = true
- next
- end
-
- abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods or the example-dev kube-dns ClusterIP")
+ abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods")
end
end
abort("sandbox-ui static DNS egress must include a kube-system kube-dns peer") unless saw_dns_peer
- abort("sandbox-ui static DNS egress must include the example-dev kube-dns ClusterIP") unless saw_dns_ipblock
end
end
abort("sandbox-ui static slice must not render HCC-owned policies: #{found.join(", ")}") unless found.empty?
missing = required - names
abort("sandbox-ui static slice must render renamed baseline policies: #{missing.join(", ")}") unless missing.empty?
-' "${GCP_DEV_RENDERED}"
-echo "PASS: gcp-dev sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and include kube-dns ClusterIP"
+' "${MINIKUBE_RENDERED}"
+echo "PASS: minikube sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and retain kube-dns-only DNS egress"
diff --git a/scripts/tests/test-postgres-single-writer-strategy.sh b/scripts/tests/test-postgres-single-writer-strategy.sh
new file mode 100755
index 000000000..40174bf46
--- /dev/null
+++ b/scripts/tests/test-postgres-single-writer-strategy.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Regression test for deploy/scripts/validate-postgres-single-writer-strategy.sh
+#
+# Captures the [R4-D] fix: a Deployment this repo DEFINES (deploy/base/<ns>/
+# exists) that is ABSENT from a rendered overlay must FAIL LOUDLY, not be
+# NOTICE-and-skipped. Before the fix the validator skipped any absent deploy,
+# so a local overlay that stopped rendering control-plane/host-context-controller
+# validated the single-writer rollout invariant over an empty set and still
+# printed "validation passed" (a fail-open / false green).
+#
+# The CI step "Validate single-writer rollout invariant on rendered overlays"
+# already runs the validator against the real minikube overlay (a happy-path
+# check). This test is complementary: it fabricates the absent-deploy condition
+# in a hermetic fixture so the fail-loud behavior itself is under test.
+#
+# The test builds a hermetic fixture ROOT_DIR, copies the validator into it so
+# its ROOT_DIR ($(dirname)/../..) resolves to the fixture, and renders overlays
+# OFFLINE with `kubectl kustomize` — the same path the validator uses. No cluster.
+#
+# Fail-loud / fail-fast: every assertion fails the suite immediately, and the
+# reporter refuses to pass unless a non-zero number of assertions actually ran.
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+VALIDATOR="${REPO_ROOT}/deploy/scripts/validate-postgres-single-writer-strategy.sh"
+
+for tool in kubectl yq jq; do
+ command -v "${tool}" >/dev/null 2>&1 ||
+ { echo "FAIL: required tool '${tool}' is missing" >&2; exit 1; }
+done
+[ -f "${VALIDATOR}" ] ||
+ { echo "FAIL: validator not found at ${VALIDATOR}" >&2; exit 1; }
+
+TESTS_RUN=0
+FAILURES=0
+pass() { TESTS_RUN=$((TESTS_RUN + 1)); echo "ok - $1"; }
+fail() { TESTS_RUN=$((TESTS_RUN + 1)); FAILURES=$((FAILURES + 1)); echo "FAIL - $1" >&2; }
+
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/single-writer-test.XXXXXX")"
+trap 'rm -rf "${WORK}"' EXIT
+
+ROOT="${WORK}/repo"
+mkdir -p "${ROOT}/deploy/scripts"
+cp "${VALIDATOR}" "${ROOT}/deploy/scripts/"
+FIXTURE_VALIDATOR="${ROOT}/deploy/scripts/$(basename "${VALIDATOR}")"
+
+# control-plane is a namespace this fixture repo DEFINES (base tree present).
+# registry is intentionally NOT defined here -> it is FOREIGN (NOTICE-and-skip).
+mkdir -p "${ROOT}/deploy/base/control-plane"
+: >"${ROOT}/deploy/base/control-plane/.keep"
+
+write_kustomization() { # dir, resource-file...
+ local dir="$1"; shift
+ {
+ echo 'apiVersion: kustomize.config.k8s.io/v1beta1'
+ echo 'kind: Kustomization'
+ echo 'resources:'
+ local r
+ for r in "$@"; do echo " - ${r}"; done
+ } >"${dir}/kustomization.yaml"
+}
+
+write_control_postgres() { # dir (valid stateful DB deploy: PVC + Recreate + replicas 1)
+ cat >"$1/control-postgres.yaml" <<'YAML'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: control-postgres
+ namespace: control-plane
+spec:
+ replicas: 1
+ strategy:
+ type: Recreate
+ template:
+ spec:
+ volumes:
+ - name: data
+ persistentVolumeClaim:
+ claimName: control-postgres-data
+YAML
+}
+
+write_hcc() { # dir (valid single-writer deploy: replicas 1 + Recreate)
+ cat >"$1/host-context-controller.yaml" <<'YAML'
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: host-context-controller
+ namespace: control-plane
+spec:
+ replicas: 1
+ strategy:
+ type: Recreate
+ template:
+ spec:
+ containers:
+ - name: host-context-controller
+ image: example/hcc:test
+YAML
+}
+
+# --- Overlay A: LOCAL single-writer deploy ABSENT (the fail-open trigger) ---
+# Renders a valid control-postgres but OMITS host-context-controller.
+OV_MISSING="${ROOT}/deploy/overlays/local-missing"
+mkdir -p "${OV_MISSING}"
+write_control_postgres "${OV_MISSING}"
+write_kustomization "${OV_MISSING}" control-postgres.yaml
+
+# --- Overlay B: every LOCAL deploy PRESENT and valid (foreign still skipped) ---
+OV_OK="${ROOT}/deploy/overlays/all-local-present"
+mkdir -p "${OV_OK}"
+write_control_postgres "${OV_OK}"
+write_hcc "${OV_OK}"
+write_kustomization "${OV_OK}" control-postgres.yaml host-context-controller.yaml
+
+run_validator() { # overlays-value ; prints combined output, returns exit code
+ SINGLE_WRITER_OVERLAYS="$1" bash "${FIXTURE_VALIDATOR}" 2>&1
+}
+
+# 1) A locally-defined single-writer deploy absent from a rendered overlay MUST
+# fail loudly (this is exactly what the pre-fix NOTICE-and-skip let pass).
+out="$(run_validator "deploy/overlays/local-missing")"; rc=$?
+if [ "${rc}" -ne 0 ] && printf '%s' "${out}" |
+ grep -q "renders no Deployment control-plane/host-context-controller"; then
+ pass "absent locally-defined single-writer deploy fails loudly (exit ${rc})"
+else
+ fail "absent local single-writer deploy did NOT fail loudly (exit ${rc}); output: ${out}"
+fi
+
+# 2) With every locally-defined deploy present and valid, a still-absent FOREIGN
+# deploy (registry/registry-postgres) is NOTICE-and-skipped, and the script
+# passes — the fix must not over-fail on foreign deploys.
+out="$(run_validator "deploy/overlays/all-local-present")"; rc=$?
+if [ "${rc}" -eq 0 ] &&
+ printf '%s' "${out}" | grep -q "Single-writer rollout strategy validation passed" &&
+ printf '%s' "${out}" | grep -q "does not render foreign Deployment registry/registry-postgres"; then
+ pass "all local deploys present -> passes; absent foreign deploy is skipped"
+else
+ fail "expected pass with foreign-skip (exit ${rc}); output: ${out}"
+fi
+
+# 3) Zero overlays must not pass vacuously.
+out="$(run_validator " ")"; rc=$?
+if [ "${rc}" -ne 0 ] && printf '%s' "${out}" | grep -q "zero overlays"; then
+ pass "whitespace-only overlay set fails loudly (no vacuous pass)"
+else
+ fail "empty overlay set did not fail loudly (exit ${rc}); output: ${out}"
+fi
+
+# --- Reporter (fail-loud / zero-tests-is-never-success) ---
+echo "1..${TESTS_RUN}"
+if [ "${TESTS_RUN}" -lt 3 ]; then
+ echo "FAIL: expected at least 3 assertions, ran ${TESTS_RUN}" >&2
+ exit 1
+fi
+if [ "${FAILURES}" -ne 0 ]; then
+ echo "FAIL: ${FAILURES}/${TESTS_RUN} assertions failed" >&2
+ exit 1
+fi
+echo "All ${TESTS_RUN} single-writer validator assertions passed"
diff --git a/scripts/tests/test-wrc-hcc-contracts-gate.sh b/scripts/tests/test-wrc-hcc-contracts-gate.sh
new file mode 100755
index 000000000..a0d6180bc
--- /dev/null
+++ b/scripts/tests/test-wrc-hcc-contracts-gate.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -u
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+GATE="${ROOT}/scripts/e2e/e2e-wrc-hcc-contracts.sh"
+FAIL=0
+
+pass() { echo "PASS: $1"; }
+fail() { echo "FAIL: $1"; FAIL=1; }
+
+if bash -n "$GATE"; then
+ pass "WRC-HCC contracts gate has valid bash syntax"
+else
+ fail "WRC-HCC contracts gate has invalid bash syntax"
+fi
+
+# The Phase 2 fixture has steps, so the admission policy requires an explicit
+# trigger. Keep this immediately alongside the fixture's agent/steps contract
+# so the E2E cannot regress into an admission-only false negative.
+phase_two_fixture="$(sed -n '/^header "Phase 2 - WRC toolsCalled args status contract"$/,/^YAML$/p' "$GATE")"
+if [[ "$phase_two_fixture" == *$' triggers:\n onDemand: {}'* ]] &&
+ [[ "$phase_two_fixture" == *$' steps:'* ]]; then
+ pass "WRC-HCC Phase 2 fixture declares its required on-demand trigger"
+else
+ fail "WRC-HCC Phase 2 fixture can be rejected before its status contract runs"
+fi
+
+exit "$FAIL"
diff --git a/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh b/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
new file mode 100755
index 000000000..98f9a9177
--- /dev/null
+++ b/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -u
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+GATE="${ROOT}/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh"
+FAIL=0
+
+pass() { echo "PASS: $1"; }
+fail() { echo "FAIL: $1"; FAIL=1; }
+
+if bash -n "$GATE"; then
+ pass "WRC internal-dependency gate has valid bash syntax"
+else
+ fail "WRC internal-dependency gate has invalid bash syntax"
+fi
+
+# WRC persists its generated workload resource names in status before
+# materializing workloads. The gate must use that public CRD contract rather
+# than assuming a workload ID is also a Deployment or Service name.
+# shellcheck disable=SC2016
+if grep -Fq 'wait_for_workload_instance() {' "$GATE" &&
+ grep -Fq 'SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID"' "$GATE" &&
+ grep -Fq 'BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID"' "$GATE" &&
+ ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID"' "$GATE" &&
+ ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID"' "$GATE" &&
+ grep -Fq 'deploy/${SOURCE_DEPLOYMENT}' "$GATE" &&
+ grep -Fq '${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local' "$GATE"; then
+ pass "WRC gate resolves persisted workload instances before runtime assertions"
+else
+ fail "WRC gate assumes raw workload IDs are runtime resource names"
+fi
+
+exit "$FAIL"
diff --git a/tests/e2e/integration/network-policies.test.ts b/tests/e2e/integration/network-policies.test.ts
index aba9302b2..07925e9db 100644
--- a/tests/e2e/integration/network-policies.test.ts
+++ b/tests/e2e/integration/network-policies.test.ts
@@ -8,12 +8,50 @@
* the NetworkPolicy is blocking traffic as expected. A successful response or
* connection refused (port not listening) would indicate a policy gap.
*/
-import { describe, expect, it } from 'vitest'
+import { beforeAll, describe, expect, it } from 'vitest'
import { execFileSync } from 'child_process'
// Short timeout for negative tests — we expect these to time out
const NP_TIMEOUT = 3
+// ─── Cluster reachability gate ───────────────────────────────────────────────
+// This suite drives kubectl against a live minikube cluster (Calico CNI + all
+// services deployed). Mirrors the opt-in-skip convention already used by the
+// sibling integration suites — channel-reader-via-api.test.ts and
+// mcpCredentialRotation.helpers.ts (E2E_SKIP_IF_CLUSTER_UNREACHABLE) plus
+// registry-pull-secret-runtime.test.ts (E2E_REQUIRE_CLUSTER / CI):
+// - Default (no flag): fail-fast when the cluster is unreachable. A silent
+// pass would hide a broken E2E setup, so the suite must NOT be neutralized
+// unconditionally.
+// - E2E_SKIP_IF_CLUSTER_UNREACHABLE=1: opt in to a clean skip for the tier
+// that runs integration WITHOUT a cluster (T1).
+// - E2E_REQUIRE_CLUSTER=1 or CI=true: force a hard failure even if the skip
+// flag is also set — CI must never skip.
+const SKIP_IF_UNREACHABLE = process.env.E2E_SKIP_IF_CLUSTER_UNREACHABLE === '1'
+const REQUIRE_CLUSTER = process.env.E2E_REQUIRE_CLUSTER === '1' || process.env.CI === 'true'
+
+let clusterReachable = false
+
+/**
+ * One-shot reachability probe: succeeds only when kubectl can talk to the API
+ * server. This is a precondition probe (the established sibling idiom —
+ * registry-pull-secret-runtime.test.ts's `kubectlOrNull`), NOT an error
+ * swallow: its boolean result drives the fail-hard-or-skip decision in
+ * `beforeAll`, which stays loud in the default and CI paths.
+ */
+function probeClusterReachable(): boolean {
+ try {
+ execFileSync('kubectl', ['get', 'ns', 'default', '-o', 'name'], {
+ encoding: 'utf-8',
+ timeout: 15_000,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ })
+ return true
+ } catch {
+ return false
+ }
+}
+
interface ConnectResult {
reachable: boolean
statusCode?: number
@@ -141,8 +179,32 @@ function testConnectivity(
}
describe('NetworkPolicy Enforcement (Calico)', () => {
+ beforeAll(() => {
+ clusterReachable = probeClusterReachable()
+ if (clusterReachable) return
+
+ const msg =
+ '[network-policies] kubernetes cluster not reachable (kubectl get ns default failed)'
+ if (REQUIRE_CLUSTER) {
+ throw new Error(
+ `${msg}. Refusing to skip: E2E_REQUIRE_CLUSTER/CI is set, and a silent skip here ` +
+ 'would report success while asserting nothing about NetworkPolicy enforcement. ' +
+ 'Bring up the cluster (e.g. `make minikube-setup`) or unset the flag.'
+ )
+ }
+ if (SKIP_IF_UNREACHABLE) {
+ console.log(`${msg} — tests will be skipped (E2E_SKIP_IF_CLUSTER_UNREACHABLE=1)`)
+ return
+ }
+ throw new Error(
+ `${msg}. This E2E suite requires a minikube cluster with Calico CNI and all services ` +
+ 'deployed. Run `make minikube-setup` first, or set E2E_SKIP_IF_CLUSTER_UNREACHABLE=1 to skip.'
+ )
+ })
+
// ─── Prerequisite Check ──────────────────────────────────────────────
- it('Calico CNI is active', () => {
+ it('Calico CNI is active', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const pods = execFileSync(
'kubectl',
['get', 'pods', '-n', 'kube-system', '-l', 'k8s-app=calico-node', '--no-headers'],
@@ -151,7 +213,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(pods).toContain('Running')
})
- it('deny-all policies exist in secured namespaces', () => {
+ it('deny-all policies exist in secured namespaces', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const namespaces = ['mcp-host', 'mcp-server', 'rpc-proxy', 'sandbox-recipes']
for (const ns of namespaces) {
let nps: string
@@ -184,7 +247,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
// ─── POSITIVE TESTS (Allowed Traffic) ────────────────────────────────
describe('Allowed traffic (positive)', () => {
- it('control-api → registry-api:8085 (cross-namespace, registry access)', () => {
+ it('control-api → registry-api:8085 (cross-namespace, registry access)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'control-plane',
'control-api',
@@ -193,7 +257,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.reachable).toBe(true)
})
- it('control-ui → control-api:8090 (UI backend path)', () => {
+ it('control-ui → control-api:8090 (UI backend path)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'control-plane',
'control-ui',
@@ -202,7 +267,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.reachable).toBe(true)
})
- it('external-rest-api → profile-control-funnel:8080 (internal control path)', () => {
+ it('external-rest-api → profile-control-funnel:8080 (internal control path)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'profiles',
'external-rest-api',
@@ -211,12 +277,14 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.reachable).toBe(true)
})
- it('rpc-proxy → mcp-host:8080 (message relay)', () => {
+ it('rpc-proxy → mcp-host:8080 (message relay)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity('rpc-proxy', 'rpc-proxy', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(true)
})
- it('mcp-host can reach the HCC gateway readiness endpoint', () => {
+ it('mcp-host can reach the HCC gateway readiness endpoint', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -230,7 +298,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
// ─── NEGATIVE TESTS (Blocked by Design) ──────────────────────────────
describe('Blocked traffic (negative — deny-all enforced)', () => {
- it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', () => {
+ it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -240,7 +309,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('mcp-host → control-api:8090 (no direct access to control-plane)', () => {
+ it('mcp-host → control-api:8090 (no direct access to control-plane)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -250,11 +320,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', () => {
+ it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const podName = getChannelReaderPodName()
if (!podName) {
console.warn('SKIPPED: no running channel-reader pod in channels')
- return
+ ctx.skip()
}
const result = testConnectivity(
@@ -266,7 +337,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('mcp-host → registry-api:8085 (no direct registry access)', () => {
+ it('mcp-host → registry-api:8085 (no direct registry access)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -276,7 +348,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('control-api → rpc-proxy:8094 (no direct control-plane hop)', () => {
+ it('control-api → rpc-proxy:8094 (no direct control-plane hop)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'control-plane',
'control-api',
@@ -286,7 +359,8 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', () => {
+ it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'profiles',
'external-rest-api',
@@ -296,19 +370,22 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {
+ it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity('control-plane', 'control-api', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(false)
expect(result.error).toContain('timeout')
})
- it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {
+ it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity('profiles', 'external-rest-api', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(false)
expect(result.error).toContain('timeout')
})
- it('mcp-proxy → external-rest-api:3000 (no reverse path)', () => {
+ it('mcp-proxy → external-rest-api:3000 (no reverse path)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const result = testConnectivity(
'mcp-server',
'mcp-proxy',
@@ -348,11 +425,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
}
}
- it('sandbox-recipes → control-api:8090 (isolated sandbox)', () => {
+ it('sandbox-recipes → control-api:8090 (isolated sandbox)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const podName = findExecPod('sandbox-recipes', 'app=db')
if (!podName) {
console.warn('SKIPPED: no running db pod in sandbox-recipes')
- return
+ ctx.skip()
}
// sandbox-recipes has deny-all; only DNS + HCC API egress allowed
const result = testConnectivity(
@@ -364,11 +442,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', () => {
+ it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', (ctx) => {
+ if (!clusterReachable) ctx.skip()
const podName = findExecPod('sandbox-recipes', 'app=db')
if (!podName) {
console.warn('SKIPPED: no running db pod in sandbox-recipes')
- return
+ ctx.skip()
}
const result = testConnectivity(
'sandbox-recipes',
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 558c31760..9981d0aa7 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.66",
+ "version": "0.1.65",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.66",
+ "version": "0.1.65",
"devDependencies": {
"@types/jsonwebtoken": "^9.0.10",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index 19145d619..63c62d203 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.66",
+ "version": "0.1.65",
"private": true,
"type": "module",
"scripts": {