Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

Revert "Merge pull request #205 from evenfire-ai/codex/issue-204-hcc-readiness-decouple"

Alfredo Lopez562741148d50
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 08541e5bc..18daf933c 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -390,21 +390,6 @@ jobs:
- name: Test HCC watch-recovery gate coordination
run: bash scripts/tests/test-hcc-watch-recovery-gate.sh
- - name: Test HCC API gateway readiness separation
- run: bash scripts/tests/test-hcc-api-gateway-readiness.sh
-
- - name: Test host-storm metric adjudication
- run: bash scripts/tests/test-host-storm-metric-gate.sh
-
- - name: Test WRC-to-HCC contracts
- run: bash scripts/tests/test-wrc-hcc-contracts-gate.sh
-
- - name: Test WRC internal dependency NetworkPolicy contracts
- run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
-
- - name: Test minikube sandbox-ui NetworkPolicy lint (static-policy ownership + DNS)
- run: bash scripts/tests/test-lint-networkpolicies.sh
-
- name: Test Plugin Workload SDK auth bootstrap policy
run: bash scripts/tests/test-plugin-workload-sdk-auth-bootstrap.sh
@@ -414,24 +399,6 @@ jobs:
- name: Test minikube detect-k8s-api-ip template rendering
run: bash scripts/tests/test-minikube-detect-k8s-api-ip.sh
- - name: Validate single-writer rollout invariant on rendered overlays
- run: |
- # The k8s-api-ip patch is rendered from a gitignored template at setup
- # time; render it with a dummy IP so `kubectl kustomize` can build the
- # minikube overlay here. The gcp-* overlays live in evenfire-infra, so
- # this repo validates only the minikube overlay it actually ships.
- sed 's#__K8S_API_IP__#10.96.0.1#g' \
- deploy/overlays/minikube/patches/k8s-api-ip.yaml.template \
- > deploy/overlays/minikube/patches/k8s-api-ip.yaml
- SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" \
- bash deploy/scripts/validate-postgres-single-writer-strategy.sh
-
- - name: Test single-writer validator fails loud on an absent local deploy
- run: bash scripts/tests/test-postgres-single-writer-strategy.sh
-
- - name: Test post-reset convergence isolates DB-dependent controllers
- run: bash scripts/tests/test-converge-control-db-after-reset.sh
-
- name: Test release coordinates (writer, checker, floor decoupling)
run: bash scripts/tests/test-release-coordinates.sh
diff --git a/Makefile b/Makefile
index 107633d70..57f47ec5a 100644
--- a/Makefile
+++ b/Makefile
@@ -934,20 +934,7 @@ test-e2e-stateless-wake-recovery: ## Run stateless wake-recovery latency gate (R
.PHONY: test-e2e-hcc-communicationchannel-watch-recovery
test-e2e-hcc-communicationchannel-watch-recovery: ## Run isolated minikube HCC watch-recovery fault-injection gate
@echo "Running HCC CommunicationChannel watch-recovery gate..."
- @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
- E2E_HCC_WATCH_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
-
-.PHONY: test-e2e-hcc-readiness-bootstrap
-test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host fleet pass remains active
- @echo "Running HCC initial-fleet readiness gate..."
- @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
- E2E_HCC_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-readiness-bootstrap.sh
-
-.PHONY: test-e2e-hcc-mcp-context-readiness
-test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Context/NetworkPolicy initial convergence
- @echo "Running HCC MCP/Context/NetworkPolicy readiness gate..."
- @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
- E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-mcp-context-readiness.sh
+ E2E_HCC_WATCH_FAULT_INJECTION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
.PHONY: test-e2e-stateless-multinode
test-e2e-stateless-multinode: ## Run stateless multi-node lane (opt-in: STATELESS_MULTINODE_GATE=1; needs >=2 schedulable nodes; exit 3 = cross-node UNVERIFIED)
diff --git a/deploy/base/control-plane/host-context-controller-api-gateway.yaml b/deploy/base/control-plane/host-context-controller-api-gateway.yaml
index 1c30ccfbc..961cebe48 100644
--- a/deploy/base/control-plane/host-context-controller-api-gateway.yaml
+++ b/deploy/base/control-plane/host-context-controller-api-gateway.yaml
@@ -43,7 +43,7 @@ spec:
timeoutSeconds: 5
readinessProbe:
httpGet:
- path: /health
+ path: /ready
port: http
scheme: HTTP
initialDelaySeconds: 3
diff --git a/deploy/base/control-plane/host-context-controller.yaml b/deploy/base/control-plane/host-context-controller.yaml
index 24a4c21dd..df8126a53 100644
--- a/deploy/base/control-plane/host-context-controller.yaml
+++ b/deploy/base/control-plane/host-context-controller.yaml
@@ -13,33 +13,21 @@ metadata:
app: host-context-controller
spec:
replicas: 1
- # HCC's safety model assumes a single NetworkPolicy writer: replicas:1 with no
- # leader election (see statelessLifecycleExecutor.ts). RollingUpdate (the
- # default) starts the new pod before terminating the old one, so every rollout
- # would overlap two writers — during which the outgoing replica can win a
- # conflict-retry replace and land its wider spec back over a policy the new
- # replica just hardened and certified, with no NetworkPolicy resync to heal it
- # and /ready still 200. Recreate terminates the old pod first, honouring the
- # single-writer assumption through the rollout too.
- strategy:
- type: Recreate
- # Recreate + replicas:1 means a botched rollout (new pod never Ready) leaves
- # the control plane down until a manual `kubectl rollout undo` — the single-
- # writer invariant forbids any overlap-based auto-recovery. This deadline is
- # DETECTION ONLY: it flips Progressing=False (ProgressDeadlineExceeded) and
- # fails `kubectl rollout status` for callers without their own --timeout; it
- # does not roll back.
+ # HCC's /ready stays 503 until its initial fleet reconciliation finishes, so time-to-Ready
+ # scales with the McpServer/Context fleet rather than with process start. clerum-dev took
+ # 651s on 2026-08-05 at 108 McpServers / 22 Contexts.
#
- # Value (origin/dev, 2026-08-05): HCC's /ready stays 503 until its initial fleet
- # reconciliation finishes, so time-to-Ready scales with the McpServer/Context
- # fleet — clerum-dev took 651s at 108 McpServers / 22 Contexts. The unset default
- # (600s) governed the deploy because `kubectl rollout status` aborts the instant
- # the deadline is exceeded, and the clock starts at apply-inter-service-tokens.sh's
- # `rollout restart`, so the wait inherits only the remaining budget (600-160=440s
- # vs a 480s timeout on 2026-08-05: #202's timeout never bound, deploy failed 50s
- # short). 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the
- # fix — #205 decouples readiness from fleet convergence, after which this should
- # come back down.
+ # Unset, this defaults to 600s, and that default is what actually governed the deploy:
+ # `kubectl rollout status` aborts the instant the controller sets ProgressDeadlineExceeded,
+ # so provision-gfs-runtime.sh's --timeout could never exceed the deadline remaining. The
+ # clock also starts several steps earlier, at apply-inter-service-tokens.sh's `rollout
+ # restart` of HCC, so the wait inherits only what is left. On 2026-08-05 that was 600-160
+ # = 440s against a 480s timeout: the timeout raised in #202 never bound, and the deploy
+ # failed 50s before HCC came Ready.
+ #
+ # 1200s covers the 900s wait plus a 300s head-start allowance. This is a mitigation, not
+ # the fix — evenfire-ai/evenfire#205 decouples readiness from fleet convergence, after
+ # which both numbers should come back down.
progressDeadlineSeconds: 1200
selector:
matchLabels:
@@ -103,8 +91,6 @@ spec:
# now fails closed at startup on any interval > overlap/2.
- name: HCC_EXTERNAL_EGRESS_RESYNC_SEC
value: "60"
- - name: HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS
- value: "5000"
- name: HCC_HOST_K8S_REQUEST_TIMEOUT_MS
value: "30000"
# Bounded cross-Host concurrency for a full/lifecycle Host fleet pass.
diff --git a/deploy/scripts/converge-control-db-after-reset.sh b/deploy/scripts/converge-control-db-after-reset.sh
index 404d61076..99f50d8bf 100755
--- a/deploy/scripts/converge-control-db-after-reset.sh
+++ b/deploy/scripts/converge-control-db-after-reset.sh
@@ -183,7 +183,7 @@ if [ "$STATE_PHASE" = converged ]; then
fi
kc -n control-plane scale deployment/host-context-controller --replicas="$HCC_REPLICAS" >/dev/null
if [ "$HCC_REPLICAS" -gt 0 ]; then
- kc -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null
+ kc -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/null
fi
assert_replacement_uid
delete_reset_state
@@ -264,7 +264,7 @@ kc -n control-plane scale deployment/host-context-controller \
--replicas="$HCC_REPLICAS" >/dev/null
if [ "$HCC_REPLICAS" -gt 0 ]; then
kc -n control-plane rollout status \
- deployment/host-context-controller --timeout=900s >/dev/null
+ deployment/host-context-controller --timeout=180s >/dev/null
fi
assert_replacement_uid
read_reset_state || fail "cannot reload reset state before completion"
diff --git a/deploy/scripts/validate-postgres-single-writer-strategy.sh b/deploy/scripts/validate-postgres-single-writer-strategy.sh
index ee90b96be..f1a77091c 100755
--- a/deploy/scripts/validate-postgres-single-writer-strategy.sh
+++ b/deploy/scripts/validate-postgres-single-writer-strategy.sh
@@ -2,86 +2,40 @@
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-# Overlays to render. Defaults to the full production set; the gcp-* overlays
-# live in keyper-labs/evenfire-infra, so this script's full coverage runs there.
-# Override (space-separated) to validate only the overlays present in a given
-# checkout — e.g. SINGLE_WRITER_OVERLAYS="deploy/overlays/minikube" in this repo.
-read -r -a OVERLAYS <<<"${SINGLE_WRITER_OVERLAYS:-deploy/overlays/minikube deploy/overlays/gcp-dev deploy/overlays/gcp-prod}"
+OVERLAYS=(
+ "deploy/overlays/minikube"
+ "deploy/overlays/gcp-dev"
+ "deploy/overlays/gcp-prod"
+)
DATABASE_DEPLOYS=(
"control-plane/control-postgres"
"registry/registry-postgres"
)
-# Deployments whose safety model assumes a single writer with NO leader election
-# (see host-context-controller/src/statelessLifecycleExecutor.ts). For these the
-# RENDERED manifest — not just deploy/base — must keep replicas<=1 and
-# strategy=Recreate with no rollingUpdate residue, or a rollout reopens the
-# two-writer NetworkPolicy overlap closed in PR #205. Unlike DATABASE_DEPLOYS
-# these mount no PVC, so they need their own list rather than the PVC heuristic.
-SINGLE_WRITER_DEPLOYS=(
- "control-plane/host-context-controller"
-)
fail() {
echo "ERROR: $*" >&2
exit 1
}
-# A Deployment id ("namespace/name") is DEFINED in this repo when its namespace
-# has a base manifest tree here (deploy/base/<namespace>/). For such a LOCAL
-# deploy the rendered overlay MUST contain it: a silent disappearance would let
-# the loops below validate the single-writer/database rollout invariant over an
-# absent deploy (a fail-open). Deploys whose base lives only in the sibling
-# evenfire-infra repo (e.g. registry/*) are FOREIGN here and are validated where
-# they are defined, so for them an absence is a NOTICE-and-skip, not a failure.
-deploy_defined_in_this_repo() {
- local id="$1"
- local namespace="${id%%/*}"
- [ -d "${ROOT_DIR}/deploy/base/${namespace}" ]
-}
-
command -v kubectl >/dev/null 2>&1 || fail "kubectl is required"
command -v jq >/dev/null 2>&1 || fail "jq is required"
-command -v yq >/dev/null 2>&1 || fail "yq is required"
-
-# Zero overlays means the loop below runs zero iterations and the script exits 0
-# having validated nothing — a fail-open. An empty or whitespace-only
-# SINGLE_WRITER_OVERLAYS must fail loudly, not pass vacuously.
-[ "${#OVERLAYS[@]}" -gt 0 ] ||
- fail "SINGLE_WRITER_OVERLAYS resolved to zero overlays — refusing to pass vacuously"
for overlay in "${OVERLAYS[@]}"; do
- # Render OFFLINE: `kubectl kustomize` builds the overlay client-side, and yq
- # converts the multi-document YAML into a single JSON array for jq. The
- # previous `kubectl create --dry-run=client` performed API discovery
- # (kind -> resource), which needs a live API server and fails in CI with
- # "unable to recognize STDIN: connection refused". This validation is a pure
- # rendered-manifest lint and must never touch a cluster.
rendered_json="$(
cd "${ROOT_DIR}" &&
kubectl kustomize "${overlay}" |
- yq ea -o=json '[.]'
+ kubectl create --dry-run=client --validate=false -f - -o json |
+ jq -s '.'
)"
for id in "${DATABASE_DEPLOYS[@]}"; do
namespace="${id%%/*}"
name="${id#*/}"
- if ! deployment="$(
+ deployment="$(
jq -e --arg namespace "${namespace}" --arg name "${name}" \
'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \
<<<"${rendered_json}"
- )"; then
- # This overlay does not render ${id}. A deploy this repo DEFINES
- # (deploy/base/<namespace>/ exists here) MUST be rendered by a local
- # overlay: a silent disappearance would validate the rollout invariant over
- # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and
- # the registry Deployment live in the sibling evenfire-infra repo) are
- # validated where they are defined, so for them a NOTICE-and-skip is right.
- if deploy_defined_in_this_repo "${id}"; then
- fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"
- fi
- echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2
- continue
- fi
+ )" || fail "${overlay}: missing Deployment ${id}"
pvc_count="$(
jq '[.spec.template.spec.volumes[]? | select(.persistentVolumeClaim.claimName != null)] | length' \
@@ -100,42 +54,6 @@ for overlay in "${OVERLAYS[@]}"; do
fail "${overlay}: Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"
fi
done
-
- for id in "${SINGLE_WRITER_DEPLOYS[@]}"; do
- namespace="${id%%/*}"
- name="${id#*/}"
- if ! deployment="$(
- jq -e --arg namespace "${namespace}" --arg name "${name}" \
- 'first(.[] | select(.kind == "Deployment" and .metadata.namespace == $namespace and .metadata.name == $name))' \
- <<<"${rendered_json}"
- )"; then
- # This overlay does not render ${id}. A deploy this repo DEFINES
- # (deploy/base/<namespace>/ exists here) MUST be rendered by a local
- # overlay: a silent disappearance would validate the rollout invariant over
- # an absent deploy (fail-open), so fail loudly. Foreign deploys (gcp-* and
- # the registry Deployment live in the sibling evenfire-infra repo) are
- # validated where they are defined, so for them a NOTICE-and-skip is right.
- if deploy_defined_in_this_repo "${id}"; then
- fail "${overlay}: renders no Deployment ${id}, but this repo defines deploy/base/${id%%/*} — a local overlay must not validate the rollout invariant over an absent deploy (did the overlay stop including it?)"
- fi
- echo "NOTICE: ${overlay} does not render foreign Deployment ${id}; skipping (validated where it is defined)" >&2
- continue
- fi
-
- replicas="$(jq -r '.spec.replicas // 1' <<<"${deployment}")"
- if [ "${replicas}" -gt 1 ]; then
- fail "${overlay}: single-writer Deployment ${id} must keep replicas<=1 (no leader election), got ${replicas}"
- fi
-
- strategy="$(jq -r '.spec.strategy.type // "RollingUpdate"' <<<"${deployment}")"
- if [ "${strategy}" != "Recreate" ]; then
- fail "${overlay}: single-writer Deployment ${id} must use strategy.type=Recreate, got ${strategy}"
- fi
-
- if jq -e '.spec.strategy.rollingUpdate != null' >/dev/null <<<"${deployment}"; then
- fail "${overlay}: single-writer Deployment ${id} uses Recreate and must not render spec.strategy.rollingUpdate"
- fi
- done
done
-echo "Single-writer rollout strategy validation passed"
+echo "Postgres single-writer rollout strategy validation passed"
diff --git a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx
index ff8667aa0..0b1007986 100644
--- a/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx
+++ b/desktop-app/ui/src/hooks/domain/__tests__/__fixtures__/appControllerHarness.tsx
@@ -52,7 +52,9 @@ function createDeferred<T>(): Deferred<T> {
}
function createSessionState(authenticated: boolean, me = HARNESS_ME) {
- return authenticated ? { authenticated: true, me } : { authenticated: false, me: null }
+ return authenticated
+ ? { authenticated: true, me }
+ : { authenticated: false, me: null }
}
function createCatalog(agentNames: string[]) {
diff --git a/docs/architecture/hcc-readiness-flows.md b/docs/architecture/hcc-readiness-flows.md
deleted file mode 100644
index 6a4e89ca9..000000000
--- a/docs/architecture/hcc-readiness-flows.md
+++ /dev/null
@@ -1,90 +0,0 @@
-# HCC readiness & external-egress — design note (PR #205)
-
-> Design note for the readiness-decoupling and single-writer NetworkPolicy work in
-> `host-context-controller` (HCC). Addresses the "no design/decision doc for the
-> 500+-line external-egress coordinator and the certification barrier" review gap.
-
-![HCC flows resolved by PR #205](./pr205-hcc-flows.png)
-
-## 1. Readiness lifecycle — decoupled from initial fleet convergence
-
-Before this work, `McpServerWatcher.start()` awaited full initial fleet
-reconciliation, so a large MCP/Context/Host fleet could keep `provider.start()`
-and `/ready` unavailable until every individual reconcile finished.
-
-Now readiness is gated on an **authoritative, fail-closed safety boundary**, not on
-full convergence:
-
-1. **HCC start** — four watch lanes (Context ingress, mcp-host egress, rpc-proxy
- egress, external egress) start. A transient LIST/WATCH failure on any lane enters
- in-process recovery and **does not kill the process**; HCC simply stays unready.
-2. **4-LIST inventory** — bounded inventory across the four lanes.
-3. **Certify safety inventory** — the authoritative pass revokes any stale /
- unprovable allow **before** recording a certificate (fail-closed).
-4. **`/ready = 200`** — served **only** when the inventory is certified and no stale
- allow survives.
-5. **Additive convergence** — the remaining fleet converges **after** ready,
- asynchronously, without blocking it.
-
-## 2. Fail-closed fences
-
-| Fence | Rule | Source |
-|-------|------|--------|
-| Readiness gate | `/ready = ready && providerAuthoritativeFn()`; constructor default `() => false`; a throwing gate → 503 | `server.ts`, `readinessGate.ts` |
-| G1 — unconfigured authority | `currentWhenUnconfigured()` returns `false` + warns once; prod always wires the source, dev never constructs the reconciler | `reconciler.ts` |
-| F1 — readiness-poll rebind | rebinds `isCurrent`/`server` on a superseding reconcile so the poll window does not self-destruct on a stale fence | `reconciler.ts` |
-| Authority fence — `isCurrent` | **required** (not an optional/defaulted arg) on all three allow-granting reconcilers — `bindingPolicyReconciler`, `reconcileContext` (L2 context-allow), `reconcileExternalEgress` (L3 external-egress ALLOW). No `?? (() => true)` fail-open default: a caller cannot compile without supplying the fence (TS2554/TS2345) | `networkPolicyReconciler.ts`, `bindingPolicyReconciler.ts` |
-
-The affirmative literal (`() => true`) is reachable only from the dev/test wiring;
-every production path resolves the real authority function, and — because the fence
-is a **required argument** on every allow-granting reconciler rather than a defaulted
-option — no production caller can silently fall back to it.
-
-## 3. DNS external-egress contract (B3) — retain / revoke decision table
-
-A DNS-derived external-egress allow is compared against the desired policy
-reconstructed from the **live** resolved CIDRs ("modulo cidr"). The decision:
-
-| Observed change | Classification | Action | Rationale |
-|-----------------|----------------|--------|-----------|
-| identity unchanged (same protocol, port, selector, ownership; cidrs only) | identity-stable | **RETAIN** while Ready; refresh cidrs in the additive lane | revoking a healthy binding every pass created a guaranteed deny window |
-| protocol drift (TCP→UDP) | identity-changed | **REVOKE before readiness** | old allow is unprovable/stale |
-| spec-level port drift | identity-changed | **REVOKE before readiness** | " |
-| ownership drift | identity-changed | **REVOKE before readiness** | " |
-| zero-cidr (deny-in-disguise) | never retained | **REVOKE** | an empty cidr set is never a valid retained allow |
-
-Revocation of an unprovable policy runs immediately (`cleanupExternalEgress`) under
-the `isCurrent()` fence, **before** the safety inventory is certified — so `/ready`
-never reports 200 with a stale divergent policy present. The unit suite pins both
-directions (retain identity-stable; revoke on each drift), and the branch-owned
-Minikube gate `e2e-hcc-mcp-context-readiness.sh` exercises both phases live.
-
-## 4. ExternalEgressConvergenceCoordinator
-
-The coordinator is the **single owner** of external-egress reconcile cadence
-(replacing an inline timer in `k8sClient`):
-
-- **Per-server FIFO + concurrency limiter** — a slow or never-resolving binding
- cannot head-of-line block the control-plane event loop.
-- **Sliding-window CIDR accumulation (#299)** — resolved IPs live for `TTL + overlap`;
- large provider pools are covered by provider CIDRs rather than per-IP `/32`s. A
- failed refresh deletes the allow (fail-closed), never widens it.
-- **TTL-aware periodic resync (H2)** — `startPeriodicResync` re-arms after each pass
- with a delay of `min(interval, TTL/2)` bounded below by a floor, so low-TTL hosts
- refresh promptly. The timer is `.unref()`'d and re-programs **only after a pass
- completes** (no overlap).
-- **Fence** — startup rejects `interval > overlap/2`, so the accumulated window can
- never lapse regardless of the configured cadence.
-- **Observability** — `clerum_hcc_external_egress_retries_at_cap` gauges how many
- servers are stuck retrying at the delay cap (a never-resolving DNS binding is
- otherwise invisible: denied with `/ready` = 200 and no signal).
-
-## 5. Deploy invariant — single writer
-
-The HCC Deployment ships `strategy: Recreate`, `replicas: 1`, and
-`progressDeadlineSeconds: 1200`: the stateless single-writer safety model has no
-leader election, so two concurrently-running controllers must be impossible during a
-rollout. The trade-off (full HCC control-plane downtime per rollout; a rollout whose
-new pod never becomes Ready needs a manual `kubectl rollout undo`) is declared in the
-manifest and asserted by the bootstrap gate. `validate-postgres-single-writer-strategy.sh`
-enforces the invariant on the rendered manifest in CI.
diff --git a/docs/architecture/pr205-before-after.png b/docs/architecture/pr205-before-after.png
deleted file mode 100644
index 524481e1f..000000000
Binary files a/docs/architecture/pr205-before-after.png and /dev/null differ
diff --git a/docs/architecture/pr205-before-after.svg b/docs/architecture/pr205-before-after.svg
deleted file mode 100644
index 471f74cc3..000000000
--- a/docs/architecture/pr205-before-after.svg
+++ /dev/null
@@ -1,71 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1620 880" font-family="Helvetica, Arial, sans-serif">
-<defs>
-<marker id="arr_2563eb" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#2563eb"/></marker>
-<marker id="arr_dc2626" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#dc2626"/></marker>
-<marker id="arr_16a34a" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#16a34a"/></marker>
-<marker id="arr_d97706" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#d97706"/></marker>
-<marker id="arr_64748b" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="#64748b"/></marker>
-</defs>
-<style>text{font-family:Helvetica,Arial,sans-serif;}</style>
-<rect x="0" y="0" width="1620" height="880" rx="0" fill="#ffffff" stroke="#ffffff" stroke-width="0" opacity="1.0"/>
-<text x="810.0" y="46" font-size="27" fill="#0f172a" font-weight="700" text-anchor="middle">PR #205 — Decoupling HCC readiness: before vs after</text>
-<text x="810.0" y="76" font-size="15" fill="#475569" font-weight="400" text-anchor="middle">The Host Context Controller's /ready endpoint stops waiting for full fleet convergence</text>
-<rect x="30" y="100" width="760" height="660" rx="18" fill="#fff7f6" stroke="#fecaca" stroke-width="2" opacity="1.0"/>
-<rect x="830" y="100" width="760" height="660" rx="18" fill="#f5fbf7" stroke="#bbf7d0" stroke-width="2" opacity="1.0"/>
-<text x="410" y="136" font-size="19" fill="#dc2626" font-weight="700" text-anchor="middle">BEFORE · origin/dev (issue #204)</text>
-<text x="1210" y="136" font-size="19" fill="#16a34a" font-weight="700" text-anchor="middle">AFTER · PR #205</text>
-<rect x="100.0" y="168" width="600" height="58" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
-<text x="400" y="201.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>
-<line x1="400" y1="226" x2="400" y2="279" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>
-<rect x="100.0" y="282" width="600" height="128" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>
-<text x="400" y="330.05" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">Waits to reconcile the ENTIRE initial fleet</text>
-<text x="400" y="357.35" font-size="14" fill="#475569" font-weight="400" text-anchor="middle">108 McpServers · 22 Contexts · Hosts</text>
-<text x="400" y="383.94" font-size="15" fill="#dc2626" font-weight="700" text-anchor="middle">/ready = 503 for the entire wait</text>
-<rect x="636" y="300" width="132" height="92" rx="12" fill="#dc2626" stroke="#dc2626" stroke-width="0" opacity="1.0"/>
-<text x="702" y="336" font-size="15" fill="#ffffff" font-weight="700" text-anchor="middle">BLOCKED</text>
-<text x="702" y="362" font-size="22" fill="#ffffff" font-weight="700" text-anchor="middle">~651 s</text>
-<line x1="400" y1="410" x2="400" y2="467" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>
-<rect x="100.0" y="470" width="600" height="58" rx="12" fill="#f1f5f9" stroke="#64748b" stroke-width="2" opacity="1.0"/>
-<text x="400" y="503.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">/ready = 200 (only after the ENTIRE fleet converges)</text>
-<line x1="400" y1="528" x2="400" y2="585" stroke="#dc2626" stroke-width="3" marker-end="url(#arr_dc2626)"/>
-<rect x="100.0" y="588" width="600" height="76" rx="12" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>
-<text x="400" y="621.7" font-size="17" fill="#dc2626" font-weight="700" text-anchor="middle">rollout status TIMEOUT → deploy FAILS</text>
-<text x="400" y="649.0" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">even though the HCC is healthy and progressing</text>
-<text x="410" y="700" font-size="14" fill="#dc2626" font-weight="700" text-anchor="middle" font-style="italic">Readiness COUPLED to full fleet convergence</text>
-<rect x="890.0" y="168" width="520" height="54" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
-<text x="1150" y="199.64" font-size="16" fill="#0f172a" font-weight="700" text-anchor="middle">HCC starts — 4 watch lanes</text>
-<line x1="1150" y1="222" x2="1150" y2="265" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
-<rect x="890.0" y="268" width="520" height="50" rx="12" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
-<text x="1150" y="297.35" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">BOUNDED inventory (bounded 4-LIST)</text>
-<line x1="1150" y1="318" x2="1150" y2="357" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
-<rect x="890.0" y="360" width="520" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>
-<text x="1150" y="391.89" font-size="15" fill="#0f172a" font-weight="700" text-anchor="middle">Certifies authoritative fail-closed BARRIER</text>
-<text x="1150" y="417.77" font-size="13" fill="#475569" font-weight="400" text-anchor="middle">revokes stale / unprovable NetworkPolicies</text>
-<text x="1150" y="442.22999999999996" font-size="13" fill="#d97706" font-weight="700" text-anchor="middle">BEFORE declaring Ready</text>
-<line x1="1150" y1="452" x2="1150" y2="493" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
-<rect x="890.0" y="496" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>
-<text x="1150" y="528.9300000000001" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">/ready = 200 (seconds)</text>
-<line x1="1150" y1="552" x2="1150" y2="593" stroke="#16a34a" stroke-width="3" marker-end="url(#arr_16a34a)"/>
-<rect x="890.0" y="596" width="520" height="56" rx="12" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>
-<text x="1150" y="628.93" font-size="17" fill="#16a34a" font-weight="700" text-anchor="middle">rollout OK → deploy OK</text>
-<path d="M 1410.0 524 H 1452 V 596" fill="none" stroke="#d97706" stroke-width="3" stroke-dasharray="7 5" marker-end="url(#arr_d97706)"/>
-<rect x="1336.0" y="596" width="232" height="92" rx="12" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>
-<text x="1452" y="629.44" font-size="13" fill="#0f172a" font-weight="700" text-anchor="middle">ADDITIVE fleet convergence</text>
-<text x="1452" y="653.19" font-size="12" fill="#475569" font-weight="400" text-anchor="middle">async, AFTER Ready</text>
-<text x="1452" y="675.5200000000001" font-size="11" fill="#475569" font-weight="400" text-anchor="middle">(the rest of the policies converge)</text>
-<rect x="1362.0" y="357" width="200.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>
-<text x="1462" y="373" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">isCurrent required (J3)</text>
-<rect x="1382.0" y="389" width="160.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>
-<text x="1462" y="405" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">B3 retain / revoke</text>
-<rect x="1358.0" y="421" width="208.0" height="22" rx="7" fill="#ffffff" stroke="#2563eb" stroke-width="1.4" opacity="1.0"/>
-<text x="1462" y="437" font-size="11" fill="#2563eb" font-weight="700" text-anchor="middle">single-writer Recreate·1</text>
-<text x="1150" y="700" font-size="14" fill="#16a34a" font-weight="700" text-anchor="middle" font-style="italic">Readiness DECOUPLED: gated on the safety barrier, not the fleet</text>
-<rect x="300" y="812" width="26" height="20" rx="5" fill="#dbeafe" stroke="#2563eb" stroke-width="2" opacity="1.0"/>
-<text x="336" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Startup step</text>
-<rect x="506" y="812" width="26" height="20" rx="5" fill="#fee2e2" stroke="#dc2626" stroke-width="2" opacity="1.0"/>
-<text x="542" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Block / failure</text>
-<rect x="742" y="812" width="26" height="20" rx="5" fill="#dcfce7" stroke="#16a34a" stroke-width="2" opacity="1.0"/>
-<text x="778" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Fast Ready / OK</text>
-<rect x="978" y="812" width="26" height="20" rx="5" fill="#fef3c7" stroke="#d97706" stroke-width="2" opacity="1.0"/>
-<text x="1014" y="827" font-size="14" fill="#0f172a" font-weight="400" text-anchor="start">Barrier / async branch</text>
-</svg>
\ No newline at end of file
diff --git a/docs/architecture/pr205-hcc-flows.png b/docs/architecture/pr205-hcc-flows.png
deleted file mode 100644
index d0bbfa3d7..000000000
Binary files a/docs/architecture/pr205-hcc-flows.png and /dev/null differ
diff --git a/docs/architecture/pr205-hcc-flows.svg b/docs/architecture/pr205-hcc-flows.svg
deleted file mode 100644
index adb22120f..000000000
--- a/docs/architecture/pr205-hcc-flows.svg
+++ /dev/null
@@ -1,126 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1280 880" font-family="Inter, Segoe UI, Helvetica, Arial, sans-serif">
-<defs>
-<marker id="aBlue" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#2563eb"/></marker>
-<marker id="aRose" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#dc2626"/></marker>
-<marker id="aViolet" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#7c3aed"/></marker>
-<marker id="aAmber" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#d97706"/></marker>
-<marker id="aGreen" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#059669"/></marker>
-<marker id="aGray" markerWidth="9" markerHeight="8" refX="7.5" refY="4" orient="auto"><path d="M0,0 L9,4 L0,8 Z" fill="#64748b"/></marker>
-<filter id="sh" x="-5%" y="-6%" width="110%" height="122%"><feDropShadow dx="0" dy="1.5" stdDeviation="1.8" flood-color="#0f172a" flood-opacity="0.16"/></filter>
-</defs>
-<rect width="1280" height="880" fill="#ffffff"/>
-<text x="40" y="36" font-size="20" font-weight="800" fill="#0f172a">host-context-controller — flujos que resuelve el PR #205</text>
-<text x="40" y="57" font-size="12.5" fill="#64748b">decouple readiness from initial fleet convergence · seguridad fail-closed · gobernanza single-writer de NetworkPolicy</text>
-<text x="40" y="92" font-size="13" font-weight="700" fill="#1d4ed8">1 · Lifecycle de readiness — DESACOPLADO de la convergencia de la flota</text>
-<rect x="40.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
-<text x="147.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">HCC start</text>
-<text x="147.0" y="146" text-anchor="middle" font-size="11" fill="#475569">4 watch lanes arrancan;</text>
-<text x="147.0" y="162" text-anchor="middle" font-size="11" fill="#475569">una lane que falla → recovery</text>
-<text x="147.0" y="178" text-anchor="middle" font-size="11" fill="#475569">in-process, NO mata el proceso</text>
-<rect x="286.5" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
-<text x="393.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">4-LIST inventory</text>
-<text x="393.5" y="146" text-anchor="middle" font-size="11" fill="#475569">Context ingress · mcp-host</text>
-<text x="393.5" y="162" text-anchor="middle" font-size="11" fill="#475569">egress · rpc-proxy egress</text>
-<text x="393.5" y="178" text-anchor="middle" font-size="11" fill="#475569">· external egress</text>
-<rect x="533.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
-<text x="640.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Certificar safety inventory</text>
-<text x="640.0" y="146" text-anchor="middle" font-size="11" fill="#475569">revoca allows stale/</text>
-<text x="640.0" y="162" text-anchor="middle" font-size="11" fill="#475569">improbables ANTES de</text>
-<text x="640.0" y="178" text-anchor="middle" font-size="11" fill="#475569">certificar (fail-closed)</text>
-<rect x="779.5" y="104" width="214" height="92" rx="11" fill="#dbeafe" stroke="#1d4ed8" stroke-width="1.8" filter="url(#sh)"/>
-<text x="886.5" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">/ready = 200</text>
-<text x="886.5" y="146" text-anchor="middle" font-size="11" fill="#475569">solo si el inventory está</text>
-<text x="886.5" y="162" text-anchor="middle" font-size="11" fill="#475569">certificado y ningún allow</text>
-<text x="886.5" y="178" text-anchor="middle" font-size="11" fill="#475569">stale sobrevive</text>
-<rect x="1026.0" y="104" width="214" height="92" rx="11" fill="#eff6ff" stroke="#2563eb" stroke-width="1.8" filter="url(#sh)"/>
-<text x="1133.0" y="127" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Additive convergence</text>
-<text x="1133.0" y="146" text-anchor="middle" font-size="11" fill="#475569">la flota converge DESPUÉS</text>
-<text x="1133.0" y="162" text-anchor="middle" font-size="11" fill="#475569">del ready — asíncrono,</text>
-<text x="1133.0" y="178" text-anchor="middle" font-size="11" fill="#475569">sin bloquearlo</text>
-<line x1="254.0" y1="150.0" x2="286.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>
-<rect x="229.05" y="130.0" width="82.4" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="270.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">inventariar</text>
-<line x1="500.5" y1="150.0" x2="533.0" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>
-<rect x="478.75" y="130.0" width="76.0" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="516.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">certificar</text>
-<line x1="747.0" y1="150.0" x2="779.5" y2="150.0" stroke="#2563eb" stroke-width="2.6" marker-end="url(#aBlue)" stroke-linecap="round"/>
-<rect x="738.05" y="130.0" width="50.400000000000006" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="763.25" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#1e293b">/ready</text>
-<line x1="993.5" y1="150.0" x2="1026.0" y2="150.0" stroke="#64748b" stroke-width="2.4" stroke-dasharray="6,4" marker-end="url(#aGray)" stroke-linecap="round"/>
-<rect x="946.15" y="130.0" width="127.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="1009.75" y="142.0" text-anchor="middle" font-size="10" font-weight="700" fill="#475569">async · post-ready</text>
-<text x="40" y="275" font-size="13" font-weight="700" fill="#b91c1c">2 · Fences fail-closed (R3 · R4 · J3)</text>
-<rect x="40.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
-<text x="230.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Readiness gate — default () =&gt; false</text>
-<text x="230.0" y="326" text-anchor="middle" font-size="11" fill="#475569">/ready = ready &amp;&amp; providerAuthoritativeFn()</text>
-<text x="230.0" y="342" text-anchor="middle" font-size="11" fill="#475569">throw → 503 · nunca un 200 equivocado</text>
-<rect x="450.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
-<text x="640.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">G1 — deny sin inventory-authority</text>
-<text x="640.0" y="326" text-anchor="middle" font-size="11" fill="#475569">currentWhenUnconfigured() → false + warn</text>
-<text x="640.0" y="342" text-anchor="middle" font-size="11" fill="#475569">dev nunca construye el reconciler</text>
-<rect x="860.0" y="284" width="380.0" height="76" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
-<text x="1050.0" y="307" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">F1 — rebind del readiness-poll</text>
-<text x="1050.0" y="326" text-anchor="middle" font-size="11" fill="#475569">re-liga isCurrent/server en reconciles</text>
-<text x="1050.0" y="342" text-anchor="middle" font-size="11" fill="#475569">superpuestos → la ventana no se autodestruye</text>
-<text x="40" y="421" font-size="13" font-weight="700" fill="#6d28d9">3 · Egress externo DNS — contrato B3 (retener/revocar) + sliding-window #299</text>
-<rect x="40" y="438" width="224" height="80" rx="11" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>
-<text x="152.0" y="461" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">DNS binding observado</text>
-<text x="152.0" y="480" text-anchor="middle" font-size="11" fill="#475569">resolveExternalEgressDns</text>
-<text x="152.0" y="496" text-anchor="middle" font-size="11" fill="#475569">timeout 5s + fence isCurrent</text>
-<polygon points="372,436 472,478 372,520 272,478" fill="#f5f3ff" stroke="#7c3aed" stroke-width="1.8" filter="url(#sh)"/>
-<text x="372" y="475" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">¿identity</text>
-<text x="372" y="491" text-anchor="middle" font-size="12" font-weight="700" fill="#0f172a">unchanged?</text>
-<line x1="264" y1="478" x2="272" y2="478" stroke="#7c3aed" stroke-width="2.6" marker-end="url(#aViolet)" stroke-linecap="round"/>
-<rect x="540" y="414" width="306" height="60" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
-<text x="693.0" y="437" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">RETAIN — allow identity-stable</text>
-<text x="693.0" y="456" text-anchor="middle" font-size="11" fill="#475569">mismo protocolo/puerto/ownership/cidrs</text>
-<text x="693.0" y="472" text-anchor="middle" font-size="11" fill="#475569">· HCC sigue Ready (módulo cidr)</text>
-<rect x="540" y="496" width="306" height="60" rx="11" fill="#fef2f2" stroke="#dc2626" stroke-width="1.8" filter="url(#sh)"/>
-<text x="693.0" y="519" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">REVOKE antes de readiness</text>
-<text x="693.0" y="538" text-anchor="middle" font-size="11" fill="#475569">drift TCP→UDP · port · ownership · zero-cidr</text>
-<text x="693.0" y="554" text-anchor="middle" font-size="11" fill="#475569">→ unprovable → cleanup fail-closed</text>
-<path d="M 472,470 H 506.0 V 444 H 540" fill="none" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linejoin="round"/>
-<rect x="423.6" y="415" width="88.80000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="468" y="427" text-anchor="middle" font-size="10" font-weight="700" fill="#047857">sí · retiene</text>
-<path d="M 472,486 H 506.0 V 526 H 540" fill="none" stroke="#dc2626" stroke-width="2.6" marker-end="url(#aRose)" stroke-linejoin="round"/>
-<rect x="417.2" y="523" width="101.60000000000001" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="468" y="535" text-anchor="middle" font-size="10" font-weight="700" fill="#b91c1c">drift · revoca</text>
-<rect x="892" y="430" width="348" height="88" rx="11" fill="#fffbeb" stroke="#d97706" stroke-width="1.8" filter="url(#sh)"/>
-<text x="1066.0" y="453" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ExternalEgressConvergenceCoordinator</text>
-<text x="1066.0" y="472" text-anchor="middle" font-size="11" fill="#475569">resync periódico TTL-aware (≤ TTL/2, floor)</text>
-<text x="1066.0" y="488" text-anchor="middle" font-size="11" fill="#475569">sliding-window accumulate · provider CIDRs</text>
-<text x="1066.0" y="504" text-anchor="middle" font-size="11" fill="#475569">fence interval ≤ overlap/2 · limiter</text>
-<line x1="846" y1="444" x2="892" y2="460" stroke="#d97706" stroke-width="2.6" marker-end="url(#aAmber)" stroke-linecap="round"/>
-<rect x="821.4" y="425" width="95.2" height="16" rx="4" fill="#ffffff" stroke="#e2e8f0" stroke-width="0.8"/>
-<text x="869" y="437" text-anchor="middle" font-size="10" font-weight="700" fill="#b45309">add (aditiva)</text>
-<text x="40" y="575" font-size="13" font-weight="700" fill="#047857">4 · Seam de rotación #223 (preservado) &amp; invariante de deploy single-writer</text>
-<rect x="40" y="584" width="236" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
-<text x="158.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">validateSecret()</text>
-<text x="158.0" y="626" text-anchor="middle" font-size="11" fill="#475569">sha256(canonicalJSON(secret.data))</text>
-<rect x="306" y="584" width="306" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
-<text x="459.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">ensureDeployment(rev, isCurrent)</text>
-<text x="459.0" y="626" text-anchor="middle" font-size="11" fill="#475569">el fence de #205 se propaga aquí</text>
-<rect x="642" y="584" width="300" height="66" rx="11" fill="#ecfdf5" stroke="#059669" stroke-width="1.8" filter="url(#sh)"/>
-<text x="792.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">pod annotation</text>
-<text x="792.0" y="626" text-anchor="middle" font-size="11" fill="#475569">clerum.io/credentials-revision → rueda 1 vez</text>
-<line x1="276" y1="617" x2="306" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>
-<line x1="612" y1="617" x2="642" y2="617" stroke="#059669" stroke-width="2.6" marker-end="url(#aGreen)" stroke-linecap="round"/>
-<rect x="972" y="584" width="268" height="66" rx="11" fill="#f8fafc" stroke="#475569" stroke-width="1.8" filter="url(#sh)"/>
-<text x="1106.0" y="607" text-anchor="middle" font-size="14" font-weight="700" fill="#0f172a">Deploy single-writer</text>
-<text x="1106.0" y="626" text-anchor="middle" font-size="11" fill="#475569">strategy: Recreate · replicas 1</text>
-<text x="1106.0" y="642" text-anchor="middle" font-size="11" fill="#475569">· progressDeadlineSeconds 1200</text>
-<text x="40" y="700" font-size="11.5" font-weight="700" fill="#334155">Leyenda</text>
-<rect x="118" y="688" width="14" height="14" rx="3" fill="#2563eb" opacity="0.9"/>
-<text x="137" y="700" font-size="11" fill="#475569">lifecycle readiness</text>
-<rect x="282.6" y="688" width="14" height="14" rx="3" fill="#dc2626" opacity="0.9"/>
-<text x="301.6" y="700" font-size="11" fill="#475569">fence fail-closed / revoke</text>
-<rect x="492.0" y="688" width="14" height="14" rx="3" fill="#7c3aed" opacity="0.9"/>
-<text x="511.0" y="700" font-size="11" fill="#475569">egress DNS (B3)</text>
-<rect x="631.0" y="688" width="14" height="14" rx="3" fill="#d97706" opacity="0.9"/>
-<text x="650.0" y="700" font-size="11" fill="#475569">coordinator #299</text>
-<rect x="776.4" y="688" width="14" height="14" rx="3" fill="#059669" opacity="0.9"/>
-<text x="795.4" y="700" font-size="11" fill="#475569">retain / rotación #223</text>
-<rect x="960.2" y="688" width="14" height="14" rx="3" fill="#475569" opacity="0.9"/>
-<text x="979.2" y="700" font-size="11" fill="#475569">invariante deploy</text>
-<text x="40" y="858" font-size="10.5" fill="#94a3b8">PR #205 · evenfire-ai/evenfire · el /ready certifica una frontera de seguridad autoritativa ANTES de que la flota converja — la convergencia restante es aditiva y asíncrona</text>
-</svg>
\ No newline at end of file
diff --git a/external-rest-api/src/releaseManifest.ts b/external-rest-api/src/releaseManifest.ts
index 0ae6125bb..e1227184a 100644
--- a/external-rest-api/src/releaseManifest.ts
+++ b/external-rest-api/src/releaseManifest.ts
@@ -9,7 +9,7 @@ export type ReleaseManifest = {
export const releaseManifest: ReleaseManifest = {
releaseId: 'v0.7.0',
externalRestApiVersion: '0.1.80',
- rpcProxyVersion: '0.1.65',
+ rpcProxyVersion: '0.1.64',
desktopVersion: '0.7.0',
minimumDesktopVersion: '0.1.252',
}
diff --git a/host-context-controller/README.md b/host-context-controller/README.md
index 169458690..8ee40e981 100644
--- a/host-context-controller/README.md
+++ b/host-context-controller/README.md
@@ -39,8 +39,6 @@ When an `McpServer` CRD is created, modified, or deleted, the host-context-contr
- **MODIFIED** — Updates the Deployment and Service to match the CRD spec (env vars, image, ports, resources, etc.).
- **DELETED** — Removes the Deployment and Service.
-Remote egress-proxy McpServers keep the author's `spec.image` as desired-state input; HCC no longer rewrites it to the platform proxy image. The former `canonicalizeRemoteEgressProxyImage` step and its `ImageCanonicalized` status condition were removed (PR #205 readiness-safety cleanup): canonicalization mutated the observed CRD cache object and bumped the resource generation, retiring the in-flight reconcile before it could create the runtime resources. The platform-owned proxy image is instead selected at Deployment build time (`buildDeployment`), leaving the CRD spec untouched.
-
The reconciler builds Deployments with labels used for pod selection and network policy targeting:
| Label | Value | Purpose |
@@ -99,21 +97,7 @@ Servers removed from a Context lose their policies on the next reconcile; deleti
- **`exact-host`** (default) — a CIDR binding, or a public DNS hostname resolved to `/32` ipBlocks, on one declared port/protocol. CIDRs and resolved IPs are rejected if they overlap `PUBLIC_EGRESS_EXCEPT_CIDRS` (RFC1918, CGNAT, loopback, link-local incl. cloud metadata, documentation, benchmarking, multicast, reserved ranges). Hostnames like `localhost`, `metadata.goog`, `*.internal`, `*.svc`, `*.cluster.local` are rejected outright.
- **`public-web`** — `0.0.0.0/0` on TCP 80/443 with `PUBLIC_EGRESS_EXCEPT_CIDRS` carved out via `except`.
-Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS-derived allows are never retained across a failed refresh: HCC revokes the unprovable policy before certifying global readiness, while only the affected runtime remains blocked until DNS recovers. Each lookup has a bounded deadline.
-
-HCC readiness certifies authoritative inventory and revocation safety, not
-completion of positive policy creation. A renamed or newly requested allow may
-therefore remain unavailable after HCC becomes Ready until its additive
-reconciliation succeeds. HCC never keeps an obsolete allow merely to avoid
-that availability gap: when old and new ownership cannot be proven equivalent,
-revocation wins and only the affected runtime remains fail-closed.
-
-During an in-place release, HCC is also the controller that stamps
-`CONTEXT_MAPPER_HOST_IMAGE` onto managed mcp-host workloads. The new HCC
-producer is consequently running before it replaces those consumers with the
-new mcp-host image. For compatibility, an omitted `status.authoritative` field
-retains the legacy fail-closed meaning; only an explicit `authoritative: false`
-asks a new mcp-host to preserve an unchanged live connection.
+Results are written to the McpServer's `status.conditions` (`ExternalEgressReady`) and `status.resolvedEgressIPs`. A failed binding blocks runtime reconciliation of that server, so a workload cannot start before its egress converges. DNS answers are re-resolved periodically (`HCC_EXTERNAL_EGRESS_RESYNC_SEC`, default 300s).
### Recipe binding policies
@@ -121,8 +105,7 @@ McpServers carrying a `clerum.io/recipe-bindings` annotation (set for WorkflowRe
### Reconciliation Lifecycle
-- **Startup safety** — Ensures L0/L1 defaults, revokes stale or orphaned context-allow, rpc-proxy-egress, and external-egress policies from authoritative Context/McpServer inventories, then certifies readiness safety.
-- **Startup additive convergence** — Reconciles Context allows after certification. The dedicated external-egress coordinator is the sole owner of McpServer egress creation, per-server startup gating, retry, and periodic DNS refresh; a failed binding blocks only that server's runtime.
+- **Startup** — Ensures L0/L1 defaults, reconciles all Contexts and all McpServer egress bindings, then deletes orphaned context-allow policies in the MCP server namespace, plus orphaned rpc-proxy-egress and external-egress policies, whose owning CRD no longer exists (`fullReconcile`).
- **Context ADDED/MODIFIED/DELETED** — Creates, updates, or removes the L2 policy set for that context.
- **McpServer ADDED/MODIFIED/DELETED** — Reconciles L3 external egress (before the workload Deployment is created) and recipe binding policies.
@@ -284,17 +267,16 @@ a permissive policy — see the NetworkPolicy and GFS tables:
### MCP server provisioning
-| Variable | Default | Description |
-| -------------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------------------------------------- |
-| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |
-| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |
-| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |
-| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync interval (seconds). `0` disables periodic refresh; watch events still reconcile. |
-| `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS` | `5000` | Per-hostname DNS lookup deadline in milliseconds. |
+| Variable | Default | Description |
+| -------------------------------------------- | ----------------------------------- | ----------------------------------------------------------------------------------------------------------- |
+| `CONTEXT_MAPPER_MCPSERVER_IMAGE_PULL_POLICY` | `IfNotPresent` | Pull policy for reconciled McpServer workloads. |
+| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | `clerum/nginx-egress-proxy:0.1.0` | Per-server nginx egress proxy for **remote** MCP servers (`spec.remote`). |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_IMAGE` | (registry-qualified `stdio-bridge`) | stdio-bridge sidecar image for managed stdio MCP servers. Manifest pins `clerum/stdio-bridge:0.9.5`. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_MEMORY` | `32Mi` | stdio-bridge sidecar memory request. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_REQUEST_CPU` | `50m` | stdio-bridge sidecar CPU request. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_MEMORY` | `128Mi` | stdio-bridge sidecar memory limit. |
+| `CONTEXT_MAPPER_STDIO_BRIDGE_LIMIT_CPU` | `200m` | stdio-bridge sidecar CPU limit. |
+| `HCC_EXTERNAL_EGRESS_RESYNC_SEC` | `300` | Periodic external-egress DNS resync (seconds), so DNS changes converge without a watch event. `0` disables. |
### mcp-host provisioning
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 9bcfc9c0f..e9b28c157 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.223",
+ "version": "0.1.170",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.223",
+ "version": "0.1.170",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 42fe8f398..17c98b81a 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.223",
+ "version": "0.1.170",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
index f4e15bf35..f8d6857cb 100644
--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
@@ -119,7 +119,6 @@ describe('connector credentials revision (issue #223)', () => {
data: { 'api-key': stored('old-key') },
})
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -221,7 +220,6 @@ describe('generation-aware rollout readiness (issue #223)', () => {
vi.clearAllMocks()
coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -324,7 +322,6 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
vi.useFakeTimers()
coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -551,111 +548,3 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)
})
})
-
-describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
- let reconciler: McpServerReconciler
- // The inventory-authority generation IS mcpWatchGeneration in production
- // (k8sClient wiring): a routine McpServer watch recovery bumps it.
- let authorityGeneration: number
- let currentServer: McpServerCRD
-
- // A rollout mid-flight (total 2 vs updated 1) that never converges on its own.
- const midRollout = {
- metadata: { generation: 2, labels: {} },
- spec: { replicas: 1 },
- status: {
- observedGeneration: 2,
- replicas: 2,
- updatedReplicas: 1,
- readyReplicas: 1,
- availableReplicas: 1,
- unavailableReplicas: 0,
- },
- }
-
- beforeEach(() => {
- vi.clearAllMocks()
- vi.useFakeTimers()
- authorityGeneration = 1
- currentServer = makeServer()
- coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(coreApi),
- customApi: asCustomApi(customApi),
- })
- reconciler.setInventoryAuthority(() => ({ known: true, generation: authorityGeneration }))
- reconciler.setResolveCurrentServer(name => (name === 'linear' ? currentServer : undefined))
- })
-
- afterEach(() => {
- vi.useRealTimers()
- })
-
- it('rebinds the fence so a converged rollout is recorded after a watch-generation bump (T1)', async () => {
- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
- // R1 arms the window under authority generation 1 (timer pending).
- await reconciler.reconcile(makeServer())
- // A routine McpServer watch recovery bumps the inventory-authority generation.
- authorityGeneration = 2
- // R2, current under generation 2, finds R1's pending window and rebinds it.
- await reconciler.reconcile(makeServer())
- // The rollout converges before the (single) pending tick fires.
- appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
- await vi.advanceTimersByTimeAsync(5000)
-
- // Without the rebind, R1's tick evaluates its stale generation-1 fence,
- // self-deletes, and DeploymentReady=True is never written.
- expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
- expect(
- (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls.size
- ).toBe(0)
- })
-
- it('control: without the generation bump the rollout converges the same way (T2)', async () => {
- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
- await reconciler.reconcile(makeServer())
- await reconciler.reconcile(makeServer()) // no bump
- appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(deploymentReadyWrites(customApi).at(-1)?.status).toBe('True')
- })
-
- it('rebind does NOT reset the attempt budget across the bump (E2 guard, T3)', async () => {
- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout) // never converges
- await reconciler.reconcile(makeServer())
- await vi.advanceTimersByTimeAsync(2 * 5000) // 2 attempts consumed
- authorityGeneration = 2
- await reconciler.reconcile(makeServer()) // rebind — must NOT reset attempts
- await vi.advanceTimersByTimeAsync(21 * 5000) // 2 + 21 = 23 attempts, still short of 24
-
- expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(false)
- await vi.advanceTimersByTimeAsync(5000) // the 24th tick
- expect(deploymentReadyWrites(customApi).some(w => w.reason === 'RolloutIncomplete')).toBe(true)
- })
-
- it('does NOT adopt a same-name recreate — a new uid gets a fresh window (T4)', async () => {
- appsApi.readNamespacedDeployment.mockResolvedValue(midRollout)
- await reconciler.reconcile(makeServer())
- const polls = (reconciler as unknown as { readinessPolls: Map<string, { attempts: number }> })
- .readinessPolls
- const firstWindow = polls.get('linear')
- await vi.advanceTimersByTimeAsync(12 * 5000)
-
- // Same name, NEW uid: identity retirement (beginStatusTracking), not a
- // rebind, must own this — a rebind adopting the recreate would carry the
- // old window's spent budget.
- const recreated = { ...makeServer(), uid: 'uid-linear-9999' }
- currentServer = recreated
- await reconciler.reconcile(recreated)
-
- const secondWindow = polls.get('linear')
- expect(secondWindow).not.toBe(firstWindow)
- expect(secondWindow?.attempts).toBe(0)
- })
-})
diff --git a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
index fa30ec7a5..a4b425ae7 100644
--- a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
@@ -89,7 +89,6 @@ describe('McpServer generated pod hardening', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.ts
index 044452efd..09c0b5b84 100644
--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts
@@ -87,7 +87,6 @@ function buildReconciler(
customApi: MockCustomApi
): McpServerReconciler {
return new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -172,14 +171,7 @@ function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {
const calls = customApi.patchNamespacedCustomObjectStatus.mock.calls
const lastCall = calls[calls.length - 1]?.[0]
const patch = lastCall?.body
- const statusAdd = Array.isArray(patch)
- ? patch.find(
- candidate =>
- candidate?.op === 'add' &&
- (candidate.path === '/status' || candidate.path === '/status/conditions')
- )
- : undefined
- const value = statusAdd?.value
+ const value = Array.isArray(patch) ? patch[0]?.value : undefined
return Array.isArray(value) ? value : (value?.conditions ?? [])
}
@@ -676,6 +668,105 @@ describe('McpServerReconciler remote egress proxy', () => {
})
})
+ describe('canonicalizeRemoteEgressProxyImage', () => {
+ it('patches stale remote McpServer spec.image to the platform egress proxy image', async () => {
+ const staleRemote = cloneServer(REMOTE_SERVER, {
+ image: LEGACY_REMOTE_EGRESS_IMAGE,
+ })
+
+ await (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)
+
+ expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(
+ expect.objectContaining({
+ group: 'clerum.io',
+ version: 'v1alpha1',
+ namespace: 'mcp-server',
+ plural: 'mcpservers',
+ name: 'mcp-sentry-remote',
+ body: {
+ spec: {
+ image: 'clerum/nginx-egress-proxy:0.1.0',
+ },
+ },
+ }),
+ expect.objectContaining({
+ middleware: expect.any(Array),
+ })
+ )
+ expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')
+ expect(lastPatchedStatusConditions(customApi)).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: 'ImageCanonicalized',
+ status: 'True',
+ reason: 'RemoteEgressProxyImageMatches',
+ message: 'Remote McpServer spec.image matches the platform egress proxy image',
+ }),
+ ])
+ )
+ })
+
+ it('does not patch spec for a remote McpServer already using the platform image', async () => {
+ const currentRemote = cloneServer(REMOTE_SERVER)
+
+ await (reconciler as any).canonicalizeRemoteEgressProxyImage(currentRemote)
+
+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
+ expect(lastPatchedStatusConditions(customApi)).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: 'ImageCanonicalized',
+ status: 'True',
+ reason: 'RemoteEgressProxyImageMatches',
+ message: 'Remote McpServer spec.image matches the platform egress proxy image',
+ }),
+ ])
+ )
+ })
+
+ it('does not patch local McpServers even when their image differs', async () => {
+ const localServer = cloneServer(LOCAL_SERVER, {
+ image: 'clerum/custom-local-mcp:1.2.3',
+ })
+
+ await (reconciler as any).canonicalizeRemoteEgressProxyImage(localServer)
+
+ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(localServer.spec.image).toBe('clerum/custom-local-mcp:1.2.3')
+ })
+
+ it('does not throw and records status when declarative image patching fails', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const staleRemote = cloneServer(REMOTE_SERVER, {
+ image: LEGACY_REMOTE_EGRESS_IMAGE,
+ })
+ customApi.patchNamespacedCustomObject.mockRejectedValueOnce(new Error('rbac denied'))
+
+ await expect(
+ (reconciler as any).canonicalizeRemoteEgressProxyImage(staleRemote)
+ ).resolves.toBeUndefined()
+
+ expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining('Failed to canonicalize remote McpServer'),
+ expect.any(Error)
+ )
+ expect(lastPatchedStatusConditions(customApi)).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: 'ImageCanonicalized',
+ status: 'False',
+ reason: 'RemoteEgressProxyImagePatchFailed',
+ message:
+ 'Failed to patch remote McpServer spec.image to the platform egress proxy image',
+ }),
+ ])
+ )
+ warn.mockRestore()
+ })
+ })
+
// ─── Test 3: buildDeployment for local servers ─────────────────────────
describe('buildDeployment for local servers', () => {
@@ -744,12 +835,16 @@ describe('McpServerReconciler remote egress proxy', () => {
expect(cmIndex).toBeLessThan(deployIndex)
})
- it('should use the platform image without rewriting a divergent remote desired image', async () => {
+ it('should canonicalize stale remote image before syncing ConfigMap and Deployment', async () => {
const callOrder: string[] = []
const staleRemote = cloneServer(REMOTE_SERVER, {
image: LEGACY_REMOTE_EGRESS_IMAGE,
})
+ customApi.patchNamespacedCustomObject.mockImplementation(async () => {
+ callOrder.push('canonicalizeImage')
+ return {}
+ })
coreApi.createNamespacedConfigMap.mockImplementation(async () => {
callOrder.push('createConfigMap')
return {}
@@ -765,28 +860,23 @@ describe('McpServerReconciler remote egress proxy', () => {
await reconciler.reconcile(staleRemote)
- expect(callOrder).toEqual(['createConfigMap', 'createService', 'createDeployment'])
- expect(staleRemote.spec.image).toBe(LEGACY_REMOTE_EGRESS_IMAGE)
- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(
- expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),
- expect.anything()
- )
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(
+ expect(callOrder).toEqual([
+ 'canonicalizeImage',
+ 'createConfigMap',
+ 'createService',
+ 'createDeployment',
+ ])
+ expect(staleRemote.spec.image).toBe('clerum/nginx-egress-proxy:0.1.0')
+ expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(
expect.objectContaining({
- body: expect.objectContaining({
- spec: expect.objectContaining({
- template: expect.objectContaining({
- spec: expect.objectContaining({
- containers: [
- expect.objectContaining({
- name: 'egress-proxy',
- image: 'clerum/nginx-egress-proxy:0.1.0',
- }),
- ],
- }),
- }),
- }),
- }),
+ body: {
+ spec: {
+ image: 'clerum/nginx-egress-proxy:0.1.0',
+ },
+ },
+ }),
+ expect.objectContaining({
+ middleware: expect.any(Array),
})
)
})
@@ -1025,12 +1115,6 @@ describe('McpServerReconciler remote egress proxy', () => {
})
describe('reconcileDelete for remote servers', () => {
- beforeEach(() => {
- reconciler.setInventoryAuthority(() => ({ known: true, generation: 1 }))
- reconciler.setResolveCurrentServer(() => undefined)
- customApi.getNamespacedCustomObject.mockRejectedValue(make404Error())
- })
-
it('should clean up ConfigMap via deleteResources', async () => {
mockHccOwnedRuntimeReads(appsApi, coreApi)
@@ -1052,14 +1136,11 @@ describe('McpServerReconciler remote egress proxy', () => {
})
it('should clear status tracking after delete', async () => {
- let current: McpServerCRD | undefined = REMOTE_SERVER
- reconciler.setResolveCurrentServer(() => current)
// First reconcile to populate status
await reconciler.reconcile(REMOTE_SERVER)
expect(reconciler.getStatus('mcp-sentry-remote').deployed).toBe(true)
// Then delete
- current = undefined
await reconciler.reconcileDelete('mcp-sentry-remote', 'mcp-server')
const status = reconciler.getStatus('mcp-sentry-remote')
expect(status.deployed).toBe(false)
diff --git a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
index 38b7c571f..b3d979815 100644
--- a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
@@ -91,7 +91,6 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -123,19 +122,6 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {
const container = capturedContainer(appsApi)
expect(container.imagePullPolicy).toBe('IfNotPresent')
})
-
- it('does not mutate a watched desired spec while building the sanitized Deployment', async () => {
- const server = makeServer({
- imagePullPolicy: 'Always',
- security: { runAsUser: 0, addCapabilities: ['CHOWN', 'SYS_ADMIN'] },
- })
- const originalSpec = structuredClone(server.spec)
-
- await reconciler.reconcile(server)
-
- expect(capturedContainer(appsApi).imagePullPolicy).toBe('IfNotPresent')
- expect(server.spec).toEqual(originalSpec)
- })
})
// ════════════════════════════════════════════════════════════════════
diff --git a/host-context-controller/src/bindingPolicyReconciler.test.ts b/host-context-controller/src/bindingPolicyReconciler.test.ts
index 48d6fe191..b2fd2bc8f 100644
--- a/host-context-controller/src/bindingPolicyReconciler.test.ts
+++ b/host-context-controller/src/bindingPolicyReconciler.test.ts
@@ -2,14 +2,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'
-function deferred<T = void>() {
- let resolve!: (value: T | PromiseLike<T>) => void
- const promise = new Promise<T>(res => {
- resolve = res
- })
- return { promise, resolve }
-}
-
function makeMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
@@ -48,9 +40,7 @@ describe('BindingPolicyReconciler', () => {
const singleBinding: BindingDef[] = [{ from: 'redis-mcp', to: 'redis', port: 6379 }]
it('creates egress + ingress policies for a binding', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
- isCurrent: () => true,
- })
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
// 2 calls: 1 egress + 1 ingress
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
@@ -85,9 +75,7 @@ describe('BindingPolicyReconciler', () => {
})
it('applies binding-allow labels to all policies', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
- isCurrent: () => true,
- })
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
for (const call of mockApi.createNamespacedNetworkPolicy.mock.calls) {
const labels = (call[0] as { body: { metadata: { labels: Record<string, string> } } }).body
@@ -101,43 +89,12 @@ describe('BindingPolicyReconciler', () => {
it('handles 409 conflict with create-or-replace pattern', async () => {
mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
- isCurrent: () => true,
- })
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()
})
- it('does not replace or start another apply after its authority lease is retired', async () => {
- const readStarted = deferred()
- const releaseRead = deferred<{ metadata: { resourceVersion: string } }>()
- mockApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
- mockApi.readNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- readStarted.resolve()
- return releaseRead.promise
- })
- let current = true
-
- const reconcile = reconciler.reconcileBindings(
- 'my-recipe',
- singleBinding,
- 'redis-mcp',
- 'redis-mcp',
- { isCurrent: () => current }
- )
- await readStarted.promise
-
- current = false
- releaseRead.resolve({ metadata: { resourceVersion: '1' } })
- await reconcile
-
- // The first create was authorized and returned 409 without mutating. Once
- // the lease is retired, no replacement or second policy apply may start.
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
it('cleanupBindings deletes policies in both namespaces', async () => {
mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
@@ -151,65 +108,8 @@ describe('BindingPolicyReconciler', () => {
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalled()
})
- it('does not delete listed policies when a same-name recipe is recreated during LIST', async () => {
- const bothListsStarted = deferred()
- const releaseLists = deferred<{
- items: Array<{ metadata: { name: string } }>
- }>()
- let listCalls = 0
- mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {
- listCalls += 1
- if (listCalls === 2) bothListsStarted.resolve()
- return releaseLists.promise
- })
- let recipeRecreated = false
- const deleteAllowed = vi.fn(async () => !recipeRecreated)
-
- const cleanup = reconciler.cleanupBindings('my-recipe', { deleteAllowed })
- await bothListsStarted.promise
-
- recipeRecreated = true
- releaseLists.resolve({
- items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
- })
- await cleanup
-
- expect(deleteAllowed).toHaveBeenCalledTimes(2)
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('rejects cleanup when listing binding policies fails', async () => {
- mockApi.listNamespacedNetworkPolicy.mockRejectedValue(new Error('transient LIST failure'))
-
- await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(
- 'Failed to clean up binding policies for recipe "my-recipe"'
- )
- })
-
- it('rejects cleanup when deleting a binding policy fails with a non-404 error', async () => {
- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
- items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
- })
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('transient DELETE failure'))
-
- await expect(reconciler.cleanupBindings('my-recipe')).rejects.toThrow(
- 'Failed to clean up binding policies for recipe "my-recipe"'
- )
- })
-
- it('treats a 404 while deleting binding policies as successful cleanup', async () => {
- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
- items: [{ metadata: { name: 'bind-my-recipe-a-b-egress' } }],
- })
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
-
- await expect(reconciler.cleanupBindings('my-recipe')).resolves.toBeUndefined()
- })
-
it('uses _from for ingress rules (K8s client ObjectSerializer maps _from → from)', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
- isCurrent: () => true,
- })
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
const ingressCall = calls.find(
@@ -226,9 +126,7 @@ describe('BindingPolicyReconciler', () => {
})
it('uses correct pod selectors and namespace-aware placement', async () => {
- await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp', 'redis-mcp', {
- isCurrent: () => true,
- })
+ await reconciler.reconcileBindings('my-recipe', singleBinding, 'redis-mcp')
const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
const egressCall = calls.find(
@@ -298,8 +196,7 @@ describe('BindingPolicyReconciler', () => {
'my-recipe',
singleBinding,
'redis-mcp',
- 'my-recipe-redis-mcp',
- { isCurrent: () => true }
+ 'my-recipe-redis-mcp'
)
const calls = mockApi.createNamespacedNetworkPolicy.mock.calls
diff --git a/host-context-controller/src/bindingPolicyReconciler.ts b/host-context-controller/src/bindingPolicyReconciler.ts
index 0281e69f6..b0486d63c 100644
--- a/host-context-controller/src/bindingPolicyReconciler.ts
+++ b/host-context-controller/src/bindingPolicyReconciler.ts
@@ -37,18 +37,6 @@ export interface BindingDef {
protocol?: 'TCP' | 'UDP'
}
-export interface BindingPolicyReconcileOptions {
- // Required (fail-closed): every caller must supply the authority predicate so
- // a retired reconcile pass cannot keep mutating binding NetworkPolicies. There
- // is no assume-current default — an omitted predicate is a compile error, not
- // a silent fail-open (the old `?? (() => true)`) nor a silent no-op.
- isCurrent: () => boolean
-}
-
-export interface BindingPolicyCleanupOptions {
- deleteAllowed?: () => Promise<boolean>
-}
-
export class BindingPolicyReconciler {
private networkingApi: k8s.NetworkingV1Api
private mcpServerNamespace: string
@@ -72,86 +60,70 @@ export class BindingPolicyReconciler {
recipeName: string,
bindings: BindingDef[],
mcpWorkloadName: string,
- mcpServerName = mcpWorkloadName,
- options: BindingPolicyReconcileOptions
+ mcpServerName = mcpWorkloadName
): Promise<void> {
- const isCurrent = options.isCurrent
- if (!isCurrent()) return
console.log(`[BindingNP] Reconciling ${bindings.length} binding(s) for recipe "${recipeName}"`)
- // Sequence writes so authority can be rechecked between every Kubernetes
- // effect. A retired pass must not start the next binding policy mutation.
- for (const binding of bindings) {
- if (!isCurrent()) return
- const protocol = binding.protocol ?? 'TCP'
+ await Promise.all(
+ bindings.map(async binding => {
+ const protocol = binding.protocol ?? 'TCP'
- // Determine namespaces based on which workload is the MCP server.
- // The MCP workload (with transport) lives in mcp-server namespace,
- // the other workload lives in sandbox-recipes namespace.
- const fromIsMcp = binding.from === mcpWorkloadName
- const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE
- const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace
- const fromPodSelector: Record<string, string> = fromIsMcp
- ? { [MCPSERVER_LABEL]: mcpServerName }
- : { app: binding.from }
- const toPodSelector: Record<string, string> = fromIsMcp
- ? { app: binding.to }
- : { [MCPSERVER_LABEL]: mcpServerName }
+ // Determine namespaces based on which workload is the MCP server.
+ // The MCP workload (with transport) lives in mcp-server namespace,
+ // the other workload lives in sandbox-recipes namespace.
+ const fromIsMcp = binding.from === mcpWorkloadName
+ const fromNs = fromIsMcp ? this.mcpServerNamespace : SANDBOX_NAMESPACE
+ const toNs = fromIsMcp ? SANDBOX_NAMESPACE : this.mcpServerNamespace
+ const fromPodSelector: Record<string, string> = fromIsMcp
+ ? { [MCPSERVER_LABEL]: mcpServerName }
+ : { app: binding.from }
+ const toPodSelector: Record<string, string> = fromIsMcp
+ ? { app: binding.to }
+ : { [MCPSERVER_LABEL]: mcpServerName }
- const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')
- const egressPolicy = this.buildEgressPolicy(
- egressName,
- recipeName,
- fromPodSelector,
- toPodSelector,
- fromNs,
- toNs,
- binding.port,
- protocol
- )
- const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')
- const ingressPolicy = this.buildIngressPolicy(
- ingressName,
- recipeName,
- fromPodSelector,
- toPodSelector,
- fromNs,
- toNs,
- binding.port,
- protocol
- )
+ const egressName = this.policyName(recipeName, binding.from, binding.to, 'egress')
+ const egressPolicy = this.buildEgressPolicy(
+ egressName,
+ recipeName,
+ fromPodSelector,
+ toPodSelector,
+ fromNs,
+ toNs,
+ binding.port,
+ protocol
+ )
+ const ingressName = this.policyName(recipeName, binding.from, binding.to, 'ingress')
+ const ingressPolicy = this.buildIngressPolicy(
+ ingressName,
+ recipeName,
+ fromPodSelector,
+ toPodSelector,
+ fromNs,
+ toNs,
+ binding.port,
+ protocol
+ )
- if (!isCurrent()) return
- await this.applyPolicy(egressName, egressPolicy, fromNs, isCurrent)
- if (!isCurrent()) return
- await this.applyPolicy(ingressName, ingressPolicy, toNs, isCurrent)
- }
+ // Egress and ingress target different namespaces — safe to parallelize
+ await Promise.all([
+ this.applyPolicy(egressName, egressPolicy, fromNs),
+ this.applyPolicy(ingressName, ingressPolicy, toNs),
+ ])
+ })
+ )
}
/**
* Delete all binding-scoped NetworkPolicies for a recipe.
*/
- async cleanupBindings(
- recipeName: string,
- options: BindingPolicyCleanupOptions = {}
- ): Promise<void> {
+ async cleanupBindings(recipeName: string): Promise<void> {
console.log(`[BindingNP] Cleaning up binding policies for recipe "${recipeName}"`)
- // LIST operations are read-only and namespace-independent. Each resulting
- // delete is separately authorized after its LIST completes.
- const results = await Promise.allSettled([
- this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE, options.deleteAllowed),
- this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace, options.deleteAllowed),
+ // Delete from both namespaces in parallel (independent operations)
+ await Promise.all([
+ this.deletePoliciesForRecipe(recipeName, SANDBOX_NAMESPACE),
+ this.deletePoliciesForRecipe(recipeName, this.mcpServerNamespace),
])
- const failures = results.flatMap(result =>
- result.status === 'rejected' ? [result.reason] : []
- )
- if (failures.length > 0) {
- throw new AggregateError(
- failures,
- `Failed to clean up binding policies for recipe "${recipeName}"`
- )
- }
}
// ─── Policy Builders ────────────────────────────────────────────
@@ -270,49 +242,43 @@ export class BindingPolicyReconciler {
private async applyPolicy(
name: string,
policy: k8s.V1NetworkPolicy,
- namespace: string,
- isCurrent: () => boolean
+ namespace: string
): Promise<void> {
- await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]', isCurrent)
+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[BindingNP]')
}
- private async deletePoliciesForRecipe(
- recipeName: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
- let response: k8s.V1NetworkPolicyList
+ private async deletePoliciesForRecipe(recipeName: string, namespace: string): Promise<void> {
try {
- response = await this.networkingApi.listNamespacedNetworkPolicy({
+ const response = await this.networkingApi.listNamespacedNetworkPolicy({
namespace,
labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=binding-allow,${RECIPE_LABEL}=${recipeName}`,
})
+
+ await Promise.all(
+ (response.items || []).map(async policy => {
+ const policyName = policy.metadata?.name
+ if (!policyName) {
+ console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)
+ return
+ }
+ try {
+ await this.networkingApi.deleteNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404) {
+ console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)
+ }
+ }
+ })
+ )
} catch (error) {
console.error(
`[BindingNP] Failed to list policies for recipe "${recipeName}" in ${namespace}:`,
error
)
- throw error
- }
-
- for (const policy of response.items || []) {
- const policyName = policy.metadata?.name
- if (!policyName) {
- console.warn(`[BindingNP] Skipping policy with no name in ${namespace}`)
- continue
- }
- try {
- if (deleteAllowed && !(await deleteAllowed())) return
- await this.networkingApi.deleteNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- })
- console.log(`[BindingNP] Deleted policy "${policyName}" in ${namespace}`)
- } catch (error: unknown) {
- if (getErrorCode(error) === 404) continue
- console.error(`[BindingNP] Failed to delete policy "${policyName}":`, error)
- throw error
- }
}
}
}
diff --git a/host-context-controller/src/config.externalEgressDnsTimeout.test.ts b/host-context-controller/src/config.externalEgressDnsTimeout.test.ts
deleted file mode 100644
index 7396924b9..000000000
--- a/host-context-controller/src/config.externalEgressDnsTimeout.test.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { afterEach, describe, expect, it, vi } from 'vitest'
-
-const originalEnv = process.env
-
-async function loadConfig(raw: string | undefined) {
- vi.resetModules()
- process.env = { ...originalEnv }
- if (raw === undefined) {
- delete process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS
- } else {
- process.env.HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS = raw
- }
- return import('./config')
-}
-
-afterEach(() => {
- process.env = originalEnv
- vi.resetModules()
-})
-
-describe('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS', () => {
- it.each([undefined, '', ' '])('defaults to 5000 for %s', async raw => {
- const { config } = await loadConfig(raw)
-
- expect(config.externalEgressDnsResolveTimeoutMs).toBe(5_000)
- })
-
- it.each([
- ['1', 1],
- ['5000', 5_000],
- ['2147483647', 2_147_483_647],
- ])('accepts the positive bounded integer %s', async (raw, expected) => {
- const { config } = await loadConfig(raw)
-
- expect(config.externalEgressDnsResolveTimeoutMs).toBe(expected)
- })
-
- it.each(['0', '-1', '1.5', '5000junk', 'NaN', 'Infinity', '2147483648'])(
- "fails configuration validation for '%s'",
- async raw => {
- await expect(loadConfig(raw)).rejects.toThrow(
- 'HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer'
- )
- }
- )
-})
diff --git a/host-context-controller/src/config.ts b/host-context-controller/src/config.ts
index 162ccd06e..7b7632b75 100644
--- a/host-context-controller/src/config.ts
+++ b/host-context-controller/src/config.ts
@@ -197,10 +197,6 @@ export interface Config {
// so accumulated IPs never expire between refreshes (issue #299).
externalEgressResyncIntervalSec: number
- // Deadline for one DNS resolution attempt. A silent resolver cannot block
- // safety convergence indefinitely.
- externalEgressDnsResolveTimeoutMs: number
-
// Grace kept after a DNS TTL before an accumulated /32 may expire (seconds).
// The sliding window that fixes #299: entries live for TTL + overlap, so a
// provider that rotates a single A record still resolves through the union of
@@ -249,17 +245,6 @@ function getEnvInt(key: string, defaultValue: number): number {
return isNaN(parsed) ? defaultValue : parsed
}
-export function parseExternalEgressDnsTimeoutMs(raw: string | undefined): number {
- if (raw === undefined || raw.trim() === '') return 5_000
- const parsed = Number(raw)
- if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > 2_147_483_647) {
- throw new Error(
- `HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS must be a positive integer no greater than 2147483647, got '${raw}'`
- )
- }
- return parsed
-}
-
/**
* Loud integer env parser for the external-egress knobs (audit R1-L1). Unlike
* getEnvInt (parseInt), this rejects non-canonical values instead of silently
@@ -729,9 +714,6 @@ export const config: Config = {
// they expire, closing the #299 rotation gap. 0 disables (reintroduces the
// stale-snapshot risk; only for operators who explicitly opt out).
externalEgressResyncIntervalSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_RESYNC_SEC', 60),
- externalEgressDnsResolveTimeoutMs: parseExternalEgressDnsTimeoutMs(
- getEnv('HCC_EXTERNAL_EGRESS_DNS_TIMEOUT_MS')
- ),
externalEgressOverlapSec: getExternalEgressEnvInt('HCC_EXTERNAL_EGRESS_OVERLAP_SEC', 300),
externalEgressRefreshFloorSec: getExternalEgressEnvInt(
'HCC_EXTERNAL_EGRESS_REFRESH_FLOOR_SEC',
diff --git a/host-context-controller/src/deploymentStrategy.test.ts b/host-context-controller/src/deploymentStrategy.test.ts
deleted file mode 100644
index 7a9258139..000000000
--- a/host-context-controller/src/deploymentStrategy.test.ts
+++ /dev/null
@@ -1,66 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import { loadAll } from 'js-yaml'
-import { readFileSync } from 'node:fs'
-import path from 'node:path'
-
-/**
- * HCC's safety model assumes a single writer: replicas:1 with NO leader
- * election (see statelessLifecycleExecutor.ts — "Running >1 replica risks two
- * controllers racing"). The default Deployment strategy is RollingUpdate with
- * maxSurge>0, which starts the new pod before the old one exits — a guaranteed
- * two-replica overlap on every rollout. During that window the outgoing replica
- * can win a conflict-retry replace (applyNetworkPolicy → replaceWithConflictRetry)
- * and land its wider spec back over a policy the new replica just hardened and
- * certified. There is no NetworkPolicy watch or periodic resync to heal it, and
- * the new replica's safety fence only sees failures in its own process — so
- * /ready stays 200 over a stale allow. Recreate closes the overlap, which is the
- * only way to honour the documented single-writer assumption during a rollout.
- */
-describe('host-context-controller Deployment strategy', () => {
- const docs = loadAll(
- readFileSync(
- path.join(__dirname, '../../deploy/base/control-plane/host-context-controller.yaml'),
- 'utf8'
- )
- ) as Array<{ kind?: string; metadata?: { name?: string }; spec?: Record<string, unknown> }>
-
- const deployment = docs.find(
- d => d?.kind === 'Deployment' && d?.metadata?.name === 'host-context-controller'
- )
-
- it('defines the host-context-controller Deployment', () => {
- expect(deployment).toBeDefined()
- })
-
- it('runs a single replica', () => {
- expect(deployment?.spec?.replicas).toBe(1)
- })
-
- it('uses Recreate so a rollout never overlaps two writers', () => {
- // RollingUpdate (the default) would start the new pod before terminating the
- // old one; with no leader election that races two NetworkPolicy writers.
- const strategy = deployment?.spec?.strategy as { type?: string } | undefined
- expect(strategy?.type).toBe('Recreate')
- })
-
- it('carries no rollingUpdate block under Recreate', () => {
- // A leftover spec.strategy.rollingUpdate is invalid under Recreate and, if a
- // future edit flipped only `type` back, would quietly reintroduce surge.
- const strategy = deployment?.spec?.strategy as { rollingUpdate?: unknown } | undefined
- expect(strategy?.rollingUpdate).toBeUndefined()
- })
-
- it('bounds rollout-stall detection above the worst legitimate startup', () => {
- // Detection only, not recovery: Recreate + replicas:1 means a botched
- // rollout is healed operationally (kubectl rollout undo), never by K8s.
- // This deadline just flips Progressing=False / fails `kubectl rollout
- // status` deterministically for observers without their own --timeout. It
- // must exceed the worst legitimate startup, which — until #205 fully
- // decouples readiness from fleet convergence — scales with the McpServer/
- // Context fleet: origin/dev observed 651s on clerum-dev (108 McpServers /
- // 22 Contexts) on 2026-08-05, so 1200s (900s wait + 300s head-start) is the
- // merged mitigation. Comes back down once readiness no longer waits on the
- // fleet.
- expect(deployment?.spec?.progressDeadlineSeconds).toBe(1200)
- })
-})
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
deleted file mode 100644
index ac8a5d926..000000000
--- a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
+++ /dev/null
@@ -1,968 +0,0 @@
-import { afterEach, describe, expect, it, vi } from 'vitest'
-import {
- ExternalEgressConvergenceCoordinator,
- type ExternalEgressRetryHandle,
- type ExternalEgressWatchEventType,
-} from './externalEgressConvergenceCoordinator'
-import { externalEgressRetriesAtCap } from './metrics'
-import type { McpServerCRD } from './types'
-
-function deferred<T = void>() {
- let resolve!: (value: T | PromiseLike<T>) => void
- let reject!: (reason?: unknown) => void
- const promise = new Promise<T>((resolvePromise, rejectPromise) => {
- resolve = resolvePromise
- reject = rejectPromise
- })
- return { promise, resolve, reject }
-}
-
-function server(
- name: string,
- generation = 1,
- options: { bindings?: boolean; namespace?: string } = {}
-): McpServerCRD {
- return {
- name,
- namespace: options.namespace ?? 'mcp-server',
- uid: `${name}-uid`,
- generation,
- spec: {
- contextRef: 'default',
- image: `clerum/${name}:v${generation}`,
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings:
- options.bindings === false ? undefined : [{ dns: `${name}.example`, port: 443 }],
- },
- }
-}
-
-type CoordinatorOverrides = Partial<
- ConstructorParameters<typeof ExternalEgressConvergenceCoordinator>[0]
->
-
-function coordinator(overrides: CoordinatorOverrides = {}) {
- const servers = new Map<string, McpServerCRD>()
- const mutate = vi.fn(async () => undefined)
- const replay = vi.fn(async () => undefined)
- const instance = new ExternalEgressConvergenceCoordinator({
- listServers: () => [...servers.values()],
- getCurrentServer: name => servers.get(name),
- inventoryAuthoritative: () => true,
- sameDesiredRevision: (left, right) =>
- left.uid === right.uid &&
- left.generation === right.generation &&
- JSON.stringify(left.spec) === JSON.stringify(right.spec),
- enqueue: async (_selected, work) => work(),
- mutate,
- replay,
- externalEgressRefreshMinTtlMs: () => Infinity,
- ...overrides,
- })
- return { instance, mutate, replay, servers }
-}
-
-async function flushMicrotasks(): Promise<void> {
- await Promise.resolve()
- await Promise.resolve()
- await Promise.resolve()
-}
-
-describe('ExternalEgressConvergenceCoordinator', () => {
- afterEach(() => {
- vi.useRealTimers()
- vi.restoreAllMocks()
- })
-
- it.each([
- ['presence', 'DELETED' as const, undefined],
- ['revision', 'MODIFIED' as const, server('redis-tools', 2)],
- ])(
- 'lets an existing timer consume the latest %s intent and resets its attempt ladder',
- async (_change, nextType, nextServer) => {
- vi.useFakeTimers()
- const initial = server('redis-tools', 1)
- const state = new Map<string, McpServerCRD>([[initial.name, initial]])
- const replayed: Array<{ type: ExternalEgressWatchEventType; server: McpServerCRD }> = []
- let instance!: ExternalEgressConvergenceCoordinator
- instance = coordinator({
- getCurrentServer: name => state.get(name),
- replay: async (type, selected) => {
- replayed.push({ type, server: selected })
- instance.scheduleRetry(type, selected)
- },
- }).instance
-
- instance.scheduleRetry('ADDED', initial)
- if (nextServer) state.set(nextServer.name, nextServer)
- else state.delete(initial.name)
- instance.scheduleRetry(nextType, nextServer ?? initial)
-
- await vi.advanceTimersByTimeAsync(5000)
- expect(replayed).toEqual([
- {
- type: nextType,
- server: expect.objectContaining({
- generation: nextServer?.generation ?? initial.generation,
- }),
- },
- ])
-
- await vi.advanceTimersByTimeAsync(4999)
- expect(replayed).toHaveLength(1)
- await vi.advanceTimersByTimeAsync(1)
- expect(replayed).toHaveLength(2)
- instance.stop()
- }
- )
-
- it('continues retrying at the capped delay until the replay succeeds', async () => {
- vi.useFakeTimers()
- const selected = server('web-search')
- let instance!: ExternalEgressConvergenceCoordinator
- let attempts = 0
- instance = coordinator({
- getCurrentServer: () => selected,
- replay: async (type, current, retry) => {
- attempts += 1
- if (attempts < 5) {
- instance.scheduleRetry(type, current)
- return
- }
- retry.complete()
- },
- }).instance
-
- instance.scheduleRetry('ADDED', selected)
- await vi.advanceTimersByTimeAsync(5000)
- await vi.advanceTimersByTimeAsync(15000)
- await vi.advanceTimersByTimeAsync(30000)
- expect(attempts).toBe(3)
- await vi.advanceTimersByTimeAsync(29999)
- expect(attempts).toBe(3)
- await vi.advanceTimersByTimeAsync(1)
- expect(attempts).toBe(4)
- await vi.advanceTimersByTimeAsync(29999)
- expect(attempts).toBe(4)
- await vi.advanceTimersByTimeAsync(1)
-
- expect(attempts).toBe(5)
- await vi.advanceTimersByTimeAsync(30000)
- expect(attempts).toBe(5)
- instance.stop()
- })
-
- it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {
- vi.useFakeTimers()
- const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')
- const selected = server('web-search')
- let instance!: ExternalEgressConvergenceCoordinator
- let succeed = false
- instance = coordinator({
- getCurrentServer: () => selected,
- replay: async (type, current, retry) => {
- if (succeed) {
- retry.complete()
- return
- }
- instance.scheduleRetry(type, current)
- },
- }).instance
-
- instance.scheduleRetry('ADDED', selected)
- // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.
- await vi.advanceTimersByTimeAsync(5000)
- expect(setSpy).not.toHaveBeenCalled()
-
- // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.
- await vi.advanceTimersByTimeAsync(15000)
- expect(setSpy).toHaveBeenLastCalledWith(1)
-
- // Once the replay finally succeeds, the capped server is cleared back to 0.
- succeed = true
- await vi.advanceTimersByTimeAsync(30000)
- expect(setSpy).toHaveBeenLastCalledWith(0)
-
- instance.stop()
- setSpy.mockRestore()
- })
-
- it('never clears a pending full replay after an egress-only success', async () => {
- vi.useFakeTimers()
- const selected = server('web-search')
- const { instance, replay, servers } = coordinator()
- servers.set(selected.name, selected)
-
- instance.scheduleRetry('ADDED', selected)
- const retry = await instance.reconcile('MODIFIED', selected)
- expect(retry).toBeDefined()
-
- await vi.advanceTimersByTimeAsync(5000)
- expect(replay).toHaveBeenCalledOnce()
- instance.stop()
- })
-
- it('clears a retry only after its complete full-replay handle finishes', async () => {
- vi.useFakeTimers()
- const selected = server('web-search')
- let instance!: ExternalEgressConvergenceCoordinator
- const replays: ExternalEgressRetryHandle[] = []
- instance = coordinator({
- getCurrentServer: () => selected,
- replay: async (type, current, retry) => {
- replays.push(retry)
- const completion = await instance.reconcile(type, current, { retry })
- expect(completion).toBe(retry)
- retry.complete()
- },
- }).instance
-
- instance.scheduleRetry('ADDED', selected)
- await vi.advanceTimersByTimeAsync(5000)
- await vi.advanceTimersByTimeAsync(30000)
-
- expect(replays).toHaveLength(1)
- instance.stop()
- })
-
- it('prevents a stale retry handle from clearing a newer intent', async () => {
- vi.useFakeTimers()
- const first = server('web-search', 1)
- const second = server('web-search', 2)
- const state = new Map<string, McpServerCRD>([[first.name, first]])
- const handles: ExternalEgressRetryHandle[] = []
- const replayed: McpServerCRD[] = []
- const firstReplayStarted = deferred()
- const releaseFirstReplay = deferred()
- const { instance } = coordinator({
- getCurrentServer: name => state.get(name),
- replay: async (_type, current, retry) => {
- replayed.push(current)
- handles.push(retry)
- if (replayed.length === 1) {
- firstReplayStarted.resolve()
- await releaseFirstReplay.promise
- }
- },
- })
-
- instance.scheduleRetry('ADDED', first)
- await vi.advanceTimersByTimeAsync(5000)
- await firstReplayStarted.promise
- expect(handles).toHaveLength(1)
-
- state.set(second.name, second)
- instance.scheduleRetry('MODIFIED', second)
- handles[0].complete()
- releaseFirstReplay.resolve()
- await flushMicrotasks()
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(replayed).toEqual([
- expect.objectContaining({ generation: 1 }),
- expect.objectContaining({ generation: 2 }),
- ])
- instance.stop()
- })
-
- it('keeps retry, resync, and startup concurrency lanes independent', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const retryRelease = deferred()
- const activeRetries = deferred()
- let retryCount = 0
- const resyncServer = server('resync-server')
- const startupServer = server('startup-server')
- const canonical = new Map<string, McpServerCRD>([
- [resyncServer.name, resyncServer],
- [startupServer.name, startupServer],
- ])
- const mutations: string[] = []
- const { instance } = coordinator({
- listServers: () => [resyncServer],
- getCurrentServer: name => canonical.get(name),
- mutate: async (_type, selected) => {
- mutations.push(selected.name)
- },
- replay: async () => {
- retryCount += 1
- if (retryCount === 10) activeRetries.resolve()
- await retryRelease.promise
- },
- })
-
- for (let index = 0; index < 10; index += 1) {
- const selected = server(`retry-${index}`, 1, { bindings: false })
- instance.scheduleRetry('ADDED', selected)
- }
- await vi.advanceTimersByTimeAsync(5000)
- await activeRetries.promise
-
- const resync = instance.runResync()
- const startup = instance.prepareStartupGates([startupServer])
- await Promise.all([resync, startup.waitFor(startupServer)])
-
- expect(mutations).toEqual(expect.arrayContaining(['resync-server', 'startup-server']))
- retryRelease.resolve()
- instance.stop()
- })
-
- it('queues at most one latest-wins retry job per key behind a saturated limiter', async () => {
- vi.useFakeTimers()
- const blockersRelease = deferred()
- const tenBlockersStarted = deferred()
- const finalReplayStarted = deferred()
- const initial = server('churning-server', 1)
- const second = server('churning-server', 2)
- const final = server('churning-server', 3)
- const state = new Map<string, McpServerCRD>([[initial.name, initial]])
- const targetEnqueues: McpServerCRD[] = []
- const targetReplays: McpServerCRD[] = []
- let blockerCount = 0
- const { instance } = coordinator({
- getCurrentServer: name => state.get(name),
- enqueue: async (selected, work) => {
- if (selected.name === initial.name) targetEnqueues.push(selected)
- await work()
- },
- replay: async (_type, selected, retry) => {
- if (selected.name.startsWith('limiter-blocker-')) {
- blockerCount += 1
- if (blockerCount === 10) tenBlockersStarted.resolve()
- await blockersRelease.promise
- return
- }
- targetReplays.push(selected)
- retry.complete()
- finalReplayStarted.resolve()
- },
- })
-
- for (let index = 0; index < 10; index += 1) {
- instance.scheduleRetry('DELETED', server(`limiter-blocker-${index}`))
- }
- await vi.advanceTimersByTimeAsync(5000)
- await tenBlockersStarted.promise
-
- instance.scheduleRetry('ADDED', initial)
- await vi.advanceTimersByTimeAsync(5000)
- state.set(second.name, second)
- instance.scheduleRetry('MODIFIED', second)
- await vi.advanceTimersByTimeAsync(5000)
- state.set(final.name, final)
- instance.scheduleRetry('MODIFIED', final)
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(targetEnqueues).toHaveLength(0)
- blockersRelease.resolve()
- await finalReplayStarted.promise
- await flushMicrotasks()
- await flushMicrotasks()
-
- expect(targetEnqueues).toHaveLength(1)
- expect(targetReplays).toEqual([expect.objectContaining({ generation: 3 })])
- instance.stop()
- })
-
- it('re-arms a retry scheduled by an active replay only after that key leaves its slot', async () => {
- vi.useFakeTimers()
- const selected = server('slow-retry')
- const firstReplayStarted = deferred()
- const releaseFirstReplay = deferred()
- let instance!: ExternalEgressConvergenceCoordinator
- let replayCount = 0
- instance = coordinator({
- getCurrentServer: () => selected,
- replay: async (type, current, retry) => {
- replayCount += 1
- if (replayCount === 1) {
- instance.scheduleRetry(type, current)
- firstReplayStarted.resolve()
- await releaseFirstReplay.promise
- return
- }
- retry.complete()
- },
- }).instance
-
- instance.scheduleRetry('ADDED', selected)
- await vi.advanceTimersByTimeAsync(5000)
- await firstReplayStarted.promise
-
- await vi.advanceTimersByTimeAsync(15000)
- expect(replayCount).toBe(1)
-
- releaseFirstReplay.resolve()
- await flushMicrotasks()
- await vi.advanceTimersByTimeAsync(14999)
- expect(replayCount).toBe(1)
- await vi.advanceTimersByTimeAsync(1)
- expect(replayCount).toBe(2)
- instance.stop()
- })
-
- it('cancels a retry waiter on stop before limiter capacity becomes available', async () => {
- vi.useFakeTimers()
- const blockersRelease = deferred()
- const tenBlockersStarted = deferred()
- const queued = server('queued-before-stop')
- let blockerCount = 0
- const enqueuedTargets: string[] = []
- const { instance } = coordinator({
- enqueue: async (selected, work) => {
- if (selected.name === queued.name) enqueuedTargets.push(selected.name)
- await work()
- },
- replay: async (_type, selected) => {
- if (!selected.name.startsWith('stop-blocker-')) return
- blockerCount += 1
- if (blockerCount === 10) tenBlockersStarted.resolve()
- await blockersRelease.promise
- },
- })
-
- for (let index = 0; index < 10; index += 1) {
- instance.scheduleRetry('DELETED', server(`stop-blocker-${index}`))
- }
- await vi.advanceTimersByTimeAsync(5000)
- await tenBlockersStarted.promise
- instance.scheduleRetry('DELETED', queued)
- await vi.advanceTimersByTimeAsync(5000)
-
- instance.stop()
- blockersRelease.resolve()
- await flushMicrotasks()
- await flushMicrotasks()
-
- expect(enqueuedTargets).toEqual([])
- })
-
- it('resolves the current canonical server after periodic jitter', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0.5)
- const stale = server('web-search', 1)
- const current = server('web-search', 2)
- const state = new Map<string, McpServerCRD>([[stale.name, stale]])
- const reconciled: McpServerCRD[] = []
- const { instance } = coordinator({
- listServers: () => [stale],
- getCurrentServer: name => state.get(name),
- mutate: async (_type, selected) => {
- reconciled.push(selected)
- },
- })
-
- const resync = instance.runResync()
- await flushMicrotasks()
- state.set(current.name, current)
- await vi.advanceTimersByTimeAsync(2500)
- await resync
-
- expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])
- instance.stop()
- })
-
- it('returns one global single-flight promise for overlapping resync requests', async () => {
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const selected = server('web-search')
- const release = deferred()
- const mutationStarted = deferred()
- const { instance, servers } = coordinator({
- mutate: async () => {
- mutationStarted.resolve()
- await release.promise
- },
- })
- servers.set(selected.name, selected)
-
- const first = instance.runResync()
- const overlapping = instance.runResync()
- expect(overlapping).toBe(first)
- await mutationStarted.promise
-
- release.resolve()
- await first
- instance.stop()
- })
-
- it('skips a resync key whose external-egress mutation is already in flight', async () => {
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const selected = server('web-search')
- const release = deferred()
- const mutationStarted = deferred()
- const mutate = vi.fn(async () => {
- mutationStarted.resolve()
- await release.promise
- })
- const { instance, servers } = coordinator({ mutate })
- servers.set(selected.name, selected)
-
- const active = instance.reconcile('MODIFIED', selected)
- await mutationStarted.promise
- await instance.runResync()
-
- expect(mutate).toHaveBeenCalledOnce()
- release.resolve()
- await active
- instance.stop()
- })
-
- it('resolves canonical state after entering the keyed resync queue', async () => {
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const stale = server('web-search', 1)
- const current = server('web-search', 2)
- const state = new Map<string, McpServerCRD>([[stale.name, stale]])
- const queueEntered = deferred()
- const releaseQueue = deferred()
- const reconciled: McpServerCRD[] = []
- const { instance } = coordinator({
- listServers: () => [stale],
- getCurrentServer: name => state.get(name),
- enqueue: async (_selected, work) => {
- queueEntered.resolve()
- await releaseQueue.promise
- await work()
- },
- mutate: async (_type, selected) => {
- reconciled.push(selected)
- },
- })
-
- const resync = instance.runResync()
- await queueEntered.promise
- state.set(current.name, current)
- releaseQueue.resolve()
- await resync
-
- expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])
- instance.stop()
- })
-
- it('keeps a startup gate false when desired revision changes during egress', async () => {
- const selected = server('web-search', 1)
- const current = server('web-search', 2)
- const state = new Map<string, McpServerCRD>([[selected.name, selected]])
- const mutationStarted = deferred()
- const releaseMutation = deferred()
- let selectedLease: (() => boolean) | undefined
- const { instance } = coordinator({
- getCurrentServer: name => state.get(name),
- mutate: async (_type, _server, options) => {
- selectedLease = options.isCurrent
- mutationStarted.resolve()
- await releaseMutation.promise
- },
- })
-
- const gates = instance.prepareStartupGates([selected])
- await mutationStarted.promise
- expect(selectedLease?.()).toBe(true)
- state.set(current.name, current)
- expect(selectedLease?.()).toBe(false)
- releaseMutation.resolve()
-
- await expect(gates.waitFor(selected)).resolves.toBe(false)
- instance.stop()
- })
-
- it('keeps a failed startup DNS gate isolated and recovers it through the dedicated retry', async () => {
- vi.useFakeTimers()
- const selected = server('dns-held-server')
- const dnsFailure = new Error('dns resolution failed')
- const mutate = vi.fn().mockRejectedValueOnce(dnsFailure).mockResolvedValue(undefined)
- const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
- let instance!: ExternalEgressConvergenceCoordinator
- const built = coordinator({
- mutate,
- replay: async (type, current, retry) => {
- const completion = await instance.reconcile(type, current, { retry })
- completion?.complete()
- },
- })
- instance = built.instance
- built.servers.set(selected.name, selected)
-
- const gates = instance.prepareStartupGates([selected])
- await expect(gates.waitFor(selected)).resolves.toBe(false)
- expect(mutate).toHaveBeenCalledOnce()
- expect(error).toHaveBeenCalledWith(
- `[K8s] Initial external egress reconciliation failed for ${selected.namespace}/${selected.name}; ` +
- 'runtime reconciliation will stay blocked until retry succeeds:',
- dnsFailure
- )
-
- await vi.advanceTimersByTimeAsync(5000)
- expect(mutate).toHaveBeenCalledTimes(2)
- await vi.advanceTimersByTimeAsync(30000)
- expect(mutate).toHaveBeenCalledTimes(2)
- instance.stop()
- })
-
- it('retires a direct mutation lease when desired revision changes in place', async () => {
- const selected = server('web-search', 1)
- const replacement = server('web-search', 2)
- const mutationStarted = deferred()
- const releaseMutation = deferred()
- let selectedLease: (() => boolean) | undefined
- const { instance, servers } = coordinator({
- mutate: async (_type, _server, options) => {
- selectedLease = options.isCurrent
- mutationStarted.resolve()
- await releaseMutation.promise
- },
- })
- servers.set(selected.name, selected)
-
- const mutation = instance.reconcile('MODIFIED', selected)
- await mutationStarted.promise
- expect(selectedLease?.()).toBe(true)
-
- servers.set(replacement.name, replacement)
- expect(selectedLease?.()).toBe(false)
- releaseMutation.resolve()
- await mutation
- instance.stop()
- })
-
- it('does not start a second mutation after stop releases an in-flight wait', async () => {
- const firstMutation = deferred()
- const selected = server('web-search')
- const mutate = vi.fn(async () => firstMutation.promise)
- const { instance, servers } = coordinator({ mutate })
- servers.set(selected.name, selected)
-
- const first = instance.reconcile('MODIFIED', selected)
- await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())
- const waiting = instance.reconcile('MODIFIED', selected)
- instance.stop()
- firstMutation.resolve()
- await Promise.all([first, waiting])
-
- expect(mutate).toHaveBeenCalledOnce()
- })
-
- it('rechecks per-key in-flight ownership when multiple waiters resume together', async () => {
- const selected = server('web-search')
- const releases = [deferred(), deferred(), deferred()]
- const starts = [deferred(), deferred(), deferred()]
- let active = 0
- let maxActive = 0
- let call = 0
- const mutate = vi.fn(async () => {
- const index = call++
- active += 1
- maxActive = Math.max(maxActive, active)
- starts[index].resolve()
- await releases[index].promise
- active -= 1
- })
- const { instance, servers } = coordinator({ mutate })
- servers.set(selected.name, selected)
-
- const first = instance.reconcile('MODIFIED', selected)
- await starts[0].promise
- const second = instance.reconcile('MODIFIED', selected)
- const third = instance.reconcile('MODIFIED', selected)
-
- try {
- releases[0].resolve()
- await starts[1].promise
- await flushMicrotasks()
- expect(maxActive).toBe(1)
- } finally {
- for (const release of releases) release.resolve()
- await Promise.all([first, second, third])
- instance.stop()
- }
- })
-
- it('rechecks a caller lease after waiting for an in-flight mutation', async () => {
- const firstMutation = deferred()
- const selected = server('web-search')
- let current = true
- const mutate = vi.fn(async () => firstMutation.promise)
- const { instance, servers } = coordinator({ mutate })
- servers.set(selected.name, selected)
-
- const first = instance.reconcile('MODIFIED', selected)
- await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())
- const waiting = instance.reconcile('MODIFIED', selected, {
- isCurrent: () => current,
- })
- current = false
- firstMutation.resolve()
- await Promise.all([first, waiting])
-
- expect(mutate).toHaveBeenCalledOnce()
- instance.stop()
- })
-
- it('keeps a retry pending while inventory authority is unavailable', async () => {
- vi.useFakeTimers()
- const selected = server('web-search')
- let authoritative = false
- const { instance, replay, servers } = coordinator({
- inventoryAuthoritative: () => authoritative,
- })
- servers.set(selected.name, selected)
-
- instance.scheduleRetry('ADDED', selected)
- await vi.advanceTimersByTimeAsync(5000)
- expect(replay).not.toHaveBeenCalled()
-
- authoritative = true
- await vi.advanceTimersByTimeAsync(14_999)
- expect(replay).not.toHaveBeenCalled()
- await vi.advanceTimersByTimeAsync(1)
- expect(replay).toHaveBeenCalledOnce()
- instance.stop()
- })
-
- it('skips periodic resync while inventory authority is unavailable', async () => {
- const selected = server('web-search')
- let authoritative = false
- const { instance, mutate, servers } = coordinator({
- inventoryAuthoritative: () => authoritative,
- })
- servers.set(selected.name, selected)
-
- await instance.runResync()
- expect(mutate).not.toHaveBeenCalled()
-
- authoritative = true
- await instance.runResync()
- expect(mutate).toHaveBeenCalledOnce()
- instance.stop()
- })
-
- it('does not resume periodic work from jitter after stop', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0.5)
- const selected = server('web-search')
- const { instance, mutate, servers } = coordinator()
- servers.set(selected.name, selected)
-
- const resync = instance.runResync()
- await flushMicrotasks()
- instance.stop()
- await vi.advanceTimersByTimeAsync(2500)
- await resync
-
- expect(mutate).not.toHaveBeenCalled()
- })
-
- it('does not start an eleventh startup mutation after stop releases the limiter', async () => {
- const release = deferred()
- const tenStarted = deferred()
- let active = 0
- const selected = Array.from({ length: 11 }, (_, index) => server(`startup-${index}`))
- const state = new Map(selected.map(item => [item.name, item]))
- const mutate = vi.fn(async () => {
- active += 1
- if (active === 10) tenStarted.resolve()
- await release.promise
- })
- const { instance } = coordinator({
- getCurrentServer: name => state.get(name),
- mutate,
- })
-
- const gates = instance.prepareStartupGates(selected)
- await tenStarted.promise
- instance.stop()
- release.resolve()
- await Promise.all(selected.map(item => gates.waitFor(item)))
-
- expect(mutate).toHaveBeenCalledTimes(10)
- })
-
- it('passes DELETE authority through to the watcher mutation without evaluating it', async () => {
- const selected = server('web-search')
- const deleteAllowed = vi.fn(async () => false)
- const mutate = vi.fn(async () => undefined)
- const { instance } = coordinator({ mutate })
-
- await instance.reconcile('DELETED', selected, { deleteAllowed })
-
- expect(mutate).toHaveBeenCalledWith('DELETED', selected, {
- deleteAllowed,
- isCurrent: expect.any(Function),
- })
- expect(deleteAllowed).not.toHaveBeenCalled()
- instance.stop()
- })
-
- it('owns periodic resync scheduling and stops it immediately', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const selected = server('web-search')
- const { instance, mutate, servers } = coordinator()
- servers.set(selected.name, selected)
-
- instance.startPeriodicResync(1, 1)
- await vi.advanceTimersByTimeAsync(1000)
- expect(mutate).toHaveBeenCalledOnce()
-
- instance.stop()
- await vi.advanceTimersByTimeAsync(5000)
- expect(mutate).toHaveBeenCalledOnce()
- })
-
- it('warns once when periodic resync is disabled (intervalSec <= 0) and arms no timer', async () => {
- vi.useFakeTimers()
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
- const selected = server('web-search')
- const { instance, mutate, servers } = coordinator()
- servers.set(selected.name, selected)
-
- instance.startPeriodicResync(0, 5)
- // No timer armed: advancing time never triggers a resync pass.
- await vi.advanceTimersByTimeAsync(60_000)
- expect(mutate).not.toHaveBeenCalled()
-
- expect(warnSpy).toHaveBeenCalledTimes(1)
- expect(warnSpy).toHaveBeenCalledWith(
- expect.stringContaining('External egress periodic DNS resync disabled')
- )
-
- // Called again while still disabled: no duplicate warning (once only).
- instance.startPeriodicResync(-1, 5)
- expect(warnSpy).toHaveBeenCalledTimes(1)
-
- instance.stop()
- })
-
- it('does not warn about a disabled resync when it is enabled (intervalSec > 0)', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
- const selected = server('web-search')
- const { instance, servers } = coordinator()
- servers.set(selected.name, selected)
-
- instance.startPeriodicResync(1, 1)
- expect(warnSpy).not.toHaveBeenCalledWith(
- expect.stringContaining('External egress periodic DNS resync disabled')
- )
-
- instance.stop()
- })
-
- it('arms one resync at a time, rescheduling only after each pass completes (H2)', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const selected = server('web-search')
- const release: Array<() => void> = []
- const mutate = vi.fn(
- () =>
- new Promise<void>(resolve => {
- release.push(resolve)
- })
- )
- const { instance, servers } = coordinator({ mutate })
- servers.set(selected.name, selected)
-
- instance.startPeriodicResync(10, 5)
- // First pass fires at the configured interval (10s) and then blocks.
- await vi.advanceTimersByTimeAsync(10_000)
- expect(mutate).toHaveBeenCalledOnce()
-
- // Complete the pass off the fixed grid (at t=17s). The successor is armed
- // relative to completion (t=27s), NOT on a fixed 20s/30s interval grid.
- await vi.advanceTimersByTimeAsync(7_000)
- release[0]()
- await flushMicrotasks()
-
- // A fixed setInterval would fire a second pass at t=20s; the self-
- // rescheduling timer must not, because it only re-armed at completion.
- await vi.advanceTimersByTimeAsync(3_000)
- expect(mutate).toHaveBeenCalledOnce()
-
- // The successor fires 10s after completion (t=27s).
- await vi.advanceTimersByTimeAsync(7_000)
- expect(mutate).toHaveBeenCalledTimes(2)
-
- instance.stop()
- release.forEach(resolve => resolve())
- })
-
- it('advances the next periodic resync to <= observed TTL/2 (H2, issue #299)', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const selected = server('web-search')
- // The mutation observes a low DNS TTL, ratcheting the coordinator's source
- // down; scheduleNext reads it AFTER the pass and must advance to <= TTL/2.
- let minObservedTtlMs = Infinity
- const mutate = vi.fn(async () => {
- minObservedTtlMs = 20_000
- })
- const { instance, servers } = coordinator({
- mutate,
- externalEgressRefreshMinTtlMs: () => minObservedTtlMs,
- })
- servers.set(selected.name, selected)
-
- // Interval 60s, floor 5s. No TTL observed yet, so the first pass fires at 60s.
- instance.startPeriodicResync(60, 5)
- await vi.advanceTimersByTimeAsync(59_999)
- expect(mutate).not.toHaveBeenCalled()
- await vi.advanceTimersByTimeAsync(1)
- expect(mutate).toHaveBeenCalledOnce()
-
- // TTL/2 = 10s now governs the next delay, far below the 60s backstop.
- await vi.advanceTimersByTimeAsync(9_999)
- expect(mutate).toHaveBeenCalledOnce()
- await vi.advanceTimersByTimeAsync(1)
- expect(mutate).toHaveBeenCalledTimes(2)
-
- instance.stop()
- })
-
- it('logs and absorbs a rejected periodic resync pass, then re-arms (no unhandledRejection)', async () => {
- vi.useFakeTimers()
- vi.spyOn(Math, 'random').mockReturnValue(0)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
- // Capture any process-level unhandledRejection: the pre-fix code left the
- // rejected `void runResync()` promise unhandled, which crashes modern Node.
- const unhandled: unknown[] = []
- const onUnhandled = (reason: unknown): void => {
- unhandled.push(reason)
- }
- process.on('unhandledRejection', onUnhandled)
-
- const selected = server('web-search')
- // Reject OUTSIDE the per-server try/catch in runResyncCore (that catch only
- // wraps the mutation). A rejecting `enqueue` propagates through the worker to
- // Promise.all -> runResyncCore -> runResync, so the whole pass rejects.
- const enqueue = vi.fn(async () => {
- throw new Error('resync enqueue boom')
- })
- const { instance, servers } = coordinator({ enqueue })
- servers.set(selected.name, selected)
-
- try {
- instance.startPeriodicResync(10, 5)
- // First pass fires at 10s and rejects.
- await vi.advanceTimersByTimeAsync(10_000)
- await flushMicrotasks()
-
- // The rejection is logged by the fix's .catch (RED without it: no such log).
- expect(errorSpy).toHaveBeenCalledWith(
- '[K8s] External egress periodic resync pass failed:',
- expect.any(Error)
- )
- expect(enqueue).toHaveBeenCalledTimes(1)
-
- // The loop RE-ARMS despite the failure: the successor fires 10s later.
- await vi.advanceTimersByTimeAsync(10_000)
- await flushMicrotasks()
- expect(enqueue).toHaveBeenCalledTimes(2)
-
- // The fix handles the rejection, so nothing escaped as unhandled.
- expect(unhandled).toEqual([])
- } finally {
- process.off('unhandledRejection', onUnhandled)
- instance.stop()
- }
- })
-})
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.ts
deleted file mode 100644
index 6a8a19db2..000000000
--- a/host-context-controller/src/externalEgressConvergenceCoordinator.ts
+++ /dev/null
@@ -1,607 +0,0 @@
-import { externalEgressRetriesAtCap } from './metrics'
-import type { McpServerCRD } from './types'
-
-const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const
-const EXTERNAL_EGRESS_MAX_CONCURRENCY = 10
-const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000
-
-export type ExternalEgressWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
-
-export type ExternalEgressRetryHandle = {
- isCurrent: () => boolean
- complete: () => void
-}
-
-type ExternalEgressRetryIntent = {
- sequence: number
- type: ExternalEgressWatchEventType
- server: McpServerCRD
-}
-
-type ExternalEgressMutationOptions = {
- deleteAllowed?: () => Promise<boolean>
- isCurrent: () => boolean
-}
-
-type ExternalEgressReconcileOptions = {
- deleteAllowed?: () => Promise<boolean>
- retry?: ExternalEgressRetryHandle
- isCurrent?: () => boolean
-}
-
-type ExternalEgressCoordinatorDependencies = {
- listServers: () => McpServerCRD[]
- getCurrentServer: (name: string) => McpServerCRD | undefined
- inventoryAuthoritative: () => boolean
- sameDesiredRevision: (left: McpServerCRD, right: McpServerCRD) => boolean
- enqueue: (server: McpServerCRD, work: () => Promise<void>) => Promise<void>
- mutate: (
- type: ExternalEgressWatchEventType,
- server: McpServerCRD,
- options: ExternalEgressMutationOptions
- ) => Promise<void>
- replay: (
- type: ExternalEgressWatchEventType,
- server: McpServerCRD,
- retry: ExternalEgressRetryHandle
- ) => Promise<void>
- // Smallest DNS TTL (ms) observed across external-egress resolutions, or
- // Infinity if none. startPeriodicResync reads it to advance the next resync to
- // <= TTL/2 (H2, issue #299).
- externalEgressRefreshMinTtlMs: () => number
-}
-
-export type ExternalEgressStartupGates = {
- waitFor: (server: Pick<McpServerCRD, 'name' | 'namespace'>) => Promise<boolean>
-}
-
-class AsyncConcurrencyLimiter {
- private active = 0
- private readonly waiters: Array<(admitted: boolean) => void> = []
- private closed = false
-
- constructor(private readonly limit: number) {}
-
- async run<T>(work: () => Promise<T>): Promise<T | undefined> {
- if (this.closed) return undefined
- if (this.active >= this.limit) {
- const admitted = await new Promise<boolean>(resolve => this.waiters.push(resolve))
- if (!admitted) return undefined
- } else {
- this.active += 1
- }
- try {
- return await work()
- } finally {
- this.release()
- }
- }
-
- cancelPending(): void {
- if (this.closed) return
- this.closed = true
- for (const resume of this.waiters.splice(0)) {
- resume(false)
- }
- }
-
- private release(): void {
- this.active -= 1
- if (this.closed) return
- const resume = this.waiters.shift()
- if (!resume) return
- // Transfer the released permit before waking the waiter. Reserving it here
- // prevents a new caller from entering between wake-up and its microtask.
- this.active += 1
- resume(true)
- }
-}
-
-function delay(ms: number): Promise<void> {
- return new Promise(resolve => setTimeout(resolve, ms))
-}
-
-/**
- * External-egress resync delay (ms), H2 (issue #299). The configured interval is
- * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a
- * rotating low-TTL host is sampled every rotation. The floor is the hard lower
- * bound to avoid a hot-loop. Pure and unit-testable; consumed by
- * {@link ExternalEgressConvergenceCoordinator.startPeriodicResync}.
- */
-export function externalEgressResyncDelayMs(
- intervalSec: number,
- floorSec: number,
- minObservedTtlMs = Infinity
-): number {
- const floorMs = floorSec * 1000
- const configuredMs = intervalSec * 1000
- const ttlAwareMs = Number.isFinite(minObservedTtlMs)
- ? Math.min(configuredMs, minObservedTtlMs / 2)
- : configuredMs
- return Math.max(ttlAwareMs, floorMs)
-}
-
-/**
- * Owns temporal coordination for external-egress convergence. Kubernetes
- * authority and mutations remain behind callbacks supplied by the watcher.
- */
-export class ExternalEgressConvergenceCoordinator {
- private readonly retryTimers = new Map<string, ReturnType<typeof setTimeout>>()
- private readonly retryAttempts = new Map<string, number>()
- private readonly retryIntents = new Map<string, ExternalEgressRetryIntent>()
- private retrySequence = 0
- private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
- private readonly retrySlots = new Map<string, object>()
- // Keys whose retry backoff is currently pinned at the capped (maximum) delay.
- // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a
- // binding stuck retrying forever is observable, not silently denied.
- private readonly retriesAtCap = new Set<string>()
- private readonly inFlight = new Map<string, Promise<void>>()
- private resyncInFlight: Promise<void> | null = null
- private resyncTimer: ReturnType<typeof setTimeout> | null = null
- private resyncDisabledWarned = false
- private stopped = false
-
- constructor(private readonly dependencies: ExternalEgressCoordinatorDependencies) {}
-
- startPeriodicResync(intervalSec: number, floorSec: number): void {
- if (this.stopped || this.resyncTimer) return
- if (intervalSec <= 0) {
- // Disabled. Warn once (mirrors the Host/SFS/GFS resync lanes) so the
- // operational consequence is visible: without periodic resync the
- // accumulated external-egress DNS allows are never refreshed, so a
- // binding whose upstream DNS rotates (issue #299) can drift until an
- // unrelated McpServer event triggers reconciliation.
- if (!this.resyncDisabledWarned) {
- this.resyncDisabledWarned = true
- console.warn(
- '[K8s] External egress periodic DNS resync disabled; accumulated DNS allows will not ' +
- 'be refreshed, so a binding whose upstream DNS rotates (issue #299) can drift until ' +
- 'another McpServer event triggers reconciliation.'
- )
- }
- return
- }
- // Self-rescheduling (not a fixed setInterval) so the delay can advance to
- // <= observed TTL/2 (H2, issue #299), and rescheduling AFTER each pass
- // completes guarantees no overlapping resyncs. runResync still dedupes any
- // in-flight pass; the floor is the hard lower bound against a hot-loop.
- const scheduleNext = (): void => {
- if (this.stopped) return
- const delayMs = externalEgressResyncDelayMs(
- intervalSec,
- floorSec,
- this.dependencies.externalEgressRefreshMinTtlMs()
- )
- this.resyncTimer = setTimeout(() => {
- // Re-arm FIRST (.finally) so a rejected pass still schedules its
- // successor — the "re-arms after completion" property must survive a
- // failure. Then .catch logs and absorbs the rejection so a rejecting
- // pass (or a throwing scheduleNext) never becomes an unhandledRejection,
- // which crashes the process on modern Node and would silently kill the
- // periodic resync loop (reopening the #299 DNS-drift window).
- void this.runResync()
- .finally(scheduleNext)
- .catch(error =>
- console.error('[K8s] External egress periodic resync pass failed:', error)
- )
- }, delayMs)
- this.resyncTimer.unref?.()
- }
- scheduleNext()
- console.log(
- `[K8s] External egress periodic DNS resync enabled (every ${intervalSec}s, advancing to <= TTL/2)`
- )
- }
-
- runResync(): Promise<void> {
- if (this.resyncInFlight) return this.resyncInFlight
- const run = this.runResyncCore()
- this.resyncInFlight = run
- const settle = () => {
- if (this.resyncInFlight === run) this.resyncInFlight = null
- }
- void run.then(settle, settle)
- return run
- }
-
- prepareStartupGates(
- servers: McpServerCRD[],
- isCurrent: () => boolean = this.dependencies.inventoryAuthoritative
- ): ExternalEgressStartupGates {
- const gates = new Map<string, Promise<boolean>>()
- const selectedServers = servers
- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
- .sort((left, right) => this.key(left).localeCompare(this.key(right)))
-
- if (selectedServers.length > 0) {
- console.log(
- `[K8s] Reconciling external egress before startup runtime reconciliation for ` +
- `${selectedServers.length} McpServer(s)`
- )
- }
-
- // Startup admission is intentionally independent from retry and periodic
- // resync capacity, so a saturated recovery lane cannot head-of-line block
- // initial runtime gates for unrelated servers.
- const limiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
- for (const selected of selectedServers) {
- const key = this.key(selected)
- const gate = limiter
- .run(async () => {
- if (this.stopped || !isCurrent()) return false
- let ready = false
- await this.dependencies.enqueue(selected, async () => {
- if (this.stopped || !isCurrent()) return
- const current = this.dependencies.getCurrentServer(selected.name)
- if (
- !current ||
- current.namespace !== selected.namespace ||
- !this.dependencies.sameDesiredRevision(selected, current)
- ) {
- return
- }
- try {
- // Startup is an egress-only gate. Any pre-existing retry intent is
- // completed only by its full replay, never by this mutation.
- await this.reconcile('MODIFIED', current, { isCurrent })
- const latest = this.dependencies.getCurrentServer(selected.name)
- ready =
- !this.stopped &&
- isCurrent() &&
- latest !== undefined &&
- latest.namespace === selected.namespace &&
- this.dependencies.sameDesiredRevision(selected, latest)
- } catch (error) {
- console.error(
- `[K8s] Initial external egress reconciliation failed for ${key}; ` +
- 'runtime reconciliation will stay blocked until retry succeeds:',
- error
- )
- this.scheduleRetry('MODIFIED', current)
- }
- })
- return ready
- })
- .then(result => result ?? false)
- // Side-attach a logger so a rejection is never *unhandled* (which crashes
- // the process on modern Node) if fullReconcile aborts before waitFor
- // awaits this gate. This does NOT swallow the failure: gates.set stores
- // the original `gate`, so the awaited path in fullReconcile still rejects
- // and fails the pass loudly — this only covers the unawaited case.
- gate.catch(error => {
- console.error(
- `[K8s] External egress startup gate for ${key} rejected before it was awaited:`,
- error
- )
- })
- gates.set(key, gate)
- }
-
- return {
- waitFor: server => gates.get(this.key(server)) ?? Promise.resolve(true),
- }
- }
-
- async reconcile(
- type: ExternalEgressWatchEventType,
- server: McpServerCRD,
- options: ExternalEgressReconcileOptions = {}
- ): Promise<ExternalEgressRetryHandle | undefined> {
- if (!this.reconcileIsCurrent(type, server, options)) return undefined
- const key = this.key(server)
- let existing = this.inFlight.get(key)
- while (existing) {
- console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)
- await existing
- if (!this.reconcileIsCurrent(type, server, options)) return undefined
- existing = this.inFlight.get(key)
- }
-
- if (!this.reconcileIsCurrent(type, server, options)) return undefined
- const completion = options.retry ?? this.currentRetryHandle(key)
- const run = this.dependencies.mutate(type, server, {
- deleteAllowed: options.deleteAllowed,
- isCurrent: () => this.reconcileIsCurrent(type, server, options),
- })
- this.inFlight.set(key, run)
- try {
- await run
- return completion
- } finally {
- if (this.inFlight.get(key) === run) {
- this.inFlight.delete(key)
- }
- }
- }
-
- scheduleRetry(type: string, server: McpServerCRD): void {
- if (!this.isWatchEventType(type) || this.stopped) return
-
- const cached = this.dependencies.getCurrentServer(server.name)
- const retryType: ExternalEgressWatchEventType =
- cached && cached.namespace === server.namespace
- ? type === 'ADDED' && this.dependencies.sameDesiredRevision(server, cached)
- ? 'ADDED'
- : 'MODIFIED'
- : 'DELETED'
- const retryServer = cached && cached.namespace === server.namespace ? cached : server
- const key = this.key(server)
- const previousIntent = this.retryIntents.get(key)
- if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {
- this.retryAttempts.delete(key)
- // A new target restarts the backoff ladder, so the key is no longer capped.
- this.clearRetryAtCap(key)
- }
-
- const intent: ExternalEgressRetryIntent = {
- sequence: ++this.retrySequence,
- type: retryType,
- server: retryServer,
- }
- this.retryIntents.set(key, intent)
- // A timer or queued/active slot already owning the key consumes the newest
- // intent when it next enters the retry and per-server lanes.
- if (this.retryTimers.has(key) || this.retrySlots.has(key)) return
- this.armRetryTimer(key)
- }
-
- private armRetryTimer(key: string): void {
- if (
- this.stopped ||
- this.retryTimers.has(key) ||
- this.retrySlots.has(key) ||
- !this.retryIntents.has(key)
- ) {
- return
- }
- const attempt = Math.min(
- (this.retryAttempts.get(key) ?? 0) + 1,
- EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
- )
- const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
- this.retryAttempts.set(key, attempt)
- const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
- if (capped) this.markRetryAtCap(key)
- console.warn(
- `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` +
- `${capped ? ' (capped)' : ''} ` +
- `for McpServer "${this.retryIntents.get(key)!.server.name}" in ${delayMs}ms`
- )
-
- const timer = setTimeout(() => {
- if (this.retryTimers.get(key) !== timer) return
- this.retryTimers.delete(key)
- this.queueRetry(key)
- }, delayMs)
- this.retryTimers.set(key, timer)
- }
-
- private queueRetry(key: string): void {
- if (this.stopped || !this.retryIntents.has(key)) return
- if (this.retrySlots.has(key)) {
- // A timer can overlap work admitted through another lane. The durable
- // slot owns the latest intent and rearms it on exit if it remains live.
- return
- }
-
- const slot = {}
- this.retrySlots.set(key, slot)
- void this.retryLimiter
- .run(async () => {
- if (
- this.stopped ||
- this.retrySlots.get(key) !== slot ||
- !this.dependencies.inventoryAuthoritative()
- ) {
- return
- }
- // Do not capture intent before the limiter. Churn while capacity is
- // saturated is represented by one slot and resolved latest-wins here.
- const admittedIntent = this.retryIntents.get(key)
- if (!admittedIntent) return
- await this.dependencies.enqueue(admittedIntent.server, async () => {
- if (
- this.stopped ||
- this.retrySlots.get(key) !== slot ||
- !this.dependencies.inventoryAuthoritative()
- ) {
- return
- }
- // The keyed queue may also wait. Resolve the intent again so only the
- // latest desired revision or absence is replayed.
- const currentIntent = this.retryIntents.get(key)
- if (!currentIntent) return
- await this.replayIntent(currentIntent)
- })
- })
- .catch(error => {
- if (!this.stopped) {
- console.error(`[K8s] External egress retry execution failed for ${key}:`, error)
- }
- })
- .finally(() => {
- if (this.retrySlots.get(key) !== slot) return
- this.retrySlots.delete(key)
- // A replay may schedule its successor while still active, or a newer
- // intent may arrive after this slot selected work. Rearm only after the
- // key leaves the slot so one key never accumulates limiter backlog.
- if (!this.stopped && this.retryIntents.has(key) && !this.retryTimers.has(key)) {
- this.armRetryTimer(key)
- }
- })
- }
-
- stop(): void {
- if (this.stopped) return
- this.stopped = true
- this.retryLimiter.cancelPending()
- if (this.resyncTimer) {
- clearTimeout(this.resyncTimer)
- this.resyncTimer = null
- }
- for (const timer of this.retryTimers.values()) {
- clearTimeout(timer)
- }
- this.retryTimers.clear()
- this.retryAttempts.clear()
- this.retryIntents.clear()
- this.retrySlots.clear()
- if (this.retriesAtCap.size > 0) {
- this.retriesAtCap.clear()
- externalEgressRetriesAtCap.set(0)
- }
- this.inFlight.clear()
- }
-
- private async runResyncCore(): Promise<void> {
- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
- const servers = this.dependencies
- .listServers()
- .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
- .sort((left, right) => this.key(left).localeCompare(this.key(right)))
- if (servers.length === 0) return
-
- console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)
- let index = 0
- const workers = Array.from(
- { length: Math.min(EXTERNAL_EGRESS_MAX_CONCURRENCY, servers.length) },
- async () => {
- while (!this.stopped && this.dependencies.inventoryAuthoritative()) {
- const server = servers[index++]
- if (!server) return
- const key = this.key(server)
- if (this.inFlight.has(key)) {
- console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)
- continue
- }
- const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)
- if (jitterMs > 0) await delay(jitterMs)
- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
-
- await this.dependencies.enqueue(server, async () => {
- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
- // The fleet snapshot selects work only. Desired state is resolved
- // after jitter and after entering the per-server FIFO lane.
- const current = this.dependencies.getCurrentServer(server.name)
- if (
- !current ||
- current.namespace !== server.namespace ||
- (current.spec.egressBindings?.length ?? 0) === 0
- ) {
- return
- }
- try {
- const retryIntent = this.retryIntents.get(key)
- if (retryIntent) {
- await this.replayIntent(retryIntent)
- return
- }
- // Periodic refresh is egress-only and therefore cannot complete
- // a retry intent that may appear while the mutation is running.
- await this.reconcile('MODIFIED', current, {
- isCurrent: this.dependencies.inventoryAuthoritative,
- })
- } catch (error) {
- console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)
- this.scheduleRetry('MODIFIED', current)
- }
- })
- }
- }
- )
- await Promise.all(workers)
- }
-
- private async replayIntent(intent: ExternalEgressRetryIntent): Promise<void> {
- if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
- const handle = this.retryHandle(intent)
- if (!handle.isCurrent()) return
-
- const current = this.dependencies.getCurrentServer(intent.server.name)
- const type: ExternalEgressWatchEventType =
- current && current.namespace === intent.server.namespace ? 'MODIFIED' : 'DELETED'
- const server = type === 'DELETED' ? intent.server : current!
- await this.dependencies.replay(type, server, handle)
- }
-
- private reconcileIsCurrent(
- type: ExternalEgressWatchEventType,
- server: McpServerCRD,
- options: ExternalEgressReconcileOptions
- ): boolean {
- const callerIsCurrent =
- !this.stopped &&
- (!options.retry || options.retry.isCurrent()) &&
- (!options.isCurrent || options.isCurrent())
- if (!callerIsCurrent || type === 'DELETED') return callerIsCurrent
-
- const current = this.dependencies.getCurrentServer(server.name)
- return (
- current !== undefined &&
- current.namespace === server.namespace &&
- this.dependencies.sameDesiredRevision(server, current)
- )
- }
-
- private currentRetryHandle(key: string): ExternalEgressRetryHandle | undefined {
- const intent = this.retryIntents.get(key)
- return intent ? this.retryHandle(intent) : undefined
- }
-
- private retryHandle(intent: ExternalEgressRetryIntent): ExternalEgressRetryHandle {
- return {
- isCurrent: () =>
- !this.stopped &&
- this.retryIntents.get(this.key(intent.server))?.sequence === intent.sequence,
- complete: () => {
- if (this.retryIntents.get(this.key(intent.server))?.sequence !== intent.sequence) return
- this.clearRetry(intent.server)
- },
- }
- }
-
- private clearRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {
- const key = this.key(server)
- const timer = this.retryTimers.get(key)
- if (timer) {
- clearTimeout(timer)
- this.retryTimers.delete(key)
- }
- this.retryAttempts.delete(key)
- this.retryIntents.delete(key)
- this.clearRetryAtCap(key)
- }
-
- private markRetryAtCap(key: string): void {
- if (this.retriesAtCap.has(key)) return
- this.retriesAtCap.add(key)
- externalEgressRetriesAtCap.set(this.retriesAtCap.size)
- }
-
- private clearRetryAtCap(key: string): void {
- if (!this.retriesAtCap.delete(key)) return
- externalEgressRetriesAtCap.set(this.retriesAtCap.size)
- }
-
- private sameRetryTarget(
- previous: ExternalEgressRetryIntent,
- type: ExternalEgressWatchEventType,
- server: McpServerCRD
- ): boolean {
- const previousPresent = previous.type !== 'DELETED'
- const nextPresent = type !== 'DELETED'
- return (
- previousPresent === nextPresent &&
- this.dependencies.sameDesiredRevision(previous.server, server)
- )
- }
-
- private key(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {
- return `${server.namespace}/${server.name}`
- }
-
- private isWatchEventType(type: string): type is ExternalEgressWatchEventType {
- return type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED'
- }
-}
diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.ts
index d2fd7965b..c9cde6bcb 100644
--- a/host-context-controller/src/gfsReconciler.test.ts
+++ b/host-context-controller/src/gfsReconciler.test.ts
@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import { GfsK8sApi, GfsReconciler, GfsSeedClient } from './gfsReconciler'
import { GfsFactoryConfig } from './k8s/gfsFactory'
-import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'
+import type { GlobalFileSystemCRD } from './types'
const config: GfsFactoryConfig = {
gfsNamespace: 'gfs',
@@ -42,12 +42,11 @@ class FakeApi implements GfsK8sApi {
netpols: string[] = []
pvcs: string[] = []
deleted: string[] = []
- statuses: GlobalFileSystemStatus[] = []
+ statuses: { phase?: string; serviceUrl?: string }[] = []
writerNeedsUpdate = true
writerAvailable = true
writerAvailabilitySequence: boolean[] = []
failPvc = false
- statusFailuresRemaining = 0
async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<void> {
if (this.failPvc) throw new Error('simulated PVC apply failure')
@@ -102,11 +101,11 @@ class FakeApi implements GfsK8sApi {
async deletePvc(name: string): Promise<void> {
this.deleted.push(`pvc/${name}`)
}
- async patchStatus(_name: string, _ns: string, status: GlobalFileSystemStatus): Promise<void> {
- if (this.statusFailuresRemaining > 0) {
- this.statusFailuresRemaining--
- throw new Error('simulated status patch failure')
- }
+ async patchStatus(
+ _name: string,
+ _ns: string,
+ status: { phase?: string; serviceUrl?: string }
+ ): Promise<void> {
this.statuses.push(status)
}
}
@@ -219,59 +218,6 @@ describe('GfsReconciler.reconcile', () => {
expect(seed.calls).toBe(1)
})
- it('does not rewrite unchanged status but still publishes a later phase transition', async () => {
- const api = new FakeApi()
- api.writerNeedsUpdate = false
- const reconciler = new GfsReconciler(api, config, undefined, 0, 0)
-
- await reconciler.reconcile(gfs)
- const observedReady = { ...gfs, status: api.statuses[0] }
- await reconciler.reconcile(observedReady)
-
- expect(api.statuses).toHaveLength(1)
- expect(api.statuses[0]?.phase).toBe('Ready')
-
- api.writerAvailable = false
- await reconciler.reconcile(observedReady)
-
- expect(api.statuses.map(status => status.phase)).toEqual(['Ready', 'Initializing'])
- })
-
- it('publishes status for a same-name recreation even when the DELETE watch was missed', async () => {
- const api = new FakeApi()
- const reconciler = new GfsReconciler(api, config)
-
- await reconciler.reconcile(gfs)
- await reconciler.reconcile({ ...gfs, status: api.statuses[0] })
- await reconciler.reconcile({ ...gfs, status: undefined })
-
- expect(api.statuses).toHaveLength(2)
- })
-
- it('repairs status that was removed externally after HCC observed its prior write', async () => {
- const api = new FakeApi()
- const reconciler = new GfsReconciler(api, config)
-
- await reconciler.reconcile(gfs)
- await reconciler.reconcile({ ...gfs, status: api.statuses[0] })
- await reconciler.reconcile({ ...gfs, status: undefined })
-
- expect(api.statuses).toHaveLength(2)
- expect(api.statuses.at(-1)?.phase).toBe('Ready')
- })
-
- it('retries an unchanged status when the previous patch failed', async () => {
- const api = new FakeApi()
- api.statusFailuresRemaining = 1
- const reconciler = new GfsReconciler(api, config)
-
- await expect(reconciler.reconcile(gfs)).rejects.toThrow('simulated status patch failure')
- await expect(reconciler.reconcile(gfs)).resolves.toBeUndefined()
-
- expect(api.statuses).toHaveLength(1)
- expect(api.statuses[0]?.phase).toBe('Ready')
- })
-
it('reports Initializing and does NOT seed when the writer is not yet available', async () => {
const api = new FakeApi()
api.writerAvailable = false
@@ -319,15 +265,4 @@ describe('GfsReconciler.reconcileDelete', () => {
expect(api.deleted).toContain('pvc/gfs-drive')
expect(api.deleted.at(-1)).toBe('pvc/gfs-drive') // bytes removed LAST
})
-
- it('publishes status again after delete clears the prior write suppression', async () => {
- const api = new FakeApi()
- const reconciler = new GfsReconciler(api, config)
-
- await reconciler.reconcile(gfs)
- await reconciler.reconcileDelete(gfs)
- await reconciler.reconcile(gfs)
-
- expect(api.statuses).toHaveLength(2)
- })
})
diff --git a/host-context-controller/src/gfsReconciler.ts b/host-context-controller/src/gfsReconciler.ts
index 0462a12e6..de5e29ea2 100644
--- a/host-context-controller/src/gfsReconciler.ts
+++ b/host-context-controller/src/gfsReconciler.ts
@@ -95,27 +95,6 @@ export class GfsReconciler {
} while (true)
}
- private async writeStatus(
- gfs: GlobalFileSystemCRD,
- status: GlobalFileSystemStatus
- ): Promise<void> {
- // Compare against status observed on this exact LIST/watch object, not
- // process memory keyed only by name. A local name cache could suppress the
- // repair of a recreated object when its DELETE event was missed, or of
- // status that another actor removed. Conversely, an identical observed
- // status proves this is our own status-only MODIFIED event and closes the
- // patch -> watch -> reconcile loop without another API write.
- if (
- gfs.status?.phase === status.phase &&
- gfs.status?.pvcName === status.pvcName &&
- gfs.status?.serviceName === status.serviceName &&
- gfs.status?.serviceUrl === status.serviceUrl
- ) {
- return
- }
- await this.api.patchStatus(gfs.name, gfs.namespace, status)
- }
-
async reconcile(gfs: GlobalFileSystemCRD): Promise<void> {
const ns = this.config.gfsNamespace
const writerDeployment = buildDeployment(gfs, this.config, 'writer')
@@ -156,7 +135,7 @@ export class GfsReconciler {
}
const phase = writerReady ? 'Ready' : 'Initializing'
- await this.writeStatus(gfs, {
+ await this.api.patchStatus(gfs.name, gfs.namespace, {
phase,
pvcName: pvcName(gfs),
serviceName: serviceName(gfs),
diff --git a/host-context-controller/src/hostFleetScheduler.test.ts b/host-context-controller/src/hostFleetScheduler.test.ts
deleted file mode 100644
index 0f6817cfa..000000000
--- a/host-context-controller/src/hostFleetScheduler.test.ts
+++ /dev/null
@@ -1,284 +0,0 @@
-import { afterEach, describe, expect, it, vi } from 'vitest'
-import {
- type HostFleetReconcileRequest,
- type HostFleetRequestResult,
- HostFleetScheduler,
-} from './hostFleetScheduler'
-
-function deferred<T = void>() {
- let resolve!: (value: T | PromiseLike<T>) => void
- let reject!: (reason?: unknown) => void
- const promise = new Promise<T>((resolvePromise, rejectPromise) => {
- resolve = resolvePromise
- reject = rejectPromise
- })
- return { promise, resolve, reject }
-}
-
-describe('HostFleetScheduler', () => {
- afterEach(() => {
- vi.useRealTimers()
- })
-
- it('merges newer work into one trailing pass with full coverage and latest inputs', async () => {
- const activePass = deferred()
- const requests: HostFleetReconcileRequest[] = []
- const results: HostFleetRequestResult[] = []
- const scheduler = new HostFleetScheduler({
- perform: async request => {
- requests.push({ ...request })
- if (requests.length === 1) await activePass.promise
- },
- recordRequest: result => results.push(result),
- })
-
- const firstGeneration = scheduler.beginLifecycleTransition()
- scheduler.bumpInputRevision()
- const active = scheduler.request('active lifecycle', firstGeneration, 'lifecycle')
- await vi.waitFor(() => expect(requests).toHaveLength(1))
-
- scheduler.bumpInputRevision()
- const pendingFull = scheduler.request('pending full', firstGeneration, 'full')
- const latestGeneration = scheduler.beginLifecycleTransition()
- scheduler.bumpInputRevision()
- const latestLifecycle = scheduler.request('latest lifecycle', latestGeneration, 'lifecycle')
-
- activePass.resolve()
- await Promise.all([active, pendingFull, latestLifecycle])
-
- expect(requests).toEqual([
- {
- reason: 'active lifecycle',
- mode: 'lifecycle',
- ccLifecycleGeneration: firstGeneration,
- inputRevision: 1,
- },
- {
- reason: 'latest lifecycle',
- mode: 'full',
- ccLifecycleGeneration: latestGeneration,
- inputRevision: 3,
- },
- ])
- expect(results).toEqual(['started', 'coalesced', 'coalesced', 'trailing'])
- })
-
- it('carries the known lifecycle generation when merging a generation-less full request', async () => {
- const activePass = deferred()
- const requests: HostFleetReconcileRequest[] = []
- const scheduler = new HostFleetScheduler({
- perform: async request => {
- requests.push({ ...request })
- if (requests.length === 1) await activePass.promise
- },
- recordRequest: vi.fn(),
- })
- const generation = scheduler.beginLifecycleTransition()
- const active = scheduler.request('active')
- await vi.waitFor(() => expect(requests).toHaveLength(1))
- scheduler.bumpInputRevision()
- const pendingLifecycle = scheduler.request('lifecycle', generation, 'lifecycle')
- scheduler.bumpInputRevision()
- const pendingFull = scheduler.request('full')
-
- activePass.resolve()
- await Promise.all([active, pendingLifecycle, pendingFull])
-
- expect(requests[1]).toMatchObject({
- reason: 'full',
- mode: 'full',
- ccLifecycleGeneration: generation,
- inputRevision: 2,
- })
- })
-
- it('reuses active work only when its mode, revision, and lifecycle generation cover the request', async () => {
- const activePass = deferred()
- const requests: HostFleetReconcileRequest[] = []
- const scheduler = new HostFleetScheduler({
- perform: async request => {
- requests.push({ ...request })
- if (requests.length === 1) await activePass.promise
- },
- recordRequest: vi.fn(),
- })
- const generation = scheduler.beginLifecycleTransition()
- scheduler.bumpInputRevision()
-
- const active = scheduler.request('full', generation, 'full')
- await vi.waitFor(() => expect(requests).toHaveLength(1))
- const covered = scheduler.request('covered lifecycle', generation, 'lifecycle')
-
- activePass.resolve()
- await Promise.all([active, covered])
-
- expect(requests).toHaveLength(1)
- })
-
- it('rejects stale or already-applied lifecycle work without suppressing a current full pass', async () => {
- const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(
- async () => undefined
- )
- const scheduler = new HostFleetScheduler({
- perform,
- recordRequest: vi.fn(),
- })
- const firstGeneration = scheduler.beginLifecycleTransition()
- scheduler.markLifecycleApplied(firstGeneration)
-
- await scheduler.request('already applied', firstGeneration, 'lifecycle')
- await scheduler.request('current full', firstGeneration, 'full')
- scheduler.beginLifecycleTransition()
- await scheduler.request('stale full', firstGeneration, 'full')
-
- expect(perform).toHaveBeenCalledOnce()
- expect(perform).toHaveBeenCalledWith(
- expect.objectContaining({ reason: 'current full', mode: 'full' })
- )
- })
-
- it('does not let a rejected pass poison later work', async () => {
- const failure = new Error('transient failure')
- const perform = vi
- .fn<(request: HostFleetReconcileRequest) => Promise<void>>()
- .mockRejectedValueOnce(failure)
- .mockResolvedValueOnce(undefined)
- const scheduler = new HostFleetScheduler({
- perform,
- recordRequest: vi.fn(),
- })
-
- await expect(scheduler.request('fails')).rejects.toThrow(failure)
- await expect(scheduler.request('recovers')).resolves.toBeUndefined()
-
- expect(perform).toHaveBeenCalledTimes(2)
- })
-
- it('releases active and queued waiters on stop without starting queued work', async () => {
- const activePass = deferred()
- const requests: HostFleetReconcileRequest[] = []
- const scheduler = new HostFleetScheduler({
- perform: async request => {
- requests.push({ ...request })
- await activePass.promise
- },
- recordRequest: vi.fn(),
- })
-
- const active = scheduler.request('active')
- await vi.waitFor(() => expect(requests).toHaveLength(1))
- scheduler.bumpInputRevision()
- const queued = scheduler.request('queued')
-
- scheduler.stop()
- await Promise.all([active, queued, scheduler.request('after stop')])
- activePass.resolve()
- await Promise.resolve()
-
- expect(requests).toHaveLength(1)
- })
-
- it('tracks lifecycle state and cancels obsolete retries', async () => {
- vi.useFakeTimers()
- const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(
- async () => undefined
- )
- const scheduler = new HostFleetScheduler({
- perform,
- recordRequest: vi.fn(),
- })
- const firstGeneration = scheduler.beginLifecycleTransition()
-
- expect(scheduler.currentLifecycleGeneration).toBe(firstGeneration)
- expect(scheduler.appliedLifecycleGeneration).toBe(-1)
-
- scheduler.scheduleLifecycleRetry({
- reason: 'failed lifecycle',
- mode: 'lifecycle',
- ccLifecycleGeneration: firstGeneration,
- inputRevision: 0,
- })
- await vi.advanceTimersByTimeAsync(4999)
- expect(perform).not.toHaveBeenCalled()
-
- const secondGeneration = scheduler.beginLifecycleTransition()
- await vi.advanceTimersByTimeAsync(1)
- expect(perform).not.toHaveBeenCalled()
-
- scheduler.scheduleLifecycleRetry({
- reason: 'current lifecycle',
- mode: 'lifecycle',
- ccLifecycleGeneration: secondGeneration,
- inputRevision: 0,
- })
- scheduler.markLifecycleApplied(secondGeneration)
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(scheduler.appliedLifecycleGeneration).toBe(secondGeneration)
- expect(perform).not.toHaveBeenCalled()
- })
-
- it('does not let a stale applied marker clear the current generation retry', async () => {
- vi.useFakeTimers()
- const perform = vi.fn<(request: HostFleetReconcileRequest) => Promise<void>>(
- async () => undefined
- )
- const scheduler = new HostFleetScheduler({
- perform,
- recordRequest: vi.fn(),
- })
- const staleGeneration = scheduler.beginLifecycleTransition()
- const currentGeneration = scheduler.beginLifecycleTransition()
- scheduler.scheduleLifecycleRetry({
- reason: 'current lifecycle',
- mode: 'lifecycle',
- ccLifecycleGeneration: currentGeneration,
- inputRevision: 0,
- })
-
- scheduler.markLifecycleApplied(staleGeneration)
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(perform).toHaveBeenCalledWith(
- expect.objectContaining({ ccLifecycleGeneration: currentGeneration })
- )
- })
-
- it('retries an unapplied current lifecycle generation with bounded backoff', async () => {
- vi.useFakeTimers()
- const requests: HostFleetReconcileRequest[] = []
- const scheduler = new HostFleetScheduler({
- perform: async request => {
- requests.push({ ...request })
- },
- recordRequest: vi.fn(),
- })
- const generation = scheduler.beginLifecycleTransition()
- const failedRequest: HostFleetReconcileRequest = {
- reason: 'failed lifecycle',
- mode: 'lifecycle',
- ccLifecycleGeneration: generation,
- inputRevision: 0,
- }
-
- scheduler.scheduleLifecycleRetry(failedRequest)
- await vi.advanceTimersByTimeAsync(5000)
- scheduler.scheduleLifecycleRetry(failedRequest)
- await vi.advanceTimersByTimeAsync(14999)
- expect(requests).toHaveLength(1)
- await vi.advanceTimersByTimeAsync(1)
-
- expect(requests).toEqual([
- expect.objectContaining({
- reason: 'CommunicationChannel lifecycle convergence retry',
- mode: 'lifecycle',
- ccLifecycleGeneration: generation,
- }),
- expect.objectContaining({
- reason: 'CommunicationChannel lifecycle convergence retry',
- mode: 'lifecycle',
- ccLifecycleGeneration: generation,
- }),
- ])
- })
-})
diff --git a/host-context-controller/src/hostFleetScheduler.ts b/host-context-controller/src/hostFleetScheduler.ts
deleted file mode 100644
index d872d6cad..000000000
--- a/host-context-controller/src/hostFleetScheduler.ts
+++ /dev/null
@@ -1,246 +0,0 @@
-const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000] as const
-
-export type HostFleetReconcileMode = 'full' | 'lifecycle'
-
-export type HostFleetReconcileRequest = {
- reason: string
- mode: HostFleetReconcileMode
- ccLifecycleGeneration?: number
- inputRevision: number
-}
-
-type PendingHostFleetReconcile = HostFleetReconcileRequest & {
- promise: Promise<void>
- resolve: () => void
-}
-
-type ActiveHostFleetReconcile = HostFleetReconcileRequest & {
- promise: Promise<void>
-}
-
-export type HostFleetRequestResult = 'started' | 'coalesced' | 'trailing'
-
-type HostFleetSchedulerDependencies = {
- perform: (request: HostFleetReconcileRequest) => Promise<void>
- recordRequest: (result: HostFleetRequestResult) => void
-}
-
-function hostFleetRequestCovers(
- active: HostFleetReconcileRequest,
- requested: HostFleetReconcileRequest
-): boolean {
- const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'
- if (!modeCovered) return false
- if (active.inputRevision !== requested.inputRevision) return false
- if (requested.ccLifecycleGeneration === undefined) return true
- return active.ccLifecycleGeneration === requested.ccLifecycleGeneration
-}
-
-function mergeHostFleetRequests(
- pending: HostFleetReconcileRequest,
- requested: HostFleetReconcileRequest
-): HostFleetReconcileRequest {
- const pendingGeneration = pending.ccLifecycleGeneration
- const requestedGeneration = requested.ccLifecycleGeneration
- return {
- reason: requested.reason,
- mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',
- inputRevision: Math.max(pending.inputRevision, requested.inputRevision),
- ccLifecycleGeneration:
- pendingGeneration === undefined
- ? requestedGeneration
- : requestedGeneration === undefined
- ? pendingGeneration
- : Math.max(pendingGeneration, requestedGeneration),
- }
-}
-
-/**
- * Owns temporal coordination for Host fleet convergence. Kubernetes reads and
- * mutations stay behind `perform`, so this scheduler only serializes requests,
- * coalesces a single trailing pass, and manages lifecycle retry/shutdown state.
- */
-export class HostFleetScheduler {
- private ccLifecycleGeneration = 0
- private ccAppliedLifecycleGeneration = -1
- private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null
- private ccFleetRetryAttempt = 0
- private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null
- private hostFleetReconcilePending: PendingHostFleetReconcile | null = null
- private hostFleetInputRevision = 0
- private resolveHostFleetShutdown: () => void = () => {}
- private readonly hostFleetShutdown = new Promise<void>(resolve => {
- this.resolveHostFleetShutdown = resolve
- })
- private stopped = false
-
- constructor(private readonly dependencies: HostFleetSchedulerDependencies) {}
-
- get currentLifecycleGeneration(): number {
- return this.ccLifecycleGeneration
- }
-
- get appliedLifecycleGeneration(): number {
- return this.ccAppliedLifecycleGeneration
- }
-
- beginLifecycleTransition(): number {
- this.ccLifecycleGeneration += 1
- this.ccFleetRetryAttempt = 0
- this.clearLifecycleRetry()
- return this.ccLifecycleGeneration
- }
-
- bumpInputRevision(): number {
- this.hostFleetInputRevision += 1
- return this.hostFleetInputRevision
- }
-
- markLifecycleApplied(generation: number): void {
- if (generation !== this.ccLifecycleGeneration) return
- this.ccAppliedLifecycleGeneration = generation
- this.ccFleetRetryAttempt = 0
- this.clearLifecycleRetry()
- }
-
- scheduleLifecycleRetry(request: HostFleetReconcileRequest): void {
- const generation = request.ccLifecycleGeneration
- if (generation === undefined) return
- if (
- this.stopped ||
- generation !== this.ccLifecycleGeneration ||
- generation === this.ccAppliedLifecycleGeneration ||
- this.ccFleetRetryTimer
- ) {
- return
- }
-
- const delayIndex = Math.min(
- this.ccFleetRetryAttempt,
- COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1
- )
- const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]
- this.ccFleetRetryAttempt += 1
- this.ccFleetRetryTimer = setTimeout(() => {
- this.ccFleetRetryTimer = null
- if (
- this.stopped ||
- generation !== this.ccLifecycleGeneration ||
- generation === this.ccAppliedLifecycleGeneration
- ) {
- return
- }
- void this.request(
- 'CommunicationChannel lifecycle convergence retry',
- generation,
- request.mode
- )
- }, retryDelay)
- }
-
- /**
- * Cache transitions, periodic resyncs, and retries share one active pass and
- * at most one merged trailing pass. Each caller waits only for work covering
- * its request, keeping startup bounded under a sustained retry stream.
- */
- request(
- reason: string,
- ccLifecycleGeneration?: number,
- mode?: HostFleetReconcileMode
- ): Promise<void> {
- if (this.stopped) return Promise.resolve()
- const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')
- if (
- ccLifecycleGeneration !== undefined &&
- (ccLifecycleGeneration !== this.ccLifecycleGeneration ||
- (requestMode === 'lifecycle' &&
- ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))
- ) {
- return Promise.resolve()
- }
-
- const request: HostFleetReconcileRequest = {
- reason,
- mode: requestMode,
- ccLifecycleGeneration,
- inputRevision: this.hostFleetInputRevision,
- }
- const active = this.hostFleetReconcileInFlight
- if (!active) {
- this.dependencies.recordRequest('started')
- return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))
- }
- if (hostFleetRequestCovers(active, request)) {
- this.dependencies.recordRequest('coalesced')
- return this.waitForHostFleetOrShutdown(active.promise)
- }
-
- if (!this.hostFleetReconcilePending) {
- let resolve!: () => void
- const promise = new Promise<void>(resolvePromise => {
- resolve = resolvePromise
- })
- this.hostFleetReconcilePending = {
- ...request,
- promise,
- resolve,
- }
- } else {
- const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)
- this.hostFleetReconcilePending.reason = merged.reason
- this.hostFleetReconcilePending.mode = merged.mode
- this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration
- this.hostFleetReconcilePending.inputRevision = merged.inputRevision
- }
- this.dependencies.recordRequest('coalesced')
- return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)
- }
-
- stop(): void {
- if (this.stopped) return
- this.stopped = true
- this.resolveHostFleetShutdown()
- this.hostFleetReconcilePending?.resolve()
- this.hostFleetReconcilePending = null
- this.clearLifecycleRetry()
- }
-
- private clearLifecycleRetry(): void {
- if (!this.ccFleetRetryTimer) return
- clearTimeout(this.ccFleetRetryTimer)
- this.ccFleetRetryTimer = null
- }
-
- private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {
- return Promise.race([coverage, this.hostFleetShutdown])
- }
-
- private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {
- const reconcileRequest: HostFleetReconcileRequest = {
- reason: request.reason,
- mode: request.mode,
- ccLifecycleGeneration: request.ccLifecycleGeneration,
- inputRevision: request.inputRevision,
- }
- const promise = this.dependencies.perform(reconcileRequest)
- const active: ActiveHostFleetReconcile = { ...reconcileRequest, promise }
- this.hostFleetReconcileInFlight = active
-
- const settle = () => {
- if (this.hostFleetReconcileInFlight !== active) return
- this.hostFleetReconcileInFlight = null
- const pending = this.hostFleetReconcilePending
- this.hostFleetReconcilePending = null
- if (!pending) return
- if (this.stopped) {
- pending.resolve()
- return
- }
- this.dependencies.recordRequest('trailing')
- const trailing = this.startHostFleetReconcile(pending)
- void trailing.then(pending.resolve, pending.resolve)
- }
- void promise.then(settle, settle)
- return promise
- }
-}
diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.ts
index 53625ff43..495a4bc38 100644
--- a/host-context-controller/src/hostReconciler.test.ts
+++ b/host-context-controller/src/hostReconciler.test.ts
@@ -943,42 +943,6 @@ describe('HostReconciler cleanup authority gate', () => {
expect(del).not.toHaveBeenCalled()
})
- it('defers orphan deletion when authority changes before serializer admission', async () => {
- const getObj = vi.fn().mockRejectedValue({ code: 404 })
- const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
- const authority = { known: true, generation: 3 }
- reconciler.setHostWatchAuthority(() => ({ ...authority }))
- reconciler.setResolveCurrentHost(() => undefined)
- const internals = reconciler as unknown as {
- lifecycle: {
- serializeByHost(name: string, task: () => Promise<void>): Promise<void>
- }
- listManagedHostDeployments(): Promise<k8s.V1Deployment[]>
- deleteHostRuntimeResources(name: string, namespace: string): Promise<void>
- }
- vi.spyOn(internals, 'listManagedHostDeployments').mockResolvedValue([
- orphanDeployment('authority-race-host'),
- ])
- const del = vi.spyOn(internals, 'deleteHostRuntimeResources').mockResolvedValue(undefined)
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = internals.lifecycle.serializeByHost('authority-race-host', async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- })
- await blockerStarted.promise
-
- const cleanup = reconciler.fullReconcile([])
- await vi.waitFor(() => expect(getObj).toHaveBeenCalledOnce())
- authority.known = false
- authority.generation = 4
- releaseBlocker.resolve(undefined)
- await occupied
- await cleanup
-
- expect(del).not.toHaveBeenCalled()
- })
-
it('defers an orphan whose fresh authoritative read fails', async () => {
const getObj = vi.fn().mockRejectedValue({ code: 503 })
const reconciler = makeCleanupReconciler({ getNamespacedCustomObject: getObj })
@@ -1032,687 +996,3 @@ describe('HostReconciler cleanup authority gate', () => {
expect(del).not.toHaveBeenCalled()
})
})
-
-describe('HostReconciler Host inventory mutation authority', () => {
- function wireAuthoritativeHost(
- reconciler: HostReconciler,
- host: HostCRD,
- authorityRef: { current: { known: boolean; generation: number } }
- ): void {
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => authorityRef.current)
- }
-
- function stubPreDeploymentReconcileEffects(reconciler: HostReconciler): void {
- vi.spyOn(reconciler as any, 'validateHostSecret').mockResolvedValue({ ok: true })
- for (const method of [
- 'ensureHostServiceAccount',
- 'ensureHostRole',
- 'ensureHostRoleBinding',
- 'ensurePvc',
- 'ensureService',
- 'ensureMcpHostIngressNetworkPolicy',
- 'ensureMcpHostGfsEgressNetworkPolicy',
- 'ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy',
- 'ensureRpcProxyMcpHostIngressNetworkPolicy',
- 'ensureRpcProxyHostEgressNetworkPolicy',
- 'ensureDesktopNetworkPolicy',
- 'ensureChannelReaderEgressNetworkPolicy',
- 'ensureWorkflowApprovalReaderHostEgressNetworkPolicy',
- ]) {
- vi.spyOn(reconciler as any, method).mockResolvedValue(undefined)
- }
- }
-
- function runtimeTokenScopeWithoutChannels(host: HostCRD): string {
- return (
- HostReconciler as unknown as {
- runtimeTokenScopeHash(selected: HostCRD, hasChannelIngress: boolean): string
- }
- ).runtimeTokenScopeHash(host, false)
- }
-
- it('rejects a queued reconcile when Host authority changes before admission', async () => {
- const reconciler = makeBasicReconciler()
- const host = makeHost({ name: 'authority-race' })
- const authority = { current: { known: true, generation: 7 } }
- wireAuthoritativeHost(reconciler, host, authority)
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }) as Promise<void>
- await blockerStarted.promise
-
- const pending = reconciler.reconcile(host)
- authority.current = { known: false, generation: 8 }
- releaseBlocker.resolve(undefined)
- await occupied
-
- await expect(pending).rejects.toThrow(/Host inventory authority/)
- expect(reconcileCore).not.toHaveBeenCalled()
- })
-
- it('rejects a queued fleet worker after Context authority retires and recovers', async () => {
- const reconciler = makeBasicReconciler()
- const hostA = makeHost({ name: 'context-authority-a' })
- const hostB = makeHost({ name: 'context-authority-b' })
- const hosts = new Map([
- [hostA.name, hostA],
- [hostB.name, hostB],
- ])
- reconciler.setResolveCurrentHost(name => hosts.get(name))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 7 }))
- const mutationAuthority = {
- current: {
- known: true,
- hostGeneration: 7,
- contextGeneration: 11,
- },
- }
- ;(reconciler as any).setHostMutationAuthority(() => mutationAuthority.current)
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
- vi.spyOn(reconciler as any, 'collectHostCleanupFailures').mockResolvedValue([])
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = (reconciler as any).lifecycle.serializeByHost(hostB.name, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }) as Promise<void>
- await blockerStarted.promise
-
- const fleet = reconciler.fullReconcile([hostA, hostB])
- await vi.waitFor(() => expect(reconcileCore).toHaveBeenCalledWith(hostA, expect.any(Function)))
-
- // Context retired and recovered to a fresh LIST -> WATCH generation while
- // Host authority remained continuously known: a host-only lease would miss
- // this ABA transition and admit Host B with stale derived mounts.
- mutationAuthority.current = {
- known: true,
- hostGeneration: 7,
- contextGeneration: 13,
- }
- releaseBlocker.resolve(undefined)
- await occupied
-
- await expect(fleet).rejects.toBeInstanceOf(HostFleetReconcileError)
- expect(reconcileCore).toHaveBeenCalledTimes(1)
- expect(reconcileCore).not.toHaveBeenCalledWith(hostB, expect.any(Function))
- })
-
- it('retires a Host reconcile when its Context is superseded within the same watch', async () => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'context-revision-race', contextRef: 'context-a' }),
- uid: 'context-revision-race-uid',
- generation: 5,
- }
- const contextV1 = { name: 'context-a', spec: { sharedFileSystems: [] } }
- const contextV2 = {
- name: 'context-a',
- spec: {
- sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
- },
- }
- let selectedContext: unknown = contextV1
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostMutationAuthority(() => ({
- known: true,
- hostGeneration: 7,
- contextGeneration: 11,
- }))
- reconciler.setResolveHostMutationDependencies(() => [selectedContext])
-
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred()
- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
- secretReadStarted.resolve(undefined)
- await releaseSecretRead.promise
- return { ok: true }
- })
- const ensureServiceAccount = vi
- .spyOn(reconciler as any, 'ensureHostServiceAccount')
- .mockResolvedValue(undefined)
-
- // R1-M2: a benign mid-pass supersession must NOT be reported as a hard
- // reconcile failure. The single reconcile() catch used to fire
- // enqueueControllerError('reconcile_exception') → controller_error telemetry
- // (status:'failed') AND an administrative 'failed' outcome to control-api for
- // these retirements. Spy (observe-only) to prove the benign branch emits
- // neither, while still re-throwing so callers treat it as a retire.
- const enqueueControllerError = vi.spyOn(reconciler as any, 'enqueueControllerError')
- const enqueueAdministrativeOutcome = vi.spyOn(reconciler as any, 'enqueueAdministrativeOutcome')
-
- const pending = reconciler.reconcile(host)
- await secretReadStarted.promise
- selectedContext = contextV2
- releaseSecretRead.resolve(undefined)
-
- await expect(pending).rejects.toThrow(/Host mutation dependency changed/)
- expect(ensureServiceAccount).not.toHaveBeenCalled()
- expect(enqueueControllerError).not.toHaveBeenCalled()
- expect(enqueueAdministrativeOutcome).not.toHaveBeenCalledWith(
- expect.anything(),
- 'failed',
- expect.anything()
- )
- })
-
- it('keeps a Host reconcile admitted when a watch replaces dependencies with deep-equal values', async () => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'deep-equal-dependency', contextRef: 'context-a' }),
- uid: 'deep-equal-dependency-uid',
- generation: 5,
- }
- let selectedContext: unknown = {
- name: 'context-a',
- spec: {
- sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
- },
- }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostMutationAuthority(() => ({
- known: true,
- hostGeneration: 7,
- contextGeneration: 11,
- }))
- reconciler.setResolveHostMutationDependencies(() => [selectedContext])
-
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred()
- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
- secretReadStarted.resolve(undefined)
- await releaseSecretRead.promise
- return { ok: true }
- })
- const admitted = new Error('deep-equal dependency admitted')
- const ensureServiceAccount = vi
- .spyOn(reconciler as any, 'ensureHostServiceAccount')
- .mockRejectedValue(admitted)
-
- const pending = reconciler.reconcile(host)
- await secretReadStarted.promise
- selectedContext = structuredClone(selectedContext)
- releaseSecretRead.resolve(undefined)
-
- await expect(pending).rejects.toBe(admitted)
- expect(ensureServiceAccount).toHaveBeenCalled()
- })
-
- it.each([
- [
- 'referenced SharedFileSystem revision',
- { name: 'documents', spec: { storageClassName: 'fast-v1' } },
- { name: 'documents', spec: { storageClassName: 'fast-v2' } },
- ],
- ['referenced SharedFileSystem mountability', true, false],
- [
- 'Host-scoped CommunicationChannel revision',
- { name: 'telegram', spec: { hostRef: 'dependency-race' } },
- {
- name: 'telegram',
- spec: {
- hostRef: 'dependency-race',
- credentialsSecretRef: { name: 'telegram-v2' },
- },
- },
- ],
- ])('retires a Host reconcile when its %s changes', async (_label, original, replacement) => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'dependency-race', contextRef: 'context-a' }),
- uid: 'dependency-race-uid',
- generation: 5,
- }
- const context = { name: 'context-a', spec: { sharedFileSystems: [] } }
- let selectedDependency: unknown = original
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostMutationAuthority(() => ({
- known: true,
- hostGeneration: 7,
- contextGeneration: 11,
- }))
- reconciler.setResolveHostMutationDependencies(() => [context, selectedDependency])
-
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred()
- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
- secretReadStarted.resolve(undefined)
- await releaseSecretRead.promise
- return { ok: true }
- })
- const ensureServiceAccount = vi
- .spyOn(reconciler as any, 'ensureHostServiceAccount')
- .mockResolvedValue(undefined)
-
- const pending = reconciler.reconcile(host)
- await secretReadStarted.promise
- selectedDependency = replacement
- releaseSecretRead.resolve(undefined)
-
- await expect(pending).rejects.toThrow(/Host mutation dependency changed/)
- expect(ensureServiceAccount).not.toHaveBeenCalled()
- })
-
- it('does not delete runtime after a Secret rejection when Host authority changes in flight', async () => {
- const reconciler = makeBasicReconciler()
- const host = { ...makeHost({ name: 'secret-authority-race' }), uid: 'host-uid', generation: 3 }
- const authority = { current: { known: true, generation: 7 } }
- wireAuthoritativeHost(reconciler, host, authority)
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred()
- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
- secretReadStarted.resolve(undefined)
- await releaseSecretRead.promise
- return {
- ok: false,
- reason: 'SecretNotFound',
- message: 'secret missing in stale spec',
- }
- })
- const deleteRuntime = vi
- .spyOn(reconciler as any, 'deleteHostRuntimeResources')
- .mockResolvedValue(undefined)
-
- const pending = reconciler.reconcile(host)
- await secretReadStarted.promise
- authority.current = { known: false, generation: 8 }
- releaseSecretRead.resolve(undefined)
-
- await expect(pending).rejects.toThrow(/Host inventory authority/)
- expect(deleteRuntime).not.toHaveBeenCalled()
- })
-
- it.each([
- [
- 'UID',
- (host: HostCRD) => ({ ...host, uid: 'replacement-host-uid', generation: 1 }),
- /Host identity/,
- ],
- [
- 'spec generation',
- (host: HostCRD) => ({
- ...host,
- generation: (host.generation ?? 0) + 1,
- spec: { ...host.spec, secretRef: 'replacement-secret' },
- }),
- /Host spec generation/,
- ],
- ] as const)(
- 'does not delete runtime after a Secret rejection when the Host %s changes in flight',
- async (_, replaceCurrent, expectedError) => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'secret-identity-race' }),
- uid: 'host-uid',
- generation: 3,
- }
- let current: HostCRD = host
- const authority = { current: { known: true, generation: 7 } }
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => authority.current)
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred()
- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
- secretReadStarted.resolve(undefined)
- await releaseSecretRead.promise
- return {
- ok: false,
- reason: 'SecretNotFound',
- message: 'secret missing in stale spec',
- }
- })
- const deleteRuntime = vi
- .spyOn(reconciler as any, 'deleteHostRuntimeResources')
- .mockResolvedValue(undefined)
-
- const pending = reconciler.reconcile(host)
- await secretReadStarted.promise
- current = replaceCurrent(host)
- releaseSecretRead.resolve(undefined)
-
- await expect(pending).rejects.toThrow(expectedError)
- expect(deleteRuntime).not.toHaveBeenCalled()
- }
- )
-
- it('revalidates the Host spec after token provisioning and before Deployment mutation', async () => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'deployment-generation-race' }),
- uid: 'host-uid',
- generation: 3,
- }
- let current = host
- const authority = { current: { known: true, generation: 11 } }
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => authority.current)
- stubPreDeploymentReconcileEffects(reconciler)
- const tokenProvisionStarted = deferred()
- const releaseTokenProvision = deferred()
- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockImplementation(async () => {
- tokenProvisionStarted.resolve(undefined)
- await releaseTokenProvision.promise
- return { revision: 'stale-revision', scopeHash: 'stale-scope' }
- })
- const ensureDeployment = vi
- .spyOn(reconciler as any, 'ensureDeployment')
- .mockResolvedValue(undefined)
-
- const pending = reconciler.reconcile(host)
- await tokenProvisionStarted.promise
- current = {
- ...host,
- generation: 4,
- spec: { ...host.spec, contextRef: 'replacement-context' },
- }
- releaseTokenProvision.resolve(undefined)
-
- await expect(pending).rejects.toThrow(/Host spec generation/)
- expect(ensureDeployment).not.toHaveBeenCalled()
- })
-
- it('revalidates the Host spec at the final Deployment create-state boundary', async () => {
- const appsApi = createMockAppsApi()
- const reconciler = new HostReconciler({} as k8s.KubeConfig, {
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(createMockCoreApi()),
- networkingApi: asNetworkingApi(createMockNetworkingApi()),
- rbacApi: asRbacApi(createMockRbacApi()),
- })
- const host = {
- ...makeHost({ name: 'deployment-create-race' }),
- uid: 'host-uid',
- generation: 3,
- }
- let current = host
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))
- stubPreDeploymentReconcileEffects(reconciler)
- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
- revision: 'runtime-revision',
- scopeHash: runtimeTokenScopeWithoutChannels(host),
- })
- let channelIngressReads = 0
- vi.spyOn(reconciler as any, 'hasChannelIngress').mockImplementation(() => {
- channelIngressReads += 1
- if (channelIngressReads === 2) {
- current = {
- ...host,
- generation: 4,
- spec: { ...host.spec, contextRef: 'replacement-context' },
- }
- }
- return false
- })
-
- await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
-
- expect(channelIngressReads).toBe(2)
- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
- expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
- })
-
- it('revalidates the Host spec after a Deployment create conflict and before replace', async () => {
- const appsApi = createMockAppsApi()
- const reconciler = new HostReconciler({} as k8s.KubeConfig, {
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(createMockCoreApi()),
- networkingApi: asNetworkingApi(createMockNetworkingApi()),
- rbacApi: asRbacApi(createMockRbacApi()),
- })
- const host = {
- ...makeHost({ name: 'deployment-replace-race' }),
- uid: 'host-uid',
- generation: 3,
- }
- let current = host
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))
- stubPreDeploymentReconcileEffects(reconciler)
- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
- revision: 'runtime-revision',
- scopeHash: runtimeTokenScopeWithoutChannels(host),
- })
- vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
- appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
- appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {
- current = {
- ...host,
- generation: 4,
- spec: { ...host.spec, contextRef: 'replacement-context' },
- }
- return { metadata: { resourceVersion: 'rv-existing' } }
- })
-
- await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
-
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
- })
-
- it('revalidates the Host spec before retrying a conflicted Deployment replace', async () => {
- const appsApi = createMockAppsApi()
- const reconciler = new HostReconciler({} as k8s.KubeConfig, {
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(createMockCoreApi()),
- networkingApi: asNetworkingApi(createMockNetworkingApi()),
- rbacApi: asRbacApi(createMockRbacApi()),
- })
- const host = {
- ...makeHost({ name: 'deployment-retry-race' }),
- uid: 'host-uid',
- generation: 3,
- }
- let current = host
- let deploymentReads = 0
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))
- stubPreDeploymentReconcileEffects(reconciler)
- vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue({
- revision: 'runtime-revision',
- scopeHash: runtimeTokenScopeWithoutChannels(host),
- })
- vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
- appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
- appsApi.readNamespacedDeployment.mockImplementation(async () => {
- deploymentReads += 1
- if (deploymentReads === 2) {
- current = {
- ...host,
- generation: 4,
- spec: { ...host.spec, contextRef: 'replacement-context' },
- }
- }
- return { metadata: { resourceVersion: `rv-existing-${deploymentReads}` } }
- })
- appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
- const random = vi.spyOn(Math, 'random').mockReturnValue(0)
-
- try {
- await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
- } finally {
- random.mockRestore()
- }
-
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
- })
-
- it('resolves the latest authoritative Host object at reconcile admission', async () => {
- const reconciler = makeBasicReconciler()
- const stale = { ...makeHost({ name: 'latest-host' }), generation: 1 }
- const current = {
- ...stale,
- generation: 2,
- spec: { ...stale.spec, contextRef: 'current-context' },
- }
- const authority = { current: { known: true, generation: 7 } }
- wireAuthoritativeHost(reconciler, current, authority)
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
-
- await reconciler.reconcile(stale)
-
- expect(reconcileCore).toHaveBeenCalledOnce()
- expect(reconcileCore).toHaveBeenCalledWith(
- expect.objectContaining({
- generation: 2,
- spec: expect.objectContaining({ contextRef: 'current-context' }),
- }),
- expect.any(Function)
- )
- })
-
- it('does not patch channel-reader after Host authority changes during revision resolution', async () => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'channel-authority-race' }),
- uid: 'channel-authority-race-uid',
- generation: 3,
- }
- const authority = { current: { known: true, generation: 9 } }
- wireAuthoritativeHost(reconciler, host, authority)
- const revisionStarted = deferred()
- const releaseRevision = deferred<string>()
- vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(
- async () => {
- revisionStarted.resolve(undefined)
- return releaseRevision.promise
- }
- )
- const patchDeployment = vi
- .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
- .mockResolvedValue(undefined)
-
- const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)
- await revisionStarted.promise
- authority.current = { known: false, generation: 10 }
- releaseRevision.resolve('resolved-revision')
- await pending
-
- expect(patchDeployment).not.toHaveBeenCalled()
- })
-
- it.each([
- ['UID', (host: HostCRD) => ({ ...host, uid: 'replacement-channel-reader-uid' })],
- [
- 'spec generation',
- (host: HostCRD) => ({
- ...host,
- generation: (host.generation ?? 0) + 1,
- spec: { ...host.spec, contextRef: 'replacement-context' },
- }),
- ],
- ] as const)(
- 'does not patch channel-reader when the Host %s changes during revision resolution',
- async (_, replaceCurrent) => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'channel-host-race' }),
- uid: 'channel-host-race-uid',
- generation: 3,
- }
- let current: HostCRD = host
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 9 }))
- const revisionStarted = deferred()
- const releaseRevision = deferred<string>()
- vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockImplementation(
- async () => {
- revisionStarted.resolve(undefined)
- return releaseRevision.promise
- }
- )
- const patchDeployment = vi
- .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
- .mockResolvedValue(undefined)
-
- const pending = reconciler.patchChannelReaderRevisionAnnotation(host.name)
- await revisionStarted.promise
- current = replaceCurrent(host)
- releaseRevision.resolve('resolved-revision')
- await pending
-
- expect(patchDeployment).not.toHaveBeenCalled()
- }
- )
-
- it('serializes a fresh channel-reader revision behind an older in-flight Host reconcile', async () => {
- const reconciler = makeBasicReconciler()
- const host = {
- ...makeHost({ name: 'channel-revision-order' }),
- uid: 'channel-revision-order-uid',
- generation: 4,
- }
- const authority = { current: { known: true, generation: 9 } }
- wireAuthoritativeHost(reconciler, host, authority)
- const staleReconcileStarted = deferred()
- const releaseStaleReconcile = deferred()
- const order: string[] = []
- let liveRevision = ''
-
- vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async () => {
- order.push('reconcile-a:start')
- staleReconcileStarted.resolve(undefined)
- await releaseStaleReconcile.promise
- liveRevision = 'revision-a'
- order.push('reconcile-a:write')
- })
- vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue(
- 'revision-b'
- )
- const patchDeployment = vi
- .spyOn((reconciler as any).appsApi, 'patchNamespacedDeployment')
- .mockImplementation(async (request: any) => {
- const { body } = request
- liveRevision = body.spec.template.metadata.annotations['clerum.io/credentials-revision']
- order.push('revision-b:patch')
- })
-
- const staleReconcile = reconciler.reconcile(host)
- await staleReconcileStarted.promise
- const freshRevision = reconciler.patchChannelReaderRevisionAnnotation(host.name)
-
- // Give B the opportunity to patch before A is released. Without a shared
- // Host lane, B writes first and A deterministically overwrites it.
- await Promise.resolve()
- await Promise.resolve()
- releaseStaleReconcile.resolve(undefined)
- await Promise.all([staleReconcile, freshRevision])
-
- expect(liveRevision).toBe('revision-b')
- expect(patchDeployment).toHaveBeenCalledOnce()
- expect(order).toEqual(['reconcile-a:start', 'reconcile-a:write', 'revision-b:patch'])
- })
-
- it('rejects a queued delete when Host authority changes before admission', async () => {
- const reconciler = makeBasicReconciler()
- const host = makeHost({ name: 'delete-authority-race' })
- const authority = { current: { known: true, generation: 11 } }
- reconciler.setResolveCurrentHost(() => undefined)
- reconciler.setHostWatchAuthority(() => authority.current)
- const deleteRuntime = vi
- .spyOn(reconciler as any, 'deleteHostRuntimeResources')
- .mockResolvedValue(undefined)
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = (reconciler as any).lifecycle.serializeByHost(host.name, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }) as Promise<void>
- await blockerStarted.promise
-
- const pending = reconciler.reconcileDelete(host.name, host.namespace)
- authority.current = { known: false, generation: 12 }
- releaseBlocker.resolve(undefined)
- await occupied
-
- await expect(pending).rejects.toThrow(/Host inventory authority/)
- expect(deleteRuntime).not.toHaveBeenCalled()
- })
-})
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 9e2f236d5..5730e5553 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -1,7 +1,6 @@
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash, randomUUID } from 'crypto'
-import { isDeepStrictEqual } from 'node:util'
import * as path from 'path'
import type { AdministrativeOutcomeReporter } from './administrativeOutcomeReporter'
import { config } from './config'
@@ -44,7 +43,7 @@ import {
resolveStatelessImagePullPolicy,
} from './statelessDeployment'
import { EffectiveHostLifecycle, SuspendFromHeartbeatOutcome } from './statelessLifecycle.types'
-import { ReflectHostOutcomeFn, StatelessLifecycleExecutor } from './statelessLifecycleExecutor'
+import { StatelessLifecycleExecutor } from './statelessLifecycleExecutor'
import {
CommunicationChannelCRD,
HostCRD,
@@ -79,74 +78,6 @@ export type HostReconcileSource = 'urgent' | 'retry' | 'fleet'
/** Immutable Host watch authority snapshot captured at a full-pass boundary. */
export type HostWatchAuthoritySnapshot = { known: boolean; generation: number }
-/**
- * Immutable lease for every inventory that contributes to a Host mutation.
- * Host resources come from the Host watch; mounted SharedFileSystems come from
- * the Context watch. Keeping both generations prevents a false-stable
- * Host-only lease when Context retires and recovers during queued work.
- */
-export type HostMutationAuthoritySnapshot = {
- known: boolean
- hostGeneration: number
- contextGeneration: number
-}
-
-class HostInventoryAuthorityUnavailableError extends Error {
- constructor(
- action: string,
- captured: HostMutationAuthoritySnapshot,
- current: HostMutationAuthoritySnapshot
- ) {
- super(
- `Host inventory authority changed before ${action} admission ` +
- `(captured known=${captured.known} hostGeneration=${captured.hostGeneration} ` +
- `contextGeneration=${captured.contextGeneration}, current known=${current.known} ` +
- `hostGeneration=${current.hostGeneration} contextGeneration=${current.contextGeneration})`
- )
- this.name = 'HostInventoryAuthorityUnavailableError'
- }
-}
-
-class HostMutationIdentityChangedError extends Error {
- constructor(action: string, hostName: string) {
- super(`Host identity changed before ${action} admission for "${hostName}"`)
- this.name = 'HostMutationIdentityChangedError'
- }
-}
-
-class HostMutationSpecRevisionChangedError extends Error {
- constructor(action: string, hostName: string) {
- super(`Host spec generation changed before ${action} admission for "${hostName}"`)
- this.name = 'HostMutationSpecRevisionChangedError'
- }
-}
-
-class HostMutationDependencyChangedError extends Error {
- constructor(action: string, hostName: string) {
- super(`Host mutation dependency changed before ${action} admission for "${hostName}"`)
- this.name = 'HostMutationDependencyChangedError'
- }
-}
-
-/**
- * The four benign mid-pass supersession signals. revalidateHostMutationBoundary()
- * throws these when the watch observes a newer spec/uid/authority-generation while
- * a reconcile is in flight; a fresh reconcile runs afterwards, so the retired pass
- * is a WITHDRAWAL, not a hard failure. Matched by `error.name` (string), NOT
- * `instanceof`: the classes are unexported and statelessLifecycleExecutor.ts
- * synthesizes name-equivalent plain Errors for the same races.
- */
-const BENIGN_SUPERSESSION_ERROR_NAMES: ReadonlySet<string> = new Set([
- 'HostInventoryAuthorityUnavailableError',
- 'HostMutationIdentityChangedError',
- 'HostMutationSpecRevisionChangedError',
- 'HostMutationDependencyChangedError',
-])
-
-function isBenignSupersessionError(error: unknown): boolean {
- return error instanceof Error && BENIGN_SUPERSESSION_ERROR_NAMES.has(error.name)
-}
-
/**
* The exact §13.2 destructive-cleanup predicate. Deletion of an orphan Host's
* owned bundle is permitted ONLY when watch authority is known, the watch
@@ -447,12 +378,6 @@ export class HostReconciler {
* snapshot for reconcile, and cleanup stays fail-closed (authority unknown).
*/
private resolveCurrentHost: ((name: string) => HostCRD | undefined) | null = null
- /**
- * H2: late-bound cache reflector. Wired by McpServerWatcher next to
- * setResolveCurrentHost; null (standalone) means lifecycle outcomes reflect
- * only onto the caller's own object (legacy contract).
- */
- private reflectHostOutcome: ReflectHostOutcomeFn | null = null
/**
* Snapshot the current Host watch authority + generation. Wired by
* McpServerWatcher. Default is fail-closed (unknown) so orphan cleanup never
@@ -462,27 +387,6 @@ export class HostReconciler {
known: false,
generation: 0,
})
- /**
- * Positive Host mutations depend on Host plus Context authority. The default
- * preserves the standalone HostReconciler contract by deriving a Host-only
- * lease; McpServerWatcher wires the complete composite lease.
- */
- private hostMutationAuthority: () => HostMutationAuthoritySnapshot = () => {
- const host = this.hostWatchAuthority()
- return {
- known: host.known,
- hostGeneration: host.generation,
- contextGeneration: 0,
- }
- }
- /**
- * Exact revisions of the cross-resource inputs selected by a Host. Watch
- * generations fence LIST -> WATCH authority, but normal MODIFIED events stay
- * in the same generation. The watcher supplies the selected Context,
- * referenced SharedFileSystems plus mountability, and Host-scoped
- * CommunicationChannels so unrelated fleet changes do not retire this lease.
- */
- private resolveHostMutationDependencies: ((host: HostCRD) => readonly unknown[]) | null = null
constructor(kc: k8s.KubeConfig, deps?: HostReconcilerDeps) {
this.appsApi =
@@ -516,10 +420,7 @@ export class HostReconciler {
now: this.now,
countCommunicationChannels: hostName => this.countCommunicationChannels(hostName),
isCommunicationChannelCacheSynced: () => this.ccCacheSyncedFn(),
- reconcileCore: (host, revalidate) => this.reconcileCore(host, revalidate),
- prepareHostMutationAdmission: (action, host) =>
- this.prepareHostMutationAdmission(action, host),
- reflectHostOutcome: (name, uid, apply) => this.reflectHostOutcome?.(name, uid, apply),
+ reconcileCore: host => this.reconcileCore(host),
onLifecycleStatusCommitted: (host, lifecycle) => {
const occurredAt = this.now().toISOString()
this.infrastructureTelemetryReporter?.enqueueHealthTransition({
@@ -585,15 +486,6 @@ export class HostReconciler {
this.resolveCurrentHost = fn
}
- /**
- * H2: late-bound setter for the guarded cache reflector. Wired by
- * McpServerWatcher from its `hosts` cache (uid-guarded apply). Pattern
- * mirrors setResolveCurrentHost.
- */
- setReflectHostOutcome(fn: ReflectHostOutcomeFn): void {
- this.reflectHostOutcome = fn
- }
-
/**
* Late-bound setter for the Host watch authority/generation snapshot. Wired by
* McpServerWatcher so orphan cleanup can require known authority and a stable
@@ -603,103 +495,6 @@ export class HostReconciler {
this.hostWatchAuthority = fn
}
- setHostMutationAuthority(fn: () => HostMutationAuthoritySnapshot): void {
- this.hostMutationAuthority = fn
- }
-
- setResolveHostMutationDependencies(fn: (host: HostCRD) => readonly unknown[]): void {
- this.resolveHostMutationDependencies = fn
- }
-
- /**
- * A standalone HostReconciler receives an explicit Host object and has no
- * watch-backed cache to fence. Once a live-cache resolver is wired (the
- * production McpServerWatcher path), every mutation captures and revalidates
- * Host LIST -> WATCH authority at its actual serializer admission boundary.
- */
- private captureHostMutationAuthority(): HostMutationAuthoritySnapshot | null {
- return this.resolveCurrentHost === null ? null : this.hostMutationAuthority()
- }
-
- private requireHostMutationAuthority(
- action: string,
- captured: HostMutationAuthoritySnapshot | null
- ): void {
- if (captured === null) return
- const current = this.hostMutationAuthority()
- if (
- !captured.known ||
- !current.known ||
- captured.hostGeneration !== current.hostGeneration ||
- captured.contextGeneration !== current.contextGeneration
- ) {
- throw new HostInventoryAuthorityUnavailableError(action, captured, current)
- }
- }
-
- /**
- * Heartbeat writers own their serializer inside StatelessLifecycleExecutor.
- * Capture authority here at dispatch, then let the returned closure re-check
- * it and resolve the current same-UID Host when that serializer admits the
- * work. This preserves one queue per Host without a nested-lock deadlock.
- */
- private prepareHostMutationAdmission(action: string, requested: HostCRD): () => HostCRD {
- const capturedAuthority = this.captureHostMutationAuthority()
- const capturedDependencies = this.resolveHostMutationDependencies?.(requested)
- return this.makeHostMutationAdmission(
- action,
- requested,
- capturedAuthority,
- capturedDependencies
- )
- }
-
- private makeHostMutationAdmission(
- action: string,
- requested: HostCRD,
- capturedAuthority: HostMutationAuthoritySnapshot | null,
- capturedDependencies: readonly unknown[] | undefined
- ): () => HostCRD {
- return () => {
- this.requireHostMutationAuthority(action, capturedAuthority)
- if (this.resolveCurrentHost === null) return requested
- const current = this.resolveCurrentHost(requested.name)
- if (
- current === undefined ||
- requested.uid === undefined ||
- current.uid === undefined ||
- requested.uid !== current.uid
- ) {
- throw new HostMutationIdentityChangedError(action, requested.name)
- }
- if (
- requested.generation === undefined ||
- current.generation === undefined ||
- requested.generation !== current.generation
- ) {
- throw new HostMutationSpecRevisionChangedError(action, requested.name)
- }
- if (this.resolveHostMutationDependencies !== null) {
- const currentDependencies = this.resolveHostMutationDependencies(current)
- const sameDependencies =
- capturedDependencies !== undefined &&
- capturedDependencies.length === currentDependencies.length &&
- capturedDependencies.every((value, index) =>
- isDeepStrictEqual(value, currentDependencies[index])
- )
- if (!sameDependencies) {
- throw new HostMutationDependencyChangedError(action, requested.name)
- }
- }
- // McpServerWatcher owns the canonical LIST/WATCH cache object. Mutation
- // bodies historically reflect status into their admitted object for
- // same-pass decisions, so detach the working snapshot whenever a live
- // cache resolver is wired. Standalone callers without a resolver retain
- // the legacy caller-visible reflection contract.
- return structuredClone(current)
- }
- }
-
/**
* Late-bound setter for the host-ref lookup used by
* `patchChannelReaderRevisionAnnotation` to discover Secret(s) referenced
@@ -2297,52 +2092,29 @@ export class HostReconciler {
* Host CRD is present).
*/
async patchChannelReaderRevisionAnnotation(hostName: string): Promise<void> {
- const capturedAuthority = this.captureHostMutationAuthority()
const depName = `channel-reader-${hostName}`
try {
- await this.lifecycle.serializeByHost(hostName, async () => {
- const action = `channel-reader "${hostName}" revision patch`
- this.requireHostMutationAuthority(action, capturedAuthority)
-
- const resolveCurrentHost = this.resolveCurrentHost
- const admittedHost = resolveCurrentHost?.(hostName)
- if (resolveCurrentHost !== null && admittedHost === undefined) return
- const revalidate = admittedHost
- ? this.makeHostMutationAdmission(
- action,
- admittedHost,
- capturedAuthority,
- this.resolveHostMutationDependencies?.(admittedHost)
- )
- : () => this.requireHostMutationAuthority(action, capturedAuthority)
-
- revalidate()
- const revision = await this.computeChannelReaderRevisionForHost(hostName)
- // The revision calculation can await multiple Secret reads. Revalidate
- // Host LIST/WATCH authority plus UID and generation immediately before
- // entering the Kubernetes mutation, while still holding this Host lane.
- revalidate()
- const patchBody = {
- spec: {
- template: {
- metadata: {
- annotations: { 'clerum.io/credentials-revision': revision },
- },
+ const revision = await this.computeChannelReaderRevisionForHost(hostName)
+ const patchBody = {
+ spec: {
+ template: {
+ metadata: {
+ annotations: { 'clerum.io/credentials-revision': revision },
},
},
+ },
+ }
+ await this.appsApi.patchNamespacedDeployment(
+ { name: depName, namespace: config.channelsNamespace, body: patchBody },
+ {
+ middleware: [
+ k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
+ ],
}
- await this.appsApi.patchNamespacedDeployment(
- { name: depName, namespace: config.channelsNamespace, body: patchBody },
- {
- middleware: [
- k8s.setHeaderMiddleware('Content-Type', 'application/strategic-merge-patch+json'),
- ],
- }
- )
- console.log(
- `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`
- )
- })
+ )
+ console.log(
+ `[HostReconciler] Patched ${depName} credentials-revision=${revision || '(empty)'}`
+ )
} catch (err) {
if (getErrorCode(err) === 404) return
console.error(`[HostReconciler] Failed to reconcile ${depName} revision:`, err)
@@ -3777,37 +3549,16 @@ export class HostReconciler {
async reconcile(host: HostCRD, source: HostReconcileSource = 'urgent'): Promise<void> {
const dispatchedAt = Date.now()
- const capturedAuthority = this.captureHostMutationAuthority()
- const capturedDependencies = this.resolveHostMutationDependencies?.(host)
return this.lifecycle.serializeByHost(host.name, async () => {
- this.requireHostMutationAuthority(`Host "${host.name}" reconcile`, capturedAuthority)
- const admittedHost = this.resolveCurrentHost ? this.resolveCurrentHost(host.name) : host
- if (!admittedHost) return
- const revalidate = this.makeHostMutationAdmission(
- `Host "${host.name}" reconcile`,
- admittedHost,
- capturedAuthority,
- capturedDependencies
- )
const admittedAt = Date.now()
hostReconcileInFlight.inc({ lane: source })
try {
- await this.reconcileCore(admittedHost, revalidate)
- this.enqueueReconcileOutcome(admittedHost)
+ await this.reconcileCore(host)
+ this.enqueueReconcileOutcome(host)
this.observeReconcileLatency(source, 'success', dispatchedAt, admittedAt)
} catch (error) {
- if (isBenignSupersessionError(error)) {
- // A newer spec/uid/authority-generation superseded this pass mid-flight;
- // a fresh reconcile runs after. Model it as a withdrawal (superseded),
- // NOT a hard failure — do NOT emit controller_error/reconcile_exception
- // or an administrative 'failed' outcome. Callers already treat the
- // rethrow as a retire (reconcileDelete → 'superseded', watch callers
- // only console.error), so the rethrow is preserved.
- this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)
- throw error
- }
- this.enqueueControllerError(admittedHost, 'reconcile_exception', error)
- this.enqueueReconcileOutcome(admittedHost)
+ this.enqueueControllerError(host, 'reconcile_exception', error)
+ this.enqueueReconcileOutcome(host)
this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)
throw error
} finally {
@@ -3818,7 +3569,7 @@ export class HostReconciler {
private observeReconcileLatency(
source: HostReconcileSource,
- outcome: 'success' | 'error' | 'superseded',
+ outcome: 'success' | 'error',
dispatchedAt: number,
admittedAt: number
): void {
@@ -3832,19 +3583,7 @@ export class HostReconciler {
)
}
- private async reconcileCore(host: HostCRD, revalidate?: () => HostCRD): Promise<void> {
- // A per-Host serializer orders work, but it does not invalidate an older
- // reconcile when the watch cache observes a newer spec while this body is
- // awaiting Kubernetes or issuer I/O. Re-check the captured authority,
- // identity, and spec generation at every material mutation boundary.
- // Individual API writes remain atomic at the Kubernetes resource level;
- // these checks prevent subsequent stale writes from spanning an observed
- // generation change.
- const revalidateHostMutationBoundary = (): void => {
- revalidate?.()
- }
-
- revalidateHostMutationBoundary()
+ private async reconcileCore(host: HostCRD): Promise<void> {
// Wake fast-path (Stage 4.3) BEFORE the heavy reconcile body: reconciles
// are serialized PER HOST (serializeByHost), so a pending wake for THIS
// Host must not wait behind token issuance, NetworkPolicies or the
@@ -3853,7 +3592,6 @@ export class HostReconciler {
// funnels through this same method, so a watch event dropped on disconnect
// is recovered here.
const forceFreshForWake = (await this.lifecycle.handleWakeFastPath(host)) === true
- revalidateHostMutationBoundary()
// Track whether this host has desktop enabled
const isDesktop = !!(host.spec.desktop?.browser || host.spec.desktop?.x11)
@@ -3864,18 +3602,11 @@ export class HostReconciler {
}
const secretResult = await this.validateHostSecret(host)
- revalidateHostMutationBoundary()
if (!secretResult.ok) {
if (
secretResult.reason === 'SecretNotFound' ||
secretResult.reason === 'SecretAccessDenied'
) {
- // This branch deletes every ephemeral runtime resource. The Secret
- // decision may have blocked on the API while Host watch authority or
- // the Host UID/spec generation changed. Revalidate at the destructive
- // commit boundary so a stale secretRef can never tear down the current
- // runtime; the recovered/current Host reconcile will decide again.
- revalidateHostMutationBoundary()
await this.deleteHostRuntimeResources(host.name, host.namespace, {
deleteWorkspacePvc: false,
})
@@ -3896,11 +3627,8 @@ export class HostReconciler {
// Per-Host SA + Role + RoleBinding must exist BEFORE the Deployment,
// otherwise the kubelet can't mount the SA token and the pod will
// crash-loop until RBAC is created.
- revalidateHostMutationBoundary()
await this.ensureHostServiceAccount(host)
- revalidateHostMutationBoundary()
await this.ensureHostRole(host)
- revalidateHostMutationBoundary()
await this.ensureHostRoleBinding(host)
let mounts: ResolvedSfsMount[] = []
@@ -3913,12 +3641,10 @@ export class HostReconciler {
// will inject them.
console.error(`[HostReconciler] Failed to resolve context mounts for "${host.name}":`, err)
}
- revalidateHostMutationBoundary()
// Stateless lifecycle (Stage 2): assess enable/reject and persist the
// durable state to the Host status subresource BEFORE building the
// Deployment, so replicas derive from the same assessment.
let lifecycle = await this.lifecycle.assessLifecycle(host, mounts)
- revalidateHostMutationBoundary()
// Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
// suspension IMMEDIATELY before replicas:0 derives from this assessment
// — see StatelessLifecycleExecutor.resolveWakeBeforeScaleDown. A null
@@ -3926,17 +3652,14 @@ export class HostReconciler {
// pass (the periodic resync retries) rather than scaling to 0 on stale
// data.
const guardedLifecycle = await this.lifecycle.resolveWakeBeforeScaleDown(host, lifecycle)
- revalidateHostMutationBoundary()
if (guardedLifecycle === null) {
return
}
lifecycle = guardedLifecycle
- revalidateHostMutationBoundary()
const lifecycleStatusCommitted = await this.lifecycle.writeLifecycleStatusToCluster(
host,
lifecycle
)
- revalidateHostMutationBoundary()
if (lifecycleStatusCommitted) {
this.enqueueHostTelemetry(host, 'lifecycle_transition', lifecycle.effective.state, {
status: 'committed',
@@ -3947,9 +3670,7 @@ export class HostReconciler {
})
}
- revalidateHostMutationBoundary()
await this.ensurePvc(host)
- revalidateHostMutationBoundary()
await this.ensureService(host)
// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and
@@ -3965,31 +3686,23 @@ export class HostReconciler {
// surface it via HostRuntimeStatus instead of swallowing it.
const npFailures: string[] = []
try {
- revalidateHostMutationBoundary()
await this.ensureMcpHostIngressNetworkPolicy(host)
- revalidateHostMutationBoundary()
await this.ensureMcpHostGfsEgressNetworkPolicy(host)
- revalidateHostMutationBoundary()
await this.ensureWorkflowApprovalReaderMcpHostIngressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure mcp-host NP for "${host.name}":`, err)
npFailures.push(`mcp-host NP: ${(err as Error).message}`)
}
try {
- revalidateHostMutationBoundary()
await this.ensureRpcProxyMcpHostIngressNetworkPolicy(host)
- revalidateHostMutationBoundary()
await this.ensureRpcProxyHostEgressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure rpc-proxy host NP for "${host.name}":`, err)
npFailures.push(`rpc-proxy NP: ${(err as Error).message}`)
}
- revalidateHostMutationBoundary()
await this.ensureDesktopNetworkPolicy(host)
try {
- revalidateHostMutationBoundary()
await this.ensureChannelReaderEgressNetworkPolicy(host)
- revalidateHostMutationBoundary()
await this.ensureWorkflowApprovalReaderHostEgressNetworkPolicy(host)
} catch (err) {
console.error(`[HostReconciler] Failed to ensure channels egress NP for "${host.name}":`, err)
@@ -4004,10 +3717,8 @@ export class HostReconciler {
)
if (bootstrapLifecycle !== lifecycle) {
lifecycle = bootstrapLifecycle
- revalidateHostMutationBoundary()
await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
}
- revalidateHostMutationBoundary()
// Bootstrap captures the scope contract used for issuance. The Deployment
// guard below compares that contract with the live channel cache after this
@@ -4016,28 +3727,23 @@ export class HostReconciler {
forceFreshForWake,
targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
- revalidateHostMutationBoundary()
// replaceWithConflictRetry may wait and re-read after a 409, so each body is
// rebuilt from a stable lifecycle plus token-scope pair before mutation.
const resolveDeploymentLifecycle = async (): Promise<EffectiveHostLifecycle> => {
- revalidateHostMutationBoundary()
const deploymentLifecycle = this.lifecycle.enforceCommunicationChannelPolicyBeforeDeployment(
host.name,
lifecycle
)
if (deploymentLifecycle !== lifecycle) {
lifecycle = deploymentLifecycle
- revalidateHostMutationBoundary()
await this.lifecycle.writeLifecycleStatusToCluster(host, lifecycle)
}
- revalidateHostMutationBoundary()
return lifecycle.effective
}
const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {
for (let attempt = 1; attempt <= 3; attempt++) {
- revalidateHostMutationBoundary()
const currentScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
this.hasChannelIngress(host)
@@ -4049,7 +3755,6 @@ export class HostReconciler {
targetSuspended:
lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
- revalidateHostMutationBoundary()
const postProvisionScopeHash = HostReconciler.runtimeTokenScopeHash(
host,
this.hasChannelIngress(host)
@@ -4069,7 +3774,6 @@ export class HostReconciler {
const resolveDeploymentState = async (): Promise<DeploymentMutationState> => {
for (let attempt = 1; attempt <= 3; attempt++) {
- revalidateHostMutationBoundary()
await resolveDeploymentLifecycle()
await ensureCurrentRuntimeTokenScope()
const effective = await resolveDeploymentLifecycle()
@@ -4078,7 +3782,6 @@ export class HostReconciler {
this.hasChannelIngress(host)
)
if (runtimeTokenProvision.scopeHash === currentScopeHash) {
- revalidateHostMutationBoundary()
return {
lifecycle: effective,
runtimeTokenRevision: runtimeTokenProvision.revision,
@@ -4095,7 +3798,6 @@ export class HostReconciler {
)
}
- revalidateHostMutationBoundary()
await this.ensureDeployment(
host,
mounts,
@@ -4103,7 +3805,6 @@ export class HostReconciler {
lifecycle.effective,
resolveDeploymentState
)
- revalidateHostMutationBoundary()
const suspended = lifecycle.effective.stateless && lifecycle.effective.state === 'suspended'
if (suspended) {
@@ -4112,7 +3813,6 @@ export class HostReconciler {
this.lifecycle.markHostNotSuspended(host.name)
}
const ready = suspended ? false : await this.checkDeploymentReady(host.name, host.namespace)
- revalidateHostMutationBoundary()
// If any per-Host NP failed to apply, mark the host as degraded:
// deployed remains true (the pod is in place), but ready=false +
// a message that names the missing security boundary. Operators see
@@ -4148,7 +3848,6 @@ export class HostReconciler {
() => this.reconcileChannelReaderDeployment(host),
]) {
try {
- revalidateHostMutationBoundary()
await reconcileResource()
} catch (err) {
channelReaderFailures.push(err)
@@ -4169,7 +3868,6 @@ export class HostReconciler {
// Best-effort — checkChannelReaderStatus never throws.
const ccCount = this.countCommunicationChannels(host.name)
const channelReaderStatus = await this.checkChannelReaderStatus(host.name, ccCount)
- revalidateHostMutationBoundary()
if (channelReaderFailures.length > 0) {
// Overlay the reconcile error onto the status message so both signals
// are visible (the deploy error may explain why the Deployment is missing).
@@ -4204,21 +3902,18 @@ export class HostReconciler {
* that path, so cache presence at admission is the correct fence. This
* mirrors the F2/#827 in-serializer re-check in collectHostCleanupFailures.
*
- * Callers that omit `opts` drop only the extra `skipIf` term; the
- * resolveCurrentHost supersede fence still gates the delete.
+ * Callers that omit `opts` keep the previous unconditional behavior exactly.
*/
async reconcileDelete(
name: string,
namespace: string,
opts?: { skipIf?: () => boolean }
): Promise<void> {
- const capturedAuthority = this.captureHostMutationAuthority()
// A DELETED event must not race an in-flight create/update for the same
// Host: route it through the same per-Host serializer as reconcile(). The
// global convergence tail no longer provides this ordering.
await this.lifecycle.serializeByHost(name, async () => {
- this.requireHostMutationAuthority(`Host "${name}" delete`, capturedAuthority)
- if (this.resolveCurrentHost?.(name) !== undefined || opts?.skipIf?.()) {
+ if (opts?.skipIf?.()) {
// Same vocabulary the watch path already reports for this condition
// (reconcileHostWatchEvent), so the skip is observable, not silent.
hostDeleteCleanupTotal.inc({ outcome: 'superseded' })
@@ -4365,15 +4060,6 @@ export class HostReconciler {
}
try {
await this.lifecycle.serializeByHost(hostName, async () => {
- // The fresh 404 and the cache omission are only destructive
- // authority while the same LIST -> WATCH generation remains live.
- // Revalidate inside the serializer: queued Host work can otherwise
- // outlive that generation and delete a same-name recreation whose
- // ADDED event has not reached the retired cache.
- if (!authorityValid()) {
- hostCleanupDeferredTotal.inc({ reason: 'watch_lost' })
- return
- }
// F2 / #827 TOCTOU: a same-name Host recreated between the fresh read
// above and admission into this serializer updates the current cache
// (its ADDED event sets the cache before its reconcile runs, and that
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index e45095c41..2aa0ccea4 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -1,19 +1,13 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import * as http from 'http'
import { HostFleetReconcileError } from './hostReconciler'
import { HostK8sRequestTimeoutError } from './k8s/hostK8sApiClient'
-import { externalEgressResyncDelayMs } from './externalEgressConvergenceCoordinator'
import {
McpServerWatcher,
- getContext,
+ externalEgressResyncDelayMs,
listAllCommunicationChannels,
- listAllGlobalFileSystems,
listAllHosts,
- listAllSharedFileSystems,
} from './k8sClient'
-import { registry } from './metrics'
-import { ContextMapperServer } from './server'
-import type { HostCRD, McpServerCRD } from './types'
+import type { HostCRD } from './types'
function deferred<T = void>(): {
promise: Promise<T>
@@ -29,116 +23,19 @@ function deferred<T = void>(): {
return { promise, resolve, reject }
}
-async function flushMicrotasks(turns = 8): Promise<void> {
- for (let turn = 0; turn < turns; turn += 1) {
- await Promise.resolve()
- }
-}
-
-async function readInitialConvergenceMetric(
- name:
- | 'clerum_hcc_initial_convergence_retries_total'
- | 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
- lane: 'McpServer' | 'NetworkPolicy'
-): Promise<number> {
- const metric = registry.getSingleMetric(name)
- if (!metric) throw new Error(`${name} is not registered`)
- const snapshot = await metric.get()
- return snapshot.values.find(entry => entry.labels.lane === lane)?.value ?? 0
-}
-
-async function requestReadyOverHttp(server: ContextMapperServer): Promise<{
- statusCode: number | undefined
- body: string
-}> {
- const listener = (server as unknown as { server: http.Server | null }).server
- const address = listener?.address()
- if (!address || typeof address === 'string') {
- throw new Error('HCC readiness listener did not bind a TCP port')
- }
-
- return new Promise((resolve, reject) => {
- const request = http.get(
- { host: '127.0.0.1', port: address.port, path: '/ready' },
- response => {
- let body = ''
- response.setEncoding('utf8')
- response.on('data', chunk => {
- body += chunk
- })
- response.on('end', () => resolve({ statusCode: response.statusCode, body }))
- }
- )
- request.once('error', reject)
- })
-}
-
-function stubAuthoritativeInventoryWatch(
- watcher: McpServerWatcher,
- kind: 'McpServer' | 'Context',
- effect?: () => void | Promise<void>
-) {
- const method = kind === 'McpServer' ? 'startMcpServerWatch' : 'startContextWatch'
- const generationField = kind === 'McpServer' ? 'mcpWatchGeneration' : 'contextWatchGeneration'
- const requestField = kind === 'McpServer' ? 'mcpWatchRequest' : 'ctxWatchRequest'
- return vi.spyOn(watcher as any, method).mockImplementation(async () => {
- const generation = ((watcher as any)[generationField] as number) + 1
- ;(watcher as any)[generationField] = generation
- ;(watcher as any)[requestField] = { abort: vi.fn() }
- await effect?.()
- return generation
- })
-}
-
-function markHostInventoryAuthoritative(watcher: McpServerWatcher): void {
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
-}
-
-function markMcpServerInventoryAuthoritative(watcher: McpServerWatcher): void {
- ;(watcher as any).mcpServerCacheSynced = true
-}
-
-function markNetworkPolicyRevocationAuthoritative(watcher: McpServerWatcher): void {
- ;(watcher as any).networkPolicyRevocationContextGeneration = (
- watcher as any
- ).contextWatchGeneration
- ;(watcher as any).networkPolicyRevocationServerGeneration = (watcher as any).mcpWatchGeneration
- ;(watcher as any).networkPolicyRevocationContextRevision = (watcher as any).contextDesiredRevision
- ;(watcher as any).networkPolicyRevocationServerRevision = (
- watcher as any
- ).mcpServerDesiredRevision
-}
-
-function newContextAuthoritativeWatcher(): McpServerWatcher {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- return watcher
-}
-
const mocks = vi.hoisted(() => {
const listNamespacedCustomObject = vi.fn()
- const getNamespacedCustomObject = vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))
const ensureDefaultPolicies = vi.fn().mockResolvedValue(undefined)
- const hasCertifiedSafetyInventory = vi.fn().mockReturnValue(true)
- const netPolFullReconcile = vi.fn().mockImplementation(async (...args: unknown[]) => {
- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
- options?.onAuthoritativeRevocationComplete?.()
- })
+ const netPolFullReconcile = vi.fn().mockResolvedValue(undefined)
const serverFullReconcile = vi.fn().mockResolvedValue(undefined)
const hostFullReconcile = vi.fn().mockResolvedValue(undefined)
const hostReconcileHosts = vi.fn().mockResolvedValue(undefined)
const sfsFullReconcile = vi.fn().mockResolvedValue(undefined)
const watch = vi.fn().mockResolvedValue({ abort: vi.fn() })
const hostListCallOptions = vi.fn()
- const sharedFileSystemListCallOptions = vi.fn()
const createAdministrativeOutcomeReporter = vi.fn().mockReturnValue(undefined)
return {
- hasCertifiedSafetyInventory,
listNamespacedCustomObject,
- getNamespacedCustomObject,
ensureDefaultPolicies,
netPolFullReconcile,
serverFullReconcile,
@@ -147,15 +44,12 @@ const mocks = vi.hoisted(() => {
sfsFullReconcile,
watch,
hostListCallOptions,
- sharedFileSystemListCallOptions,
createAdministrativeOutcomeReporter,
}
})
-// Kept at externalEgressResyncIntervalSec: 0 in every test (reset in afterEach):
-// the external-egress periodic resync timer now lives in the convergence
-// coordinator and is unit-tested in externalEgressConvergenceCoordinator.test.ts,
-// so the k8sClient integration tests leave it disabled to avoid a stray timer.
+// Mutable so a test can enable the external-egress resync timer (R1-M4); every
+// other test keeps externalEgressResyncIntervalSec: 0 (reset in afterEach).
const mockConfig = vi.hoisted(() => ({
devMode: false,
namespace: 'mcp-server',
@@ -196,12 +90,9 @@ vi.mock('@kubernetes/client-node', () => {
return {
listNamespacedCustomObject: async (request: { plural?: string }, options?: unknown) => {
if (request.plural === 'hosts') mocks.hostListCallOptions(options)
- if (request.plural === 'sharedfilesystems')
- mocks.sharedFileSystemListCallOptions(options)
const response = await mocks.listNamespacedCustomObject(request)
if (
- request.plural &&
- ['hosts', 'mcpservers', 'contexts'].includes(request.plural) &&
+ request.plural === 'hosts' &&
response &&
typeof response === 'object' &&
!('metadata' in response)
@@ -211,17 +102,11 @@ vi.mock('@kubernetes/client-node', () => {
// production missing-resourceVersion failure path.
return {
...response,
- metadata: {
- resourceVersion:
- request.plural === 'hosts'
- ? 'test-host-collection-rv'
- : `test-${request.plural}-collection-rv`,
- },
+ metadata: { resourceVersion: 'test-host-collection-rv' },
}
}
return response
},
- getNamespacedCustomObject: mocks.getNamespacedCustomObject,
}
}
return {}
@@ -246,8 +131,6 @@ vi.mock('./reconciler', () => ({
reconcile = vi.fn()
reconcileDelete = vi.fn()
getStatus = vi.fn()
- setInventoryAuthority = vi.fn()
- setResolveCurrentServer = vi.fn()
},
}))
@@ -270,28 +153,6 @@ vi.mock('./hostReconciler', () => ({
// and the watch-authority snapshot into HostReconciler at construction.
setResolveCurrentHost = vi.fn()
setHostWatchAuthority = vi.fn()
- setHostMutationAuthority = vi.fn()
- // H2: capture the injected uid-guarded cache reflector so Test C can invoke it.
- _reflectHostOutcomeFn:
- | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
- | null = null
- setReflectHostOutcome = vi.fn().mockImplementation(function (
- this: {
- _reflectHostOutcomeFn:
- | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
- | null
- },
- fn: (name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void
- ) {
- this._reflectHostOutcomeFn = fn
- })
- _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null = null
- setResolveHostMutationDependencies = vi.fn().mockImplementation(function (
- this: { _hostMutationDependenciesFn: ((host: HostCRD) => readonly unknown[]) | null },
- fn: (host: HostCRD) => readonly unknown[]
- ) {
- this._hostMutationDependenciesFn = fn
- })
// Stores the injected callback so tests can invoke it directly.
_countCommunicationChannelsFn: ((host: string) => number) | null = null
setCountCommunicationChannels = vi.fn().mockImplementation(function (
@@ -311,26 +172,16 @@ vi.mock('./hostReconciler', () => ({
// B2: mock the CC cache-sync setter so McpServerWatcher constructor succeeds.
setIsCommunicationChannelCacheSynced = vi.fn()
patchChannelReaderRevisionAnnotation = vi.fn().mockResolvedValue(undefined)
- reconcileChannelReaderRevision = vi.fn().mockResolvedValue(undefined)
},
}))
vi.mock('./networkPolicyReconciler', () => ({
- sameContextDesiredRevision: (
- expected: { name: string; namespace: string; spec: unknown },
- current: { name: string; namespace: string; spec: unknown }
- ) =>
- expected.name === current.name &&
- expected.namespace === current.namespace &&
- JSON.stringify(expected.spec) === JSON.stringify(current.spec),
NetworkPolicyReconciler: class {
ensureDefaultPolicies = mocks.ensureDefaultPolicies
fullReconcile = mocks.netPolFullReconcile
reconcileExternalEgress = vi.fn()
reconcileContext = vi.fn()
- reconcileDeleteContext = vi.fn()
cleanupExternalEgress = vi.fn()
- hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory
},
}))
@@ -348,7 +199,6 @@ vi.mock('./sharedFileSystemReconciler', () => ({
reconcileDelete = vi.fn()
getStatus = vi.fn()
setListContextRefs = vi.fn()
- isMountable = vi.fn().mockReturnValue(false)
},
}))
@@ -363,7 +213,7 @@ describe('McpServerWatcher startup', () => {
mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'mcpservers') return { items: [] }
- if (plural === 'contexts') return { items: [] }
+ if (plural === 'contexts') throw new Error('context discovery failed')
if (plural === 'hosts') return { items: [] }
if (plural === 'sharedfilesystems') return { items: [] }
if (plural === 'communicationchannels')
@@ -374,24 +224,60 @@ describe('McpServerWatcher startup', () => {
afterEach(() => {
vi.useRealTimers()
- vi.restoreAllMocks()
- // Certified is the resting state. vi.clearAllMocks() in beforeEach clears
- // calls but not implementations, so a test that degrades this getter and
- // fails before its own restore would silently re-gate readiness for every
- // later test in the file. Restore it structurally rather than by
- // per-test discipline.
- mocks.hasCertifiedSafetyInventory.mockReset().mockReturnValue(true)
- mockConfig.externalEgressResyncIntervalSec = 0 // keep periodic resync off by default
+ mockConfig.externalEgressResyncIntervalSec = 0 // R1-M4: keep the timer off by default
mocks.ensureDefaultPolicies.mockReset().mockResolvedValue(undefined)
- mocks.netPolFullReconcile.mockReset().mockImplementation(async (...args: unknown[]) => {
- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
- options?.onAuthoritativeRevocationComplete?.()
- })
+ mocks.netPolFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.serverFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.sfsFullReconcile.mockReset().mockResolvedValue(undefined)
})
+ // R1-M4 (zach88): the self-rescheduling resync timer (only the pure delay
+ // helper was covered). Assert the behavioral invariants the comments promise:
+ // arms once, reschedules only AFTER a pass resolves, and stop() ends the chain.
+ it('R1-M4: external-egress resync timer arms once, reschedules after each pass, and stops', async () => {
+ vi.useFakeTimers()
+ mockConfig.externalEgressResyncIntervalSec = 60
+ const watcher = new McpServerWatcher()
+ for (const m of [
+ 'startMcpServerWatch',
+ 'startContextWatch',
+ 'startSharedFileSystemWatch',
+ 'startGlobalFileSystemWatch',
+ 'startCommunicationChannelWatch',
+ ]) {
+ vi.spyOn(watcher as any, m).mockImplementation(async () => undefined)
+ }
+ // Control each resync pass's completion so we can prove the reschedule waits.
+ let pending = deferred()
+ const runSpy = vi
+ .spyOn(watcher as any, 'runExternalEgressResync')
+ .mockImplementation(() => pending.promise)
+
+ await watcher.start()
+ expect(runSpy).not.toHaveBeenCalled() // not until the first delay elapses
+
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(runSpy).toHaveBeenCalledTimes(1) // armed once, fired once
+
+ // The NEXT pass must not be scheduled until this one resolves.
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(runSpy).toHaveBeenCalledTimes(1) // still 1 — no overlap while pending
+
+ pending.resolve()
+ await Promise.resolve()
+ pending = deferred()
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(runSpy).toHaveBeenCalledTimes(2) // rescheduled after the pass resolved
+
+ // stop() ends the chain — no further fires.
+ watcher.stop()
+ pending.resolve()
+ await Promise.resolve()
+ await vi.advanceTimersByTimeAsync(60_000 * 5)
+ expect(runSpy).toHaveBeenCalledTimes(2)
+ })
+
it('passes the governed tracing switch to the administrative reporter factory', () => {
new McpServerWatcher()
@@ -400,83 +286,35 @@ describe('McpServerWatcher startup', () => {
})
})
- it('ensures baseline policies before opening any inventory watch', async () => {
+ it('ensures default policies even when initial Context discovery fails', async () => {
const watcher = new McpServerWatcher()
+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ // #281 (Task 9): startCommunicationChannelWatch is wired into start() — stub to prevent the real watch from firing.
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
await watcher.start()
expect(mocks.ensureDefaultPolicies).toHaveBeenCalledTimes(1)
- expect(mocks.ensureDefaultPolicies.mock.invocationCallOrder[0]).toBeLessThan(
- mocks.watch.mock.invocationCallOrder[0]
- )
- await vi.waitFor(() =>
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {
- runEffect: expect.any(Function),
- })
- )
+ expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])
expect(mocks.hostFullReconcile).toHaveBeenCalledWith([])
expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([])
- await watcher.stop()
})
- it('loads the SharedFileSystem inventory before scheduling the first Host fleet pass', async () => {
- const sharedFileSystemsListed = deferred<{ items: unknown[] }>()
+ it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'sharedfilesystems') return sharedFileSystemsListed.promise
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'sfs-ordering-cc-rv' }, items: [] }
- }
+ if (plural === 'hosts') return { items: [] }
return { items: [] }
})
- const watcher = new McpServerWatcher()
- // Sampling "not yet called" after a fixed number of microtask turns cannot
- // distinguish "correctly waiting" from "scheduled but not drained yet".
- // Record what the first pass actually observed instead: the invariant is
- // that it never runs against an unpopulated SharedFileSystem cache, because
- // that is what writes mount-less templates and rerolls the fleet.
- let sfsCacheSizeAtFirstPass: number | undefined
- mocks.hostFullReconcile.mockImplementation(async () => {
- if (sfsCacheSizeAtFirstPass === undefined) {
- sfsCacheSizeAtFirstPass = (watcher as any).sharedFileSystems.size
- }
- })
- const start = watcher.start()
-
- try {
- // Await the whole startup first. That is the positive signal that the
- // cold-start Host recovery already reached the point where it decides
- // whether to schedule the fleet pass, so the absence assertion below
- // cannot pass merely because nothing has been drained yet.
- await start
- await flushMicrotasks()
- expect(mocks.hostFullReconcile).not.toHaveBeenCalled()
- sharedFileSystemsListed.resolve({
- items: [{ metadata: { name: 'initial-sfs', namespace: 'mcp-host' }, spec: {} }],
- })
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
- expect(sfsCacheSizeAtFirstPass).toBe(1)
- expect((watcher as any).sharedFileSystems.get('initial-sfs')).toEqual(
- expect.objectContaining({ name: 'initial-sfs' })
- )
- } finally {
- sharedFileSystemsListed.resolve({ items: [] })
- await start.catch(() => undefined)
- await watcher.stop()
- }
- })
-
- it('lists the SharedFileSystem inventory through the deadline-bearing client', async () => {
- // The cold-start Host fleet pass waits on this inventory. Bounding the wait
- // is not enough: an unbounded request stays hung, so the socket leaks and
- // `startSharedFileSystemWatch()` — which runs after this await — never
- // starts. Only the deadline-bearing client aborts the request itself, and
- // it is observable because it appends its deadline middleware to the call.
- mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })
-
- await listAllSharedFileSystems()
+ await expect(listAllHosts()).resolves.toEqual([])
- expect(mocks.sharedFileSystemListCallOptions).toHaveBeenCalledWith(
+ expect(mocks.hostListCallOptions).toHaveBeenCalledWith(
expect.objectContaining({
middleware: [expect.objectContaining({ pre: expect.any(Function) })],
middlewareMergeStrategy: 'append',
@@ -484,4142 +322,125 @@ describe('McpServerWatcher startup', () => {
)
})
- it('schedules the first Host fleet pass when the SharedFileSystem watch never starts', async () => {
- // `k8s.Watch` carries no transport deadline — client-node overwrites its own
- // AbortSignal.timeout with a bare controller signal — so awaiting the watch
- // start inside the inventory promise re-strands the cold-start fleet pass
- // that bounding the LIST was meant to protect.
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'sfs-watch-stall-cc-rv' }, items: [] }
- }
- return { items: [] }
- })
- mocks.watch.mockImplementation(async (path: string) => {
- if (path.endsWith('/sharedfilesystems')) return new Promise(() => undefined)
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- const start = watcher.start()
-
- try {
- await start
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
- } finally {
- await watcher.stop()
- await start.catch(() => undefined)
- }
- })
-
- it('schedules the first Host fleet pass when the SharedFileSystem inventory fails', async () => {
- // The inventory LIST runs on the deadline-bearing client, so an apiserver
- // that never answers is aborted and surfaces here as a rejection. The Host
- // fleet must still converge: stranding it would leave every Host
- // unreconciled behind an already-certified readiness.
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'sharedfilesystems') throw new Error('sharedfilesystems LIST aborted')
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'sfs-abort-cc-rv' }, items: [] }
- }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- const start = watcher.start()
-
- try {
- await start
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
- } finally {
- await watcher.stop()
- await start.catch(() => undefined)
- }
- })
-
- it('retains Context uid and generation from the authoritative initial snapshot', async () => {
+ it('preserves Host UID from the Kubernetes inventory boundary', async () => {
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'contexts') {
+ if (plural === 'hosts') {
return {
items: [
{
metadata: {
- name: 'identity-context',
- namespace: 'mcp-server',
- uid: 'context-uid-1',
- generation: 7,
+ name: 'uid-host',
+ namespace: 'mcp-host',
+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
+ generation: 4,
},
- spec: { contextId: 'identity-context', mcpServers: [] },
+ spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },
},
],
}
}
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'context-identity-cc-rv' }, items: [] }
- }
return { items: [] }
})
- const watcher = new McpServerWatcher()
-
- try {
- await watcher.start()
- expect((watcher as any).contexts.get('identity-context')).toEqual(
- expect.objectContaining({ uid: 'context-uid-1', generation: 7 })
- )
- } finally {
- await watcher.stop()
- }
- })
- it('fails safe bootstrap and keeps readiness unavailable when baseline NetworkPolicies fail', async () => {
- const policyError = new Error('baseline NetworkPolicies unavailable')
- mocks.ensureDefaultPolicies.mockRejectedValueOnce(policyError)
- const watcher = new McpServerWatcher()
- const server = new ContextMapperServer(watcher, 0)
- await server.start()
- const bootstrap = watcher.start().then(() => server.setReady(true))
-
- try {
- await expect(bootstrap).rejects.toThrow(policyError)
- const ready = await requestReadyOverHttp(server)
- expect(ready.statusCode).toBe(503)
- expect(JSON.parse(ready.body)).toEqual({ status: 'starting', ready: false })
- expect(mocks.watch).not.toHaveBeenCalled()
- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- } finally {
- await server.stop()
- await watcher.stop()
- }
+ await expect(listAllHosts()).resolves.toEqual([
+ expect.objectContaining({
+ name: 'uid-host',
+ uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
+ generation: 4,
+ }),
+ ])
})
- it.each(['LIST', 'WATCH'] as const)(
- 'stays live and recovers readiness after a transient Host %s bootstrap failure',
- async failureBoundary => {
- vi.useFakeTimers()
- let hostListAttempts = 0
- let hostWatchAttempts = 0
- mocks.listNamespacedCustomObject.mockImplementation(
- async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- hostListAttempts += 1
- if (failureBoundary === 'LIST' && hostListAttempts === 1) {
- throw new Error('host discovery temporarily unavailable')
- }
- return { metadata: { resourceVersion: 'host-recovery-rv' }, items: [] }
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
- return { items: [] }
- }
- )
- mocks.watch.mockImplementation(async path => {
- if (path.endsWith('/hosts')) {
- hostWatchAttempts += 1
- if (failureBoundary === 'WATCH' && hostWatchAttempts === 1) {
- throw new Error('host watch temporarily unavailable')
- }
- }
- return { abort: vi.fn() }
- })
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
-
- try {
- await expect(watcher.start()).resolves.toBeUndefined()
- server.setReady(true)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
-
- await vi.advanceTimersByTimeAsync(5000)
- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
-
- expect(hostListAttempts).toBe(2)
- expect(hostWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)
- expect((watcher as any).ccAppliedLifecycleGeneration).toBe(1)
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
- } finally {
- await server.stop()
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
- }
- )
-
- it.each(['LIST', 'WATCH'] as const)(
- 'stays live and recovers readiness after a transient McpServer %s bootstrap failure',
- async failureBoundary => {
- vi.useFakeTimers()
- let mcpServerListAttempts = 0
- let mcpServerWatchAttempts = 0
- mocks.listNamespacedCustomObject.mockImplementation(
- async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') {
- mcpServerListAttempts += 1
- if (failureBoundary === 'LIST' && mcpServerListAttempts === 1) {
- throw new Error('mcpserver discovery temporarily unavailable')
- }
- return { metadata: { resourceVersion: 'mcpserver-recovery-rv' }, items: [] }
- }
- if (plural === 'hosts') {
- return { metadata: { resourceVersion: 'host-rv' }, items: [] }
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
- return { items: [] }
- }
- )
- mocks.watch.mockImplementation(async path => {
- if (path.endsWith('/mcpservers')) {
- mcpServerWatchAttempts += 1
- if (failureBoundary === 'WATCH' && mcpServerWatchAttempts === 1) {
- throw new Error('mcpserver watch temporarily unavailable')
- }
- }
- return { abort: vi.fn() }
- })
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- // The McpServer lane runs for real (LIST + WATCH) so its bootstrap
- // failure and recovery exercise production code. Context stays
- // authoritative so readiness can be promoted once the lane recovers.
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
-
- try {
- await expect(watcher.start()).resolves.toBeUndefined()
- server.setReady(true)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- expect((watcher as any).mcpServerCacheRecoveryTimer).not.toBeNull()
-
- await vi.advanceTimersByTimeAsync(5000)
- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
-
- expect(mcpServerListAttempts).toBe(2)
- expect(mcpServerWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
- } finally {
- await server.stop()
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
- }
- )
-
- it.each(['LIST', 'WATCH'] as const)(
- 'stays live and recovers readiness after a transient Context %s bootstrap failure',
- async failureBoundary => {
- vi.useFakeTimers()
- let contextListAttempts = 0
- let contextWatchAttempts = 0
- mocks.listNamespacedCustomObject.mockImplementation(
- async ({ plural }: { plural: string }) => {
- if (plural === 'contexts') {
- contextListAttempts += 1
- if (failureBoundary === 'LIST' && contextListAttempts === 1) {
- throw new Error('context discovery temporarily unavailable')
- }
- return { metadata: { resourceVersion: 'context-recovery-rv' }, items: [] }
- }
- if (plural === 'hosts') {
- return { metadata: { resourceVersion: 'host-rv' }, items: [] }
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
- return { items: [] }
- }
- )
- mocks.watch.mockImplementation(async path => {
- if (path.endsWith('/contexts')) {
- contextWatchAttempts += 1
- if (failureBoundary === 'WATCH' && contextWatchAttempts === 1) {
- throw new Error('context watch temporarily unavailable')
- }
- }
- return { abort: vi.fn() }
- })
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- // The Context lane runs for real (LIST + WATCH) so its bootstrap
- // failure and recovery exercise production code. McpServer stays
- // authoritative so readiness can be promoted once the lane recovers.
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
-
- try {
- await expect(watcher.start()).resolves.toBeUndefined()
- server.setReady(true)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- expect((watcher as any).contextCacheRecoveryTimer).not.toBeNull()
-
- await vi.advanceTimersByTimeAsync(5000)
- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
-
- expect(contextListAttempts).toBe(2)
- expect(contextWatchAttempts).toBe(failureBoundary === 'LIST' ? 1 : 2)
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
- } finally {
- await server.stop()
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
+ it('reconciles startup external egress before runtime full reconciliation', async () => {
+ const eventLog: string[] = []
+ const server = {
+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
+ spec: {
+ contextRef: 'wf-research',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
+ },
}
- )
- it('cancels a pending Host bootstrap recovery when the watcher stops', async () => {
- vi.useFakeTimers()
- let hostListAttempts = 0
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- hostListAttempts += 1
- throw new Error('host inventory remains unavailable')
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: '1' }, items: [] }
return { items: [] }
})
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ mocks.ensureDefaultPolicies.mockImplementation(async () => {
+ eventLog.push('defaults')
+ })
+ mocks.serverFullReconcile.mockImplementation(async () => {
+ eventLog.push('runtime')
+ })
+
const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockImplementation(async () => {
+ eventLog.push('egress')
+ })
+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
- try {
- await watcher.start()
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
+ await watcher.start()
- await watcher.stop()
- await vi.advanceTimersByTimeAsync(5000)
+ expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([
+ expect.objectContaining({ name: 'web-search' }),
+ ])
- expect(hostListAttempts).toBe(1)
- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
- } finally {
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
+ watcher.stop()
})
- it('defers every Host-dependent effect while a bootstrap LIST lacks WATCH authority', async () => {
+ it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {
vi.useFakeTimers()
- const staleHost = {
- metadata: {
- name: 'stale-bootstrap-host',
- namespace: 'mcp-host',
- generation: 1,
- uid: 'stale-bootstrap-host-uid',
- },
+ const server = {
+ metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
spec: {
- host: 'stale-bootstrap-host',
- contextRef: 'bootstrap-context',
- secretRef: 'bootstrap-secret',
- model: { provider: 'openai', name: 'gpt-4o-mini' },
+ contextRef: 'wf-research',
+ image: 'clerum/web-search:test',
+ transport: { type: 'streamableHttp' as const, port: 3000 },
+ egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
},
}
+
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- return {
- metadata: { resourceVersion: 'stale-bootstrap-rv' },
- items: [staleHost],
- }
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
+ if (plural === 'mcpservers') return { items: [server] }
+ if (plural === 'contexts') return { items: [] }
+ if (plural === 'hosts') return { items: [] }
+ if (plural === 'sharedfilesystems') return { items: [] }
+ if (plural === 'communicationchannels')
+ return { metadata: { resourceVersion: '1' }, items: [] }
return { items: [] }
})
- mocks.watch.mockImplementation(async path => {
- if (path.endsWith('/hosts')) {
- throw new Error('host watch remains unavailable')
- }
- return { abort: vi.fn() }
- })
+
+ const watcher = new McpServerWatcher()
+ const netPol = (watcher as any).netPolReconciler
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('dns resolution failed'))
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- const hostReconciler = (watcher as any).hostReconciler
-
- try {
- await watcher.start()
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
-
- expect(watcher.getHost('stale-bootstrap-host')).toBeUndefined()
- const ccWatchCallback = (watcher as any).getCommunicationChannelWatchCallback()
- await ccWatchCallback('ADDED', {
- metadata: { name: 'bootstrap-channel', namespace: 'channels' },
- spec: { hostRef: 'stale-bootstrap-host' },
- })
- ;(watcher as any).contexts.set('bootstrap-context', {
- name: 'bootstrap-context',
- namespace: 'mcp-server',
- spec: {
- contextId: 'bootstrap-context',
- sharedFileSystems: [{ name: 'bootstrap-sfs', mountPath: '/shared' }],
- },
- })
- await (watcher as any).reconcileHostsReferencingSfs('bootstrap-sfs')
- await (watcher as any).reconcileHostsReferencingContext('bootstrap-context')
- ;(watcher as any).ccCacheSynced = false
- await (watcher as any).reconcileChannelReaderRevision('bootstrap-channel-secret', 'channels')
-
- expect(hostReconciler.reconcile).not.toHaveBeenCalled()
- expect(hostReconciler.patchChannelReaderRevisionAnnotation).not.toHaveBeenCalled()
- expect(hostReconciler.reconcileChannelReaderRevision).not.toHaveBeenCalled()
- expect(warnSpy).toHaveBeenCalledWith(
- '[K8s] Deferring CommunicationChannel Secret "channels/bootstrap-channel-secret" Host convergence; CommunicationChannel cache is not authoritative'
- )
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
- } finally {
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
- })
-
- it('defers Host fleet convergence until both Host and Context inventories are authoritative', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).contextCacheSynced = false
- ;(watcher as any).hosts.set('context-dependent-host', {
- name: 'context-dependent-host',
- namespace: 'mcp-host',
- spec: {
- host: 'context-dependent-host',
- contextRef: 'current-context',
- secretRef: 'host-secret',
- },
- })
-
- await (watcher as any).performHostFleetReconcileOnce({
- reason: 'Context authority test',
- mode: 'full',
- })
-
- expect(mocks.hostFullReconcile).not.toHaveBeenCalled()
- ;(watcher as any).contextCacheSynced = true
- await (watcher as any).performHostFleetReconcileOnce({
- reason: 'Context authority recovered',
- mode: 'full',
- })
-
- expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
- expect(mocks.hostFullReconcile).toHaveBeenCalledWith([
- expect.objectContaining({ name: 'context-dependent-host' }),
- ])
- await watcher.stop()
- })
-
- it('applies a pending CommunicationChannel lifecycle generation during Context recovery', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).contextCacheSynced = false
- const pendingGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
-
- await (watcher as any).recoverContextInventoryAndWatch()
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledOnce())
-
- expect((watcher as any).ccAppliedLifecycleGeneration).toBe(pendingGeneration)
- await watcher.stop()
- })
-
- it('starts McpServer and Context watches from their inventory resourceVersions', async () => {
- const watcher = new McpServerWatcher()
-
- await (watcher as any).startMcpServerWatch('mcp-inventory-rv')
- await (watcher as any).startContextWatch('context-inventory-rv')
-
- expect(mocks.watch).toHaveBeenNthCalledWith(
- 1,
- expect.stringContaining('/mcpservers'),
- { resourceVersion: 'mcp-inventory-rv' },
- expect.any(Function),
- expect.any(Function)
- )
- expect(mocks.watch).toHaveBeenNthCalledWith(
- 2,
- expect.stringContaining('/contexts'),
- { resourceVersion: 'context-inventory-rv' },
- expect.any(Function),
- expect.any(Function)
- )
- watcher.stop()
- })
-
- it.each([
- ['Context', 'startContextWatch', 'contextWatchGeneration'],
- ['Host', 'startHostWatch', 'hostWatchGeneration'],
- ] as const)(
- 'assigns a fresh generation to every replacement %s watch',
- async (_kind, method, generationField) => {
- const watcher = new McpServerWatcher()
- const before = (watcher as any)[generationField] as number
-
- const first = await (watcher as any)[method]('first-rv')
- const second = await (watcher as any)[method]('second-rv')
-
- expect(first).toBe(before + 1)
- expect(second).toBe(before + 2)
- expect((watcher as any)[generationField]).toBe(before + 2)
- await watcher.stop()
- }
- )
-
- it.each([
- [
- 'McpServer',
- 'restartMcpServerWatch',
- 'startMcpServerWatch',
- 'mcpWatchGeneration',
- 'mcpServerCacheSynced',
- { resourceVersion: 'mcp-rv', servers: [] },
- ],
- [
- 'Context',
- 'restartContextWatch',
- 'startContextWatch',
- 'contextWatchGeneration',
- 'contextCacheSynced',
- { resourceVersion: 'context-rv', contexts: [] },
- ],
- ] as const)(
- 'rejects a %s LIST snapshot that is not paired with a live watch request',
- async (_kind, restartMethod, startMethod, generationField, cacheField, snapshot) => {
- const watcher = new McpServerWatcher()
- vi.spyOn(watcher as any, startMethod).mockImplementation(async () => {
- ;(watcher as any)[generationField] += 1
- return (watcher as any)[generationField]
- })
-
- await expect((watcher as any)[restartMethod](snapshot)).rejects.toThrow(
- 'snapshot could not be paired with an active watch'
- )
- expect((watcher as any)[cacheField]).toBe(false)
- await watcher.stop()
- }
- )
-
- it.each([
- ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],
- ['Context', 'contexts', 'contextCacheRecoveryTimer'],
- ] as const)(
- 'stays live and arms in-process recovery when the %s inventory LIST is unavailable',
- async (kind, failingPlural, recoveryTimerField) => {
- mocks.listNamespacedCustomObject.mockImplementation(
- async ({ plural }: { plural: string }) => {
- if (plural === failingPlural) throw new Error(`${plural} inventory unavailable`)
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
- return { items: [] }
- }
- )
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
-
- try {
- await expect(watcher.start()).resolves.toBeUndefined()
-
- expect(errorSpy).toHaveBeenCalledWith(
- `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,
- expect.objectContaining({ message: `${failingPlural} inventory unavailable` })
- )
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect((watcher as any)[recoveryTimerField]).not.toBeNull()
- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- } finally {
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
- }
- )
-
- it.each([
- ['McpServer', 'mcpservers', 'mcpServerCacheRecoveryTimer'],
- ['Context', 'contexts', 'contextCacheRecoveryTimer'],
- ] as const)(
- 'stays live and arms in-process recovery when the %s collection resourceVersion is missing',
- async (kind, missingPlural, recoveryTimerField) => {
- mocks.listNamespacedCustomObject.mockImplementation(
- async ({ plural }: { plural: string }) => {
- if (plural === missingPlural) return { metadata: {}, items: [] }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
- return { items: [] }
- }
- )
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
-
- try {
- await expect(watcher.start()).resolves.toBeUndefined()
-
- expect(errorSpy).toHaveBeenCalledWith(
- `[K8s] Initial ${kind} inventory is unavailable; HCC remains unready while in-process recovery continues:`,
- expect.objectContaining({ message: `${kind} snapshot missing resourceVersion` })
- )
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect((watcher as any)[recoveryTimerField]).not.toBeNull()
- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- } finally {
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
- }
- )
-
- it('replays McpServer and Context changes from the inventory resourceVersion before convergence', async () => {
- const staleServer = {
- metadata: {
- name: 'gap-server',
- namespace: 'mcp-server',
- uid: 'gap-server-uid',
- generation: 1,
- },
- spec: {
- contextRef: 'default',
- image: 'clerum/gap-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const staleContext = {
- metadata: { name: 'gap-context', namespace: 'mcp-server' },
- spec: { contextId: 'gap-context', mcpServers: [] },
- }
- const currentContext = {
- metadata: { name: 'gap-context', namespace: 'mcp-server' },
- spec: { contextId: 'gap-context', mcpServers: ['current-server'] },
- }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') {
- return { metadata: { resourceVersion: 'opaque/mcp:101' }, items: [staleServer] }
- }
- if (plural === 'contexts') {
- return { metadata: { resourceVersion: 'opaque/context:202' }, items: [staleContext] }
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'cc-rv' }, items: [] }
- }
- return { items: [] }
- })
- mocks.watch.mockImplementation(async (path, options, callback) => {
- if (path.endsWith('/mcpservers')) {
- expect(options).toEqual({ resourceVersion: 'opaque/mcp:101' })
- await callback('DELETED', staleServer)
- }
- if (path.endsWith('/contexts')) {
- expect(options).toEqual({ resourceVersion: 'opaque/context:202' })
- await callback('MODIFIED', currentContext)
- }
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
-
- await watcher.start()
-
- await vi.waitFor(() => {
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([], {
- runEffect: expect.any(Function),
- })
- expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(
- [expect.objectContaining({ spec: currentContext.spec })],
- [],
- expect.objectContaining({
- serverInventoryComplete: true,
- ensureDefaults: false,
- contextInventoryAuthoritative: expect.any(Function),
- serverInventoryAuthoritative: expect.any(Function),
- contextDesiredRevision: expect.any(Function),
- serverDesiredRevision: expect.any(Function),
- })
- )
- })
- expect(watcher.getAllServers()).toEqual([])
- await watcher.stop()
- })
-
- it('recovers a closed McpServer watch through a fresh authoritative snapshot', async () => {
- vi.useFakeTimers()
- const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []
- const doneCallbacks: Array<(error: Error | null) => void> = []
- const watchQueries: unknown[] = []
- mocks.watch.mockImplementation(async (path, options, callback, done) => {
- if (path.endsWith('/mcpservers')) {
- callbacks.push(callback)
- doneCallbacks.push(done)
- watchQueries.push(options)
- }
- return { abort: vi.fn() }
- })
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') {
- return {
- metadata: { resourceVersion: 'mcp-recovery-rv' },
- items: [
- {
- metadata: {
- name: 'recovered-server',
- namespace: 'mcp-server',
- uid: 'recovered-uid',
- generation: 1,
- },
- spec: {
- contextRef: 'default',
- image: 'clerum/recovered:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- },
- ],
- }
- }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- await (watcher as any).restartMcpServerWatch({
- resourceVersion: 'mcp-start-rv',
- servers: [],
- })
- markNetworkPolicyRevocationAuthoritative(watcher)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
-
- doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- await vi.advanceTimersByTimeAsync(5000)
- await vi.waitFor(() => expect(watchQueries).toHaveLength(2))
-
- expect(watchQueries).toEqual([
- { resourceVersion: 'mcp-start-rv' },
- { resourceVersion: 'mcp-recovery-rv' },
- ])
- expect(watcher.getAllServers()).toEqual([expect.objectContaining({ name: 'recovered-server' })])
- expect(callbacks).toHaveLength(2)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- await watcher.stop()
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- })
-
- it('withdraws readiness when the safety pass leaves its inventory uncertified', async () => {
- // Losing a delete fence is the one doom cause that bumps no watch
- // generation and moves no desired revision, so every equality below stays
- // satisfied while an allow this pass classified as stale is still live.
- // The reconciler already degrades its fence at the point of loss; readiness
- // has to read it, or /ready keeps answering 200 over that allow until a
- // retry lands (>=5s, and indefinitely against a persistent second writer —
- // e.g. the two-replica overlap every rolling update opens, since the
- // Deployment carries no leader election).
- const watcher = newContextAuthoritativeWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
-
- try {
- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- } finally {
- // vi.clearAllMocks() clears calls, not implementations, so an escaped
- // mockReturnValue would silently re-gate every later test in this file.
- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
- }
- })
-
- it('restores readiness once a later safety pass certifies its inventory again', async () => {
- // The withdrawal above must not be a one-way latch: a fence raised by a
- // transient loss has to clear on the next completed pass, or the retry
- // machinery would leave the controller permanently unready.
- const watcher = newContextAuthoritativeWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- try {
- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
-
- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- } finally {
- // An assertion failure above must not escape a false implementation into
- // the rest of the file — vi.clearAllMocks() would not undo it.
- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
- }
- })
-
- it('ignores a late McpServer callback from a retired watch generation', async () => {
- const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []
- mocks.watch.mockImplementation(async (path, _options, callback) => {
- if (path.endsWith('/mcpservers')) callbacks.push(callback)
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- await (watcher as any).restartMcpServerWatch({
- resourceVersion: 'mcp-old-rv',
- servers: [],
- })
- await (watcher as any).restartMcpServerWatch({
- resourceVersion: 'mcp-new-rv',
- servers: [
- {
- name: 'current-server',
- namespace: 'mcp-server',
- uid: 'current-uid',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/current:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- },
- ],
- })
-
- await callbacks[0]('ADDED', {
- metadata: {
- name: 'stale-server',
- namespace: 'mcp-server',
- uid: 'stale-uid',
- generation: 1,
- },
- spec: {
- contextRef: 'default',
- image: 'clerum/stale:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- })
-
- expect(watcher.getAllServers().map(server => server.name)).toEqual(['current-server'])
- await watcher.stop()
- })
-
- it('recovers a closed Context watch through a fresh authoritative snapshot', async () => {
- vi.useFakeTimers()
- const doneCallbacks: Array<(error: Error | null) => void> = []
- const watchQueries: unknown[] = []
- mocks.watch.mockImplementation(async (path, options, _callback, done) => {
- if (path.endsWith('/contexts')) {
- doneCallbacks.push(done)
- watchQueries.push(options)
- }
- return { abort: vi.fn() }
- })
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'contexts') {
- return {
- metadata: { resourceVersion: 'context-recovery-rv' },
- items: [
- {
- metadata: { name: 'recovered-context', namespace: 'mcp-server' },
- spec: { contextId: 'recovered-context', mcpServers: [] },
- },
- ],
- }
- }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- await (watcher as any).restartContextWatch({
- resourceVersion: 'context-start-rv',
- contexts: [],
- })
-
- doneCallbacks[0](Object.assign(new Error('resource version expired'), { statusCode: 410 }))
- await vi.advanceTimersByTimeAsync(5000)
- await vi.waitFor(() => expect(watchQueries).toHaveLength(2))
-
- expect(watchQueries).toEqual([
- { resourceVersion: 'context-start-rv' },
- { resourceVersion: 'context-recovery-rv' },
- ])
- expect((watcher as any).contexts.get('recovered-context')).toEqual(
- expect.objectContaining({ name: 'recovered-context' })
- )
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalled())
- await watcher.stop()
- })
-
- it('becomes ready after authoritative revocation while every additive convergence lane remains pending', async () => {
- const initialServerFleet = deferred()
- const initialNetworkPolicies = deferred()
- const initialHostFleet = deferred()
- const initialSfsFleet = deferred()
- const initialGfsFleet = deferred()
- let completeAuthoritativeRevocation: (() => void) | undefined
- mocks.serverFullReconcile.mockImplementationOnce(() => initialServerFleet.promise)
- mocks.netPolFullReconcile.mockImplementationOnce(
- async (
- _contexts: unknown,
- _servers: unknown,
- options?: { onAuthoritativeRevocationComplete?: () => void }
- ) => {
- completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete
- await initialNetworkPolicies.promise
- }
- )
- mocks.hostFullReconcile.mockImplementationOnce(() => initialHostFleet.promise)
- mocks.sfsFullReconcile.mockImplementationOnce(() => initialSfsFleet.promise)
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'safe-bootstrap-rv' }, items: [] }
- }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- const startMcpServerWatch = stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- const startContextWatch = stubAuthoritativeInventoryWatch(watcher, 'Context')
- const startSharedFileSystemWatch = vi
- .spyOn(watcher as any, 'startSharedFileSystemWatch')
- .mockResolvedValue(undefined)
- const startGlobalFileSystemWatch = vi
- .spyOn(watcher as any, 'startGlobalFileSystemWatch')
- .mockResolvedValue(undefined)
- const gfsFullReconcile = vi
- .spyOn((watcher as any).gfsReconciler, 'fullReconcile')
- .mockImplementationOnce(() => initialGfsFleet.promise)
-
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- const bootstrap = watcher.start()
-
- try {
- await vi.waitFor(() => {
- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
- expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
- expect(mocks.sfsFullReconcile).toHaveBeenCalledOnce()
- expect(gfsFullReconcile).toHaveBeenCalledOnce()
- })
- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
-
- await expect(bootstrap).resolves.toBeUndefined()
- server.setReady(true)
- const beforeRevocation = await requestReadyOverHttp(server)
- expect(beforeRevocation.statusCode).toBe(503)
- expect(completeAuthoritativeRevocation).toBeTypeOf('function')
-
- completeAuthoritativeRevocation?.()
- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
- const ready = await requestReadyOverHttp(server)
- expect(ready.statusCode).toBe(200)
- expect(JSON.parse(ready.body)).toEqual({ status: 'ready', ready: true })
- expect(startMcpServerWatch).toHaveBeenCalledOnce()
- expect(startContextWatch).toHaveBeenCalledOnce()
- expect(startSharedFileSystemWatch).toHaveBeenCalledOnce()
- expect(startGlobalFileSystemWatch).toHaveBeenCalledOnce()
- expect(watcher.isCommunicationChannelCacheSynced()).toBe(true)
-
- // Runtime convergence starts at the safety-certification boundary, not
- // after the NetworkPolicy additive fleet completes. This prevents the
- // startup egress lane from racing the policy safety pass while retaining
- // readiness decoupling from both additive fleets.
- expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
- } finally {
- initialServerFleet.resolve(undefined)
- initialNetworkPolicies.resolve(undefined)
- initialHostFleet.resolve(undefined)
- initialSfsFleet.resolve(undefined)
- initialGfsFleet.resolve(undefined)
- await bootstrap.catch(() => undefined)
- await server.stop()
- watcher.stop()
- }
- })
-
- it('keeps readiness after a current Context safety delta while an older additive pass is pending', async () => {
- const olderAdditivePass = deferred()
- const currentDeltaSafety = deferred<boolean>()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- let networkPolicyPasses = 0
- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
- if (path.endsWith('/contexts')) contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
- networkPolicyPasses += 1
- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
- options?.onAuthoritativeRevocationComplete?.()
- if (networkPolicyPasses === 1) await olderAdditivePass.promise
- })
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('context-delta-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('delta-context', {
- name: 'delta-context',
- namespace: 'mcp-server',
- uid: 'delta-context-uid',
- spec: { contextId: 'delta-context', mcpServers: [] },
- })
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
- ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
- () => currentDeltaSafety.promise
- )
-
- try {
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
-
- const currentDelta = contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'delta-context',
- namespace: 'mcp-server',
- uid: 'delta-context-uid',
- },
- spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },
- })
-
- await vi.waitFor(() =>
- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()
- )
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- // The scoped revocation completed, which is what authorises the delta
- // certificate; see the aborted-pass twin above for the negative side.
- currentDeltaSafety.resolve(true)
- await currentDelta
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
- } finally {
- olderAdditivePass.resolve(undefined)
- await initialPass
- await server.stop()
- await watcher.stop()
- }
- })
-
- it('does not certify a Context delta whose stale-allow revocation aborted mid-pass', async () => {
- const olderAdditivePass = deferred()
- const abortedDeltaSafety = deferred<boolean>()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- let networkPolicyPasses = 0
- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
- if (path.endsWith('/contexts')) contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
- networkPolicyPasses += 1
- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
- options?.onAuthoritativeRevocationComplete?.()
- if (networkPolicyPasses === 1) await olderAdditivePass.promise
- })
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('aborted-delta-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('delta-context', {
- name: 'delta-context',
- namespace: 'mcp-server',
- uid: 'delta-context-uid',
- spec: { contextId: 'delta-context', mcpServers: [] },
- })
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
- // `reconcileContext` reports that its authority fence broke, so the scoped
- // revocation never finished deleting this Context's stale allows.
- ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
- () => abortedDeltaSafety.promise
- )
-
- try {
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
-
- const abortedDelta = contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'delta-context',
- namespace: 'mcp-server',
- uid: 'delta-context-uid',
- },
- spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },
- })
-
- await vi.waitFor(() =>
- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()
- )
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- abortedDeltaSafety.resolve(false)
- await abortedDelta
-
- // An incomplete revocation must leave readiness withheld until the
- // authoritative full pass re-certifies. Certifying here would report
- // Ready while a stale allow is still live.
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- } finally {
- olderAdditivePass.resolve(undefined)
- await initialPass
- await server.stop()
- await watcher.stop()
- }
- })
-
- it('does not certify a Context delta while the namespace-wide inventory is uncertified', async () => {
- const olderAdditivePass = deferred()
- const uncertifiedDelta = deferred<boolean>()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- let networkPolicyPasses = 0
- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
- if (path.endsWith('/contexts')) contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- mocks.netPolFullReconcile.mockImplementation(async (...args: unknown[]) => {
- networkPolicyPasses += 1
- const options = args[2] as { onAuthoritativeRevocationComplete?: () => void } | undefined
- options?.onAuthoritativeRevocationComplete?.()
- if (networkPolicyPasses === 1) await olderAdditivePass.promise
- })
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('uncertified-inventory-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('delta-context', {
- name: 'delta-context',
- namespace: 'mcp-server',
- uid: 'delta-context-uid',
- spec: { contextId: 'delta-context', mcpServers: [] },
- })
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
- const initialPass = (watcher as any).runInitialNetworkPolicyConvergence() as Promise<void>
- ;(watcher as any).netPolReconciler.reconcileContext.mockImplementationOnce(
- () => uncertifiedDelta.promise
- )
- try {
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
- // Baseline: the inventory is still certified here, so readiness is
- // genuinely green. Degrading the fence before this point would assert a
- // 200 that only the missing readiness gate could produce.
- expect((await requestReadyOverHttp(server)).statusCode).toBe(200)
-
- // The delta's own revocation completes, but the last authoritative pass left
- // the namespace-wide inventory uncertified. A label-scoped delta cannot
- // vouch for that namespace, so readiness must stay withheld.
- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
-
- const delta = contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'delta-context',
- namespace: 'mcp-server',
- uid: 'delta-context-uid',
- },
- spec: { contextId: 'delta-context', mcpServers: ['delta-server'] },
- })
-
- await vi.waitFor(() =>
- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledOnce()
- )
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- uncertifiedDelta.resolve(true)
- await delta
-
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- } finally {
- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
- olderAdditivePass.resolve(undefined)
- await initialPass
- await server.stop()
- await watcher.stop()
- }
- })
-
- it('recovers when a Context delta loses the delete fence with no certificate to withhold', async () => {
- // A MODIFIED that does not move the desired revision — a status or label
- // write by another controller — still runs a scoped revocation, but builds
- // no delta certificate. Nothing reads the returned boolean there, so a lost
- // delete fence must stay loud and reach the convergence retry instead of
- // leaving a stale allow live with no NetworkPolicy resync to recover it.
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string; generation?: number }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (path, _options, callback) => {
- if (path.endsWith('/contexts')) contextWatchCallback = callback
- return { abort: vi.fn() }
- })
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('lost-fence-no-certificate-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('steady-context', {
- name: 'steady-context',
- namespace: 'mcp-server',
- uid: 'steady-context-uid',
- generation: 4,
- spec: { contextId: 'steady-context', mcpServers: [] },
- })
- markNetworkPolicyRevocationAuthoritative(watcher)
- const passesBefore = mocks.netPolFullReconcile.mock.calls.length
- // With honorsLostFence false the real reconciler throws the 409 rather than
- // reporting it, which is the whole point: the throw is what reaches the
- // convergence retry below.
- ;(watcher as any).netPolReconciler.reconcileContext.mockRejectedValueOnce(
- Object.assign(new Error('the UID in the precondition does not match'), { code: 409 })
- )
-
- try {
- await contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'steady-context',
- namespace: 'mcp-server',
- uid: 'steady-context-uid',
- generation: 4,
- },
- spec: { contextId: 'steady-context', mcpServers: [] },
- })
-
- // This lane cannot act on the boolean, so it must not have opted into the
- // reported outcome.
- expect((watcher as any).netPolReconciler.reconcileContext).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'steady-context' }),
- expect.objectContaining({ honorsLostFence: false })
- )
- await vi.waitFor(() =>
- expect(mocks.netPolFullReconcile.mock.calls.length).toBeGreaterThan(passesBefore)
- )
- } finally {
- await watcher.stop()
- }
- })
-
- it('does not issue a Context delta certificate before any authoritative safety certificate', async () => {
- const pendingFullSafetyPass = deferred()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('uncertified-context-delta-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('uncertified-context', {
- name: 'uncertified-context',
- namespace: 'mcp-server',
- uid: 'uncertified-context-uid',
- spec: { contextId: 'uncertified-context', mcpServers: [] },
- })
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
-
- try {
- await contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'uncertified-context',
- namespace: 'mcp-server',
- uid: 'uncertified-context-uid',
- },
- spec: { contextId: 'uncertified-context', mcpServers: ['delta-server'] },
- })
-
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- } finally {
- pendingFullSafetyPass.resolve(undefined)
- await watcher.stop()
- await server.stop()
- }
- })
-
- it('does not let a Context delta certify an outstanding McpServer revision', async () => {
- const pendingFullSafetyPass = deferred()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('server-revision-context-delta-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('unrelated-context', {
- name: 'unrelated-context',
- namespace: 'mcp-server',
- uid: 'unrelated-context-uid',
- spec: { contextId: 'unrelated-context', mcpServers: [] },
- })
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).mcpServerDesiredRevision += 1
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
-
- try {
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- await contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'unrelated-context',
- namespace: 'mcp-server',
- uid: 'unrelated-context-uid',
- },
- spec: { contextId: 'unrelated-context', mcpServers: ['delta-server'] },
- })
-
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- } finally {
- pendingFullSafetyPass.resolve(undefined)
- await watcher.stop()
- await server.stop()
- }
- })
-
- it('does not let one Context delta certify another Context revision still in flight', async () => {
- const pendingFullSafetyPass = deferred()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string; uid: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- mocks.netPolFullReconcile.mockImplementationOnce(() => pendingFullSafetyPass.promise)
-
- const watcher = new McpServerWatcher()
- await (watcher as any).startContextWatch('parallel-context-delta-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 1
- ;(watcher as any).contexts.set('first-context', {
- name: 'first-context',
- namespace: 'mcp-server',
- uid: 'first-context-uid',
- spec: { contextId: 'first-context', mcpServers: [] },
- })
- ;(watcher as any).contexts.set('second-context', {
- name: 'second-context',
- namespace: 'mcp-server',
- uid: 'second-context-uid',
- spec: { contextId: 'second-context', mcpServers: [] },
- })
- markNetworkPolicyRevocationAuthoritative(watcher)
- // This represents a prior Context change whose global safety sweep is
- // pending. It invalidates the old certificate before the scoped delta.
- ;(watcher as any).contextDesiredRevision += 1
- const server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
-
- try {
- await contextWatchCallback!('MODIFIED', {
- metadata: {
- name: 'second-context',
- namespace: 'mcp-server',
- uid: 'second-context-uid',
- },
- spec: { contextId: 'second-context', mcpServers: ['delta-server'] },
- })
-
- expect((await requestReadyOverHttp(server)).statusCode).toBe(503)
- } finally {
- pendingFullSafetyPass.resolve(undefined)
- await watcher.stop()
- await server.stop()
- }
- })
-
- it.each([
- ['Context', 'contextDesiredRevision'],
- ['McpServer', 'mcpServerDesiredRevision'],
- ] as const)(
- 'does not restore readiness from a stale %s delta safety certificate',
- async (_kind, revisionField) => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 4
- ;(watcher as any).mcpWatchGeneration = 7
- markNetworkPolicyRevocationAuthoritative(watcher)
- const certificate = {
- contextGeneration: (watcher as any).contextWatchGeneration,
- serverGeneration: (watcher as any).mcpWatchGeneration,
- contextRevision: (watcher as any).contextDesiredRevision,
- serverRevision: (watcher as any).mcpServerDesiredRevision,
- }
-
- ;(watcher as any)[revisionField] += 1
-
- expect((watcher as any).recordNetworkPolicySafetyCertificate(certificate)).toBe(false)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- await watcher.stop()
- }
- )
-
- it.each([
- ['SharedFileSystem', 'LIST', 'sharedfilesystems', 'startSharedFileSystemWatch'],
- ['SharedFileSystem', 'WATCH', 'sharedfilesystems', 'startSharedFileSystemWatch'],
- ['GlobalFileSystem', 'LIST', 'globalfilesystems', 'startGlobalFileSystemWatch'],
- ['GlobalFileSystem', 'WATCH', 'globalfilesystems', 'startGlobalFileSystemWatch'],
- ] as const)(
- 'does not let a pending %s %s delay NetworkPolicy safety, bootstrap, or readiness certification',
- async (_kind, boundary, plural, watchMethod) => {
- const pendingBoundary = deferred<unknown>()
- const pendingAdditivePolicies = deferred()
- let completeAuthoritativeRevocation: (() => void) | undefined
- mocks.listNamespacedCustomObject.mockImplementation(
- async ({ plural: requestedPlural }: { plural: string }) => {
- if (requestedPlural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'background-lane-rv' }, items: [] }
- }
- if (boundary === 'LIST' && requestedPlural === plural) {
- return pendingBoundary.promise
- }
- return { items: [] }
- }
- )
- mocks.netPolFullReconcile.mockImplementationOnce(
- async (
- _contexts: unknown,
- _servers: unknown,
- options?: { onAuthoritativeRevocationComplete?: () => void }
- ) => {
- completeAuthoritativeRevocation = options?.onAuthoritativeRevocationComplete
- await pendingAdditivePolicies.promise
- }
- )
-
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- const sfsWatch = vi
- .spyOn(watcher as any, 'startSharedFileSystemWatch')
- .mockResolvedValue(undefined)
- const gfsWatch = vi
- .spyOn(watcher as any, 'startGlobalFileSystemWatch')
- .mockResolvedValue(undefined)
- const targetWatch = watchMethod === 'startSharedFileSystemWatch' ? sfsWatch : gfsWatch
- if (boundary === 'WATCH') {
- targetWatch.mockImplementationOnce(() => pendingBoundary.promise as Promise<void>)
- }
- const bootstrap = watcher.start()
-
- try {
- await vi.waitFor(() => expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce())
- await expect(bootstrap).resolves.toBeUndefined()
-
- expect(completeAuthoritativeRevocation).toBeTypeOf('function')
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- if (boundary === 'LIST') {
- expect(targetWatch).not.toHaveBeenCalled()
- } else {
- expect(targetWatch).toHaveBeenCalledOnce()
- }
-
- completeAuthoritativeRevocation?.()
- await vi.waitFor(() => expect(watcher.isReadinessInventoryAuthoritative()).toBe(true))
- } finally {
- pendingBoundary.resolve({ items: [] })
- pendingAdditivePolicies.resolve(undefined)
- await flushMicrotasks()
- await watcher.stop()
- }
- }
- )
-
- it('starts every initial background convergence lane from the caches current after watches', async () => {
- const staleServer = {
- metadata: { name: 'stale-server', namespace: 'mcp-server' },
- spec: {
- contextRef: 'default',
- image: 'clerum/stale-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const currentServer = {
- name: 'current-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/current-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const staleContext = {
- metadata: { name: 'stale-context', namespace: 'mcp-server' },
- spec: { contextId: 'stale-context', mcpServers: [] },
- }
- const currentContext = {
- name: 'current-context',
- namespace: 'mcp-server',
- spec: { contextId: 'current-context', mcpServers: [] },
- }
- const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
- const currentSfs = { name: 'current-sfs', namespace: 'mcp-host', spec: {} }
- const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }
- const currentGfs = { name: 'current-gfs', namespace: 'mcp-host', spec: {} }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [staleServer] }
- if (plural === 'contexts') return { items: [staleContext] }
- if (plural === 'sharedfilesystems') return { items: [staleSfs] }
- if (plural === 'globalfilesystems') return { items: [staleGfs] }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'current-cache-rv' }, items: [] }
- }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer', () => {
- ;(watcher as any).servers.clear()
- ;(watcher as any).servers.set(currentServer.name, currentServer)
- })
- stubAuthoritativeInventoryWatch(watcher, 'Context', () => {
- ;(watcher as any).contexts.clear()
- ;(watcher as any).contexts.set(currentContext.name, currentContext)
- })
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => {
- ;(watcher as any).sharedFileSystems.clear()
- ;(watcher as any).sharedFileSystems.set(currentSfs.name, currentSfs)
- })
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => {
- ;(watcher as any).globalFileSystems.clear()
- ;(watcher as any).globalFileSystems.set(currentGfs.name, currentGfs)
- })
- const gfsFullReconcile = vi.spyOn((watcher as any).gfsReconciler, 'fullReconcile')
-
- await watcher.start()
-
- await vi.waitFor(() => {
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith([currentServer], {
- runEffect: expect.any(Function),
- })
- expect(mocks.netPolFullReconcile).toHaveBeenCalledWith(
- [currentContext],
- [currentServer],
- expect.objectContaining({
- serverInventoryComplete: true,
- ensureDefaults: false,
- contextInventoryAuthoritative: expect.any(Function),
- serverInventoryAuthoritative: expect.any(Function),
- })
- )
- expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([currentSfs])
- expect(gfsFullReconcile).toHaveBeenCalledWith([currentGfs])
- })
-
- watcher.stop()
- })
-
- it('updates discovery without re-reconciling a status-only McpServer watch event', async () => {
- const previousServer = {
- name: 'status-server',
- namespace: 'mcp-server',
- uid: 'status-server-uid',
- generation: 7,
- annotations: { 'clerum.io/recipe': 'status-recipe' },
- labels: { 'clerum.io/workload': 'status-server' },
- spec: {
- contextRef: 'default',
- image: 'clerum/status-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const watcher = new McpServerWatcher()
- ;(watcher as any).servers.set(previousServer.name, previousServer)
- const changed = vi.fn()
- watcher.onChange(changed)
- const reconciler = (watcher as any).reconciler
- const netPol = (watcher as any).netPolReconciler
- const callback = (watcher as any).getMcpServerWatchCallback()
- expect((watcher as any).mcpServerDesiredRevision).toBe(0)
-
- await callback('MODIFIED', {
- metadata: {
- name: previousServer.name,
- namespace: previousServer.namespace,
- uid: previousServer.uid,
- generation: previousServer.generation,
- annotations: previousServer.annotations,
- labels: previousServer.labels,
- },
- spec: previousServer.spec,
- status: { conditions: [{ type: 'Ready', status: 'True' }] },
- })
-
- expect(watcher.getAllServers()[0]?.status).toEqual({
- conditions: [{ type: 'Ready', status: 'True' }],
- })
- expect(changed).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).not.toHaveBeenCalled()
- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
- expect((watcher as any).mcpServerDesiredRevision).toBe(0)
-
- await callback('MODIFIED', {
- metadata: {
- name: previousServer.name,
- namespace: previousServer.namespace,
- uid: previousServer.uid,
- generation: 8,
- annotations: previousServer.annotations,
- labels: previousServer.labels,
- },
- spec: { ...previousServer.spec, image: 'clerum/status-server:next' },
- })
- expect(reconciler.reconcile).toHaveBeenCalledOnce()
- expect((watcher as any).mcpServerDesiredRevision).toBe(1)
- await watcher.stop()
- })
-
- it('does not advance desired revision for the controller-owned network-ready handshake', async () => {
- const previousServer = {
- name: 'network-ready-server',
- namespace: 'mcp-server',
- uid: 'network-ready-server-uid',
- generation: 7,
- annotations: { 'clerum.io/pre-deploy': 'true' },
- labels: { 'clerum.io/workload': 'network-ready-server' },
- spec: {
- contextRef: 'default',
- image: 'clerum/network-ready-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const watcher = new McpServerWatcher()
- ;(watcher as any).servers.set(previousServer.name, previousServer)
- const callback = (watcher as any).getMcpServerWatchCallback()
-
- await callback('MODIFIED', {
- metadata: {
- name: previousServer.name,
- namespace: previousServer.namespace,
- uid: previousServer.uid,
- generation: previousServer.generation,
- annotations: {
- ...previousServer.annotations,
- 'clerum.io/network-ready': 'true',
- },
- labels: previousServer.labels,
- },
- spec: previousServer.spec,
- })
-
- expect((watcher as any).mcpServerDesiredRevision).toBe(0)
- expect((watcher as any).reconciler.reconcile).not.toHaveBeenCalled()
- await watcher.stop()
- })
-
- it('advances McpServer desired revision for owner addition and deletion', async () => {
- const serverObject = {
- metadata: {
- name: 'revision-server',
- namespace: 'mcp-server',
- uid: 'revision-server-uid',
- generation: 1,
- },
- spec: {
- contextRef: 'default',
- image: 'clerum/revision-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const watcher = new McpServerWatcher()
- const callback = (watcher as any).getMcpServerWatchCallback()
-
- expect((watcher as any).mcpServerDesiredRevision).toBe(0)
- await callback('ADDED', serverObject)
- expect((watcher as any).mcpServerDesiredRevision).toBe(1)
- await callback('DELETED', serverObject)
- expect((watcher as any).mcpServerDesiredRevision).toBe(2)
-
- await watcher.stop()
- })
-
- it('advances Context desired revision only when the authoritative desired cache changes', async () => {
- const watcher = new McpServerWatcher()
- let contextWatchCallback:
- | ((
- type: string,
- apiObj: {
- metadata: { name: string; namespace: string }
- spec: { contextId: string; mcpServers: string[] }
- }
- ) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- await (watcher as any).startContextWatch('context-desired-revision-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
-
- const added = {
- metadata: { name: 'revision-context', namespace: 'mcp-server' },
- spec: { contextId: 'revision-context', mcpServers: [] as string[] },
- }
- expect((watcher as any).contextDesiredRevision).toBe(0)
- await contextWatchCallback!('ADDED', added)
- expect((watcher as any).contextDesiredRevision).toBe(1)
-
- await contextWatchCallback!('MODIFIED', added)
- expect((watcher as any).contextDesiredRevision).toBe(1)
-
- const modified = {
- ...added,
- spec: { ...added.spec, mcpServers: ['revision-server'] },
- }
- await contextWatchCallback!('MODIFIED', modified)
- expect((watcher as any).contextDesiredRevision).toBe(2)
-
- await contextWatchCallback!('DELETED', modified)
- expect((watcher as any).contextDesiredRevision).toBe(3)
-
- await watcher.stop()
- })
-
- it('uses only the bounded fleet pass for the cold-start Host snapshot', async () => {
- const initialHosts = Array.from({ length: 50 }, (_, index) => ({
- metadata: {
- name: `initial-host-${index}`,
- namespace: 'mcp-host',
- uid: `initial-host-uid-${index}`,
- generation: 1,
- },
- spec: {
- host: `initial-host-${index}`,
- contextRef: 'default',
- secretRef: 'host-secret',
- },
- }))
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- return {
- metadata: { resourceVersion: 'initial-host-rv' },
- items: initialHosts,
- }
- }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'initial-cc-rv' }, items: [] }
- }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- const urgentReconcile = vi.spyOn(watcher.getHostReconciler(), 'reconcile')
-
- await watcher.start()
- await vi.waitFor(() =>
- expect(mocks.hostFullReconcile).toHaveBeenCalledWith(
- initialHosts.map(host =>
- expect.objectContaining({
- name: host.metadata.name,
- generation: host.metadata.generation,
- })
- )
- )
- )
- await flushMicrotasks()
-
- expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
- expect(urgentReconcile).not.toHaveBeenCalled()
- await watcher.stop()
- })
-
- it('admits a McpServer watch reconciliation while initial convergence is blocked on another server', async () => {
- const staleServer = {
- name: 'stale-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/stale-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'stale.example', port: 443 }],
- },
- }
- const currentServer = {
- metadata: { name: 'current-server', namespace: 'mcp-server' },
- spec: {
- contextRef: 'default',
- image: 'clerum/current-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const reconciliationOrder: string[] = []
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).servers.set(staleServer.name, staleServer)
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
- await Promise.all(
- servers.map((selected: { name: string }) =>
- options.runEffect(selected.name, async () => {
- reconciliationOrder.push('initial')
- })
- )
- )
- })
- const reconciler = (watcher as any).reconciler
- reconciler.reconcile.mockImplementationOnce(async () => {
- reconciliationOrder.push('watch')
- })
-
- const convergence = (watcher as any).runInitialMcpServerConvergence()
- await egressStarted.promise
- const watchConvergence = (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)
-
- await flushMicrotasks()
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'current-server' }),
- { isCurrent: expect.any(Function) }
- )
-
- releaseEgress.resolve(undefined)
- await Promise.all([convergence, watchConvergence])
-
- expect(reconciliationOrder).toEqual(['watch'])
- expect(reconciler.reconcile).toHaveBeenLastCalledWith(
- expect.objectContaining({ name: 'current-server' }),
- { isCurrent: expect.any(Function) }
- )
- await watcher.stop()
- })
-
- it('continues a same-server reconciliation lane after a rejected operation', async () => {
- const watcher = new McpServerWatcher()
- const server = { name: 'retry-safe-server', namespace: 'mcp-server' }
- const order: string[] = []
- const firstStarted = deferred()
- const rejectFirst = deferred()
-
- const rejected = (watcher as any).enqueueMcpServerReconciliation(server, async () => {
- order.push('rejected')
- firstStarted.resolve(undefined)
- await rejectFirst.promise
- throw new Error('transient lane failure')
- }) as Promise<void>
- await firstStarted.promise
- const recovered = (watcher as any).enqueueMcpServerReconciliation(server, async () => {
- order.push('recovered')
- }) as Promise<void>
- rejectFirst.resolve(undefined)
-
- await expect(rejected).rejects.toThrow('transient lane failure')
- await expect(recovered).resolves.toBeUndefined()
- expect(order).toEqual(['rejected', 'recovered'])
- expect((watcher as any).mcpServerReconciliationQueues.size).toBe(0)
- await watcher.stop()
- })
-
- it('revalidates env-Secret matches inside each server lane before reconciling', async () => {
- const currentServer = {
- name: 'current-secret-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/current-secret-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- envSecret: { name: 'shared-env' },
- },
- }
- const staleServer = {
- name: 'stale-secret-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/stale-secret-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- envSecret: { name: 'shared-env' },
- },
- }
- const unrelatedServer = {
- name: 'unrelated-secret-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/unrelated-secret-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- envSecret: { name: 'other-env' },
- },
- }
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- const reconciler = (watcher as any).reconciler
- for (const server of [currentServer, staleServer, unrelatedServer]) {
- ;(watcher as any).servers.set(server.name, server)
- }
-
- const blocker = (watcher as any).enqueueMcpServerReconciliation(staleServer, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- })
- await blockerStarted.promise
-
- const convergence = watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
- ;(watcher as any).servers.set(staleServer.name, {
- ...staleServer,
- spec: { ...staleServer.spec, envSecret: { name: 'replacement-env' } },
- })
- await vi.waitFor(() =>
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: currentServer.name }),
- { isCurrent: expect.any(Function) }
- )
- )
- releaseBlocker.resolve(undefined)
- await Promise.all([blocker, convergence])
-
- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
- expect(
- reconciler.reconcile.mock.calls.some(
- ([server]: [McpServerCRD]) => server.name === staleServer.name
- )
- ).toBe(false)
- expect(
- reconciler.reconcile.mock.calls.some(
- ([server]: [McpServerCRD]) => server.name === unrelatedServer.name
- )
- ).toBe(false)
- await watcher.stop()
- })
-
- it('defers env-Secret reconciliation until the McpServer inventory is authoritative', async () => {
- const selected = {
- name: 'authority-fenced-secret-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/authority-fenced-secret-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- envSecret: { name: 'shared-env' },
- },
- }
- const watcher = new McpServerWatcher()
- const reconciler = (watcher as any).reconciler
- ;(watcher as any).servers.set(selected.name, selected)
- ;(watcher as any).mcpServerCacheSynced = false
-
- await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
-
- expect(reconciler.reconcile).not.toHaveBeenCalled()
-
- markMcpServerInventoryAuthoritative(watcher)
- await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
-
- expect(reconciler.reconcile).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: selected.name }),
- { isCurrent: expect.any(Function) }
- )
- await watcher.stop()
- })
-
- it('fences Context-triggered SharedFileSystem convergence to the current Context lease', async () => {
- const context = {
- name: 'sfs-context',
- namespace: 'mcp-server',
- spec: {
- contextId: 'sfs-context',
- sharedFileSystems: [{ name: 'shared-data', mountPath: '/shared' }],
- },
- }
- const sharedFileSystem = {
- name: 'shared-data',
- namespace: 'mcp-host',
- spec: {},
- }
- const watcher = new McpServerWatcher()
- const reconciler = (watcher as any).sharedFileSystemReconciler
- ;(watcher as any).sharedFileSystems.set(sharedFileSystem.name, sharedFileSystem)
-
- await (watcher as any).reconcileSharedFileSystemsReferencedByContext(
- undefined,
- context,
- () => false
- )
-
- expect(reconciler.reconcile).not.toHaveBeenCalled()
-
- await (watcher as any).reconcileSharedFileSystemsReferencedByContext(
- undefined,
- context,
- () => true
- )
-
- expect(reconciler.reconcile).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: sharedFileSystem.name })
- )
- await watcher.stop()
- })
-
- it('fences Context-triggered Host convergence to the current Context lease', async () => {
- const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
- ;(watcher as any).hosts.set('context-host', {
- name: 'context-host',
- namespace: 'mcp-host',
- spec: {
- host: 'context-host',
- contextRef: 'context-a',
- secretRef: 'host-secret',
- },
- })
- const hostReconciler = (watcher as any).hostReconciler
-
- await (watcher as any).reconcileHostsReferencingContext('context-a', () => false)
-
- expect(hostReconciler.reconcile).not.toHaveBeenCalled()
-
- await (watcher as any).reconcileHostsReferencingContext('context-a', () => true)
-
- expect(hostReconciler.reconcile).toHaveBeenCalledOnce()
- expect(hostReconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'context-host' })
- )
- await watcher.stop()
- })
-
- it('keeps env-Secret reconciliation a no-op after stop', async () => {
- const selected = {
- name: 'stopped-secret-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/stopped-secret-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- envSecret: { name: 'shared-env' },
- },
- }
- const watcher = new McpServerWatcher()
- const reconciler = (watcher as any).reconciler
- ;(watcher as any).servers.set(selected.name, selected)
-
- await watcher.stop()
- await watcher.reconcileByEnvSecret('shared-env', 'mcp-server')
-
- expect(reconciler.reconcile).not.toHaveBeenCalled()
- })
-
- it('admits an initial server without egress while another initial server waits on DNS', async () => {
- const dnsBlockedServer = {
- name: 'dns-blocked-server',
- namespace: 'mcp-server',
- uid: 'dns-blocked-server-uid',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/dns-blocked-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'blocked.example', port: 443 }],
- },
- }
- const independentServer = {
- name: 'independent-server',
- namespace: 'mcp-server',
- uid: 'independent-server-uid',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/independent-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).servers.set(dnsBlockedServer.name, dnsBlockedServer)
- ;(watcher as any).servers.set(independentServer.name, independentServer)
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- mocks.serverFullReconcile.mockImplementationOnce(
- async (
- servers: Array<typeof dnsBlockedServer | typeof independentServer>,
- options: { runEffect: (serverName: string, work: () => Promise<void>) => Promise<void> }
- ) => {
- await Promise.all(
- servers.map(server => options.runEffect(server.name, () => reconciler.reconcile(server)))
- )
- }
- )
-
- const convergence = (watcher as any).runInitialMcpServerConvergence()
- await egressStarted.promise
- await flushMicrotasks()
-
- expect(reconciler.reconcile).toHaveBeenCalledWith(independentServer)
- expect(reconciler.reconcile).not.toHaveBeenCalledWith(dnsBlockedServer)
-
- releaseEgress.resolve(undefined)
- await convergence
-
- expect(reconciler.reconcile).toHaveBeenCalledWith(dnsBlockedServer)
- await watcher.stop()
- })
-
- it('admits a Context watch reconciliation while initial policy convergence is blocked', async () => {
- const staleContext = {
- name: 'stale-context',
- namespace: 'mcp-server',
- spec: { contextId: 'stale-context', mcpServers: [] },
- }
- const currentContext = {
- metadata: { name: 'current-context', namespace: 'mcp-server' },
- spec: { contextId: 'current-context', mcpServers: [] },
- }
- const fullReconcileStarted = deferred()
- const releaseFullReconcile = deferred()
- const reconciliationOrder: string[] = []
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).contexts.set(staleContext.name, staleContext)
- mocks.netPolFullReconcile.mockImplementationOnce(async () => {
- fullReconcileStarted.resolve(undefined)
- await releaseFullReconcile.promise
- reconciliationOrder.push('initial')
- })
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileContext.mockImplementationOnce(async () => {
- reconciliationOrder.push('watch')
- })
- let contextWatchCallback:
- | ((type: string, apiObj: typeof currentContext) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- await (watcher as any).startContextWatch('context-queue-rv')
-
- const convergence = (watcher as any).runInitialNetworkPolicyConvergence(true)
- await fullReconcileStarted.promise
- const watchConvergence = contextWatchCallback!('MODIFIED', currentContext)
-
- await flushMicrotasks()
- expect(netPol.reconcileContext).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'current-context' }),
- expect.objectContaining({ isCurrent: expect.any(Function) })
- )
-
- releaseFullReconcile.resolve(undefined)
- await Promise.all([convergence, watchConvergence])
-
- expect(reconciliationOrder).toEqual(['watch', 'initial'])
- expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
- expect.objectContaining({ name: 'current-context' }),
- expect.objectContaining({ isCurrent: expect.any(Function) })
- )
- await watcher.stop()
- })
-
- it('makes a recreated Context the last writer after same-contextId orphan cleanup', async () => {
- const contextName = 'recreated-context-resource'
- const contextId = 'recreated-context'
- const recreatedContext = {
- metadata: { name: contextName, namespace: 'mcp-server' },
- spec: {
- contextId,
- description: 'recreated',
- mcpServers: [],
- },
- }
- const orphanCleanupStarted = deferred()
- const releaseOrphanCleanup = deferred()
- const order: string[] = []
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- mocks.netPolFullReconcile.mockImplementationOnce(
- async (
- _contexts: unknown[],
- _servers: unknown[],
- options: {
- runContextEffect: (effectContextId: string, work: () => Promise<void>) => Promise<void>
- }
- ) => {
- await options.runContextEffect(contextId, async () => {
- orphanCleanupStarted.resolve(undefined)
- await releaseOrphanCleanup.promise
- order.push('orphan-delete')
- })
- }
- )
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileContext.mockImplementationOnce(async () => {
- order.push('live-recreate')
- })
- let contextWatchCallback:
- | ((type: string, apiObj: typeof recreatedContext) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- await (watcher as any).startContextWatch('recreated-context-rv')
-
- const convergence = (watcher as any).runInitialNetworkPolicyConvergence()
- await orphanCleanupStarted.promise
- const recreation = contextWatchCallback!('ADDED', recreatedContext)
- await flushMicrotasks()
-
- expect(netPol.reconcileContext).not.toHaveBeenCalled()
-
- releaseOrphanCleanup.resolve(undefined)
- await Promise.all([convergence, recreation])
-
- expect(order).toEqual(['orphan-delete', 'live-recreate'])
- expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
- expect.objectContaining({
- name: contextName,
- spec: expect.objectContaining({ contextId, description: 'recreated' }),
- }),
- expect.objectContaining({ isCurrent: expect.any(Function) })
- )
- await watcher.stop()
- })
-
- it('retires old contextId policies before reconciling a mutable Context identity', async () => {
- const oldContext = {
- name: 'mutable-context-resource',
- namespace: 'mcp-server',
- spec: {
- contextId: 'old-context-id',
- mcpServers: ['redis-tools'],
- },
- }
- const newContext = {
- metadata: { name: oldContext.name, namespace: oldContext.namespace },
- spec: {
- contextId: 'new-context-id',
- mcpServers: ['redis-tools'],
- },
- }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'contexts') {
- return {
- metadata: { resourceVersion: 'mutable-context-rv' },
- items: [newContext],
- }
- }
- return { items: [] }
- })
- let contextWatchCallback:
- | ((type: string, apiObj: typeof newContext) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- ;(watcher as any).contexts.set(oldContext.name, oldContext)
- ;(watcher as any).contextCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- const order: string[] = []
- netPol.reconcileDeleteContext.mockImplementation(async (contextId: string) => {
- order.push(`delete:${contextId}`)
- })
- netPol.reconcileContext.mockImplementation(async (context: typeof oldContext) => {
- order.push(`reconcile:${context.spec.contextId}`)
- })
- await (watcher as any).startContextWatch('mutable-context-rv')
-
- await contextWatchCallback!('MODIFIED', newContext)
-
- expect(order).toEqual(['delete:old-context-id', 'reconcile:new-context-id'])
- expect(netPol.reconcileDeleteContext).toHaveBeenCalledWith(
- 'old-context-id',
- expect.any(Function)
- )
- await watcher.stop()
- })
-
- it('retires queued Context policy work immediately when its watch ends', async () => {
- const context = {
- metadata: { name: 'retired-context-resource', namespace: 'mcp-server' },
- spec: { contextId: 'retired-context-id', mcpServers: [] },
- }
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined
- let contextWatchDone: ((error: Error | null) => void) | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {
- contextWatchCallback = callback
- contextWatchDone = done
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- await (watcher as any).startContextWatch('retired-context-rv')
- const blocker = (watcher as any).enqueueContextReconciliation(
- context.spec.contextId,
- async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }
- )
- await blockerStarted.promise
-
- const event = contextWatchCallback!('MODIFIED', context)
- contextWatchDone!(null)
- releaseBlocker.resolve(undefined)
- await Promise.all([blocker, event])
-
- expect(netPol.reconcileContext).not.toHaveBeenCalled()
- await watcher.stop()
- })
-
- it('serializes McpServer-driven Context policy work with a newer Context watch revision', async () => {
- const server = {
- metadata: {
- name: 'context-policy-server',
- namespace: 'mcp-server',
- uid: 'context-policy-server-uid',
- generation: 1,
- },
- spec: {
- contextRef: 'context-policy',
- image: 'clerum/context-policy-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const oldContext = {
- name: 'context-policy-resource',
- namespace: 'mcp-server',
- spec: {
- contextId: 'context-policy',
- description: 'old',
- mcpServers: [server.metadata.name],
- },
- }
- const newContext = {
- metadata: { name: oldContext.name, namespace: oldContext.namespace },
- spec: {
- ...oldContext.spec,
- description: 'new',
- },
- }
- const oldStarted = deferred()
- const releaseOld = deferred()
- const order: string[] = []
- let oldLease: (() => boolean) | undefined
- let contextWatchCallback:
- | ((type: string, apiObj: typeof newContext) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- ;(watcher as any).contexts.set(oldContext.name, oldContext)
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileContext.mockImplementation(
- async (context: typeof oldContext, options: { isCurrent: () => boolean }) => {
- if (context.spec.description === 'old') {
- oldLease = options.isCurrent
- oldStarted.resolve(undefined)
- await releaseOld.promise
- order.push('old')
- return
- }
- order.push('new')
- }
- )
- await (watcher as any).startContextWatch('context-policy-rv')
-
- const serverEvent = (watcher as any).getMcpServerWatchCallback()('MODIFIED', server)
- await oldStarted.promise
- const contextEvent = contextWatchCallback!('MODIFIED', newContext)
- await flushMicrotasks()
-
- expect(netPol.reconcileContext).toHaveBeenCalledTimes(1)
- expect(oldLease?.()).toBe(false)
- releaseOld.resolve(undefined)
- await Promise.all([serverEvent, contextEvent])
-
- expect(order).toEqual(['old', 'new'])
- expect(netPol.reconcileContext).toHaveBeenLastCalledWith(
- expect.objectContaining({
- spec: expect.objectContaining({ description: 'new' }),
- }),
- expect.objectContaining({ isCurrent: expect.any(Function) })
- )
- await watcher.stop()
- })
-
- it('retires a blocked Context policy lease when MODIFIED replaces it in the same watch', async () => {
- const oldContext = {
- metadata: { name: 'same-stream-context', namespace: 'mcp-server' },
- spec: {
- contextId: 'same-stream-context',
- description: 'old',
- mcpServers: ['old-server'],
- },
- }
- const newContext = {
- metadata: oldContext.metadata,
- spec: {
- ...oldContext.spec,
- description: 'new',
- mcpServers: ['new-server'],
- },
- }
- const mutationStarted = deferred()
- const releaseMutation = deferred()
- let oldLease: (() => boolean) | undefined
- let contextWatchCallback:
- | ((type: string, apiObj: typeof oldContext) => Promise<void>)
- | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileContext.mockImplementationOnce(
- async (_context: unknown, options: { isCurrent: () => boolean }) => {
- oldLease = options.isCurrent
- mutationStarted.resolve()
- await releaseMutation.promise
- }
- )
- await (watcher as any).startContextWatch('same-stream-context-rv')
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
-
- const oldReconcile = contextWatchCallback!('MODIFIED', oldContext)
- await mutationStarted.promise
- expect(oldLease?.()).toBe(true)
-
- const newReconcile = contextWatchCallback!('MODIFIED', newContext)
- expect(oldLease?.()).toBe(false)
-
- releaseMutation.resolve()
- await Promise.all([oldReconcile, newReconcile])
- await watcher.stop()
- })
-
- it('keeps live control-plane events responsive during a periodic external-egress fleet resync', async () => {
- const server = {
- name: 'periodic-egress-server',
- namespace: 'mcp-server',
- uid: 'periodic-egress-uid',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/periodic-egress:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'periodic.example', port: 443 }],
- },
- }
- const context = {
- metadata: { name: 'live-context', namespace: 'mcp-server' },
- spec: { contextId: 'live-context', mcpServers: [] },
- }
- const resyncStarted = deferred()
- const releaseResync = deferred()
- let contextWatchCallback: ((type: string, apiObj: typeof context) => Promise<void>) | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback) => {
- contextWatchCallback = callback
- return { abort: vi.fn() }
- })
- const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0)
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- ;(watcher as any).servers.set(server.name, server)
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- resyncStarted.resolve(undefined)
- await releaseResync.promise
- })
- await (watcher as any).startContextWatch('live-context-rv')
-
- const firstResync = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
- const overlappingResync = (
- watcher as any
- ).externalEgressCoordinator.runResync() as Promise<void>
- expect(overlappingResync).toBe(firstResync)
- await resyncStarted.promise
-
- const liveEvent = contextWatchCallback!('MODIFIED', context)
- await flushMicrotasks()
- expect(netPol.reconcileContext).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'live-context' }),
- expect.objectContaining({ isCurrent: expect.any(Function) })
- )
-
- releaseResync.resolve(undefined)
- await Promise.all([firstResync, liveEvent])
- await watcher.stop()
- randomSpy.mockRestore()
- })
-
- it('retires a blocked periodic egress lease when MODIFIED replaces it in the same watch', async () => {
- const oldServer = {
- name: 'same-stream-egress',
- namespace: 'mcp-server',
- uid: 'same-stream-egress-uid',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/same-stream-egress:old',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'old.example', port: 443 }],
- },
- }
- const newServer = {
- metadata: {
- name: oldServer.name,
- namespace: oldServer.namespace,
- uid: oldServer.uid,
- generation: 2,
- },
- spec: {
- ...oldServer.spec,
- image: 'clerum/same-stream-egress:new',
- egressBindings: [{ dns: 'new.example', port: 443 }],
- },
- }
- const mutationStarted = deferred()
- const releaseMutation = deferred()
- let oldLease: (() => boolean) | undefined
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- ;(watcher as any).servers.set(oldServer.name, oldServer)
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(
- async (_server: unknown, options: { isCurrent: () => boolean }) => {
- oldLease = options.isCurrent
- mutationStarted.resolve()
- await releaseMutation.promise
- }
- )
- vi.spyOn(Math, 'random').mockReturnValue(0)
-
- const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
- await mutationStarted.promise
- expect(oldLease?.()).toBe(true)
-
- const replacement = (watcher as any).getMcpServerWatchCallback()('MODIFIED', newServer)
- expect(oldLease?.()).toBe(false)
-
- releaseMutation.resolve()
- await Promise.all([periodic, replacement])
- await watcher.stop()
- })
-
- it('uses the latest cached McpServer after periodic external-egress jitter', async () => {
- vi.useFakeTimers()
- const staleServer = {
- name: 'periodic-egress-server',
- namespace: 'mcp-server',
- uid: 'periodic-egress-uid',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/periodic-egress:old',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'old.example', port: 443 }],
- },
- }
- const currentServer = {
- metadata: {
- name: staleServer.name,
- namespace: staleServer.namespace,
- uid: staleServer.uid,
- generation: 2,
- },
- spec: {
- ...staleServer.spec,
- image: 'clerum/periodic-egress:new',
- egressBindings: [{ dns: 'new.example', port: 443 }],
- },
- }
- const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5)
- const watcher = new McpServerWatcher()
- ;(watcher as any).servers.set(staleServer.name, staleServer)
- const netPol = (watcher as any).netPolReconciler
-
- const periodic = (watcher as any).externalEgressCoordinator.runResync() as Promise<void>
- await flushMicrotasks()
- await (watcher as any).getMcpServerWatchCallback()('MODIFIED', currentServer)
- await vi.advanceTimersByTimeAsync(2500)
- await periodic
-
- expect(netPol.reconcileExternalEgress).toHaveBeenLastCalledWith(
- expect.objectContaining({
- name: staleServer.name,
- generation: 2,
- spec: expect.objectContaining({
- egressBindings: [{ dns: 'new.example', port: 443 }],
- }),
- }),
- { isCurrent: expect.any(Function) }
- )
- await watcher.stop()
- randomSpy.mockRestore()
- })
-
- it('queues same-object SharedFileSystem and GlobalFileSystem watch effects behind initial passes', async () => {
- const staleSfs = { name: 'stale-sfs', namespace: 'mcp-host', spec: {} }
- const currentSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
- const staleGfs = { name: 'stale-gfs', namespace: 'mcp-host', spec: {} }
- const currentGfs = {
- metadata: { name: 'stale-gfs', namespace: 'mcp-host' },
- spec: {},
- status: {
- phase: 'Ready',
- pvcName: 'gfs-drive',
- serviceName: 'gfsc',
- serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',
- },
- }
- const sfsInitialStarted = deferred()
- const releaseSfsInitial = deferred()
- const gfsInitialStarted = deferred()
- const releaseGfsInitial = deferred()
- const sfsOrder: string[] = []
- const gfsOrder: string[] = []
- const watcher = new McpServerWatcher()
- ;(watcher as any).sharedFileSystems.set(staleSfs.name, staleSfs)
- ;(watcher as any).globalFileSystems.set(staleGfs.name, staleGfs)
-
- mocks.sfsFullReconcile.mockImplementationOnce(async () => {
- sfsInitialStarted.resolve(undefined)
- await releaseSfsInitial.promise
- sfsOrder.push('initial')
- })
- const sfsReconciler = (watcher as any).sharedFileSystemReconciler
- sfsReconciler.reconcile.mockImplementationOnce(async () => {
- sfsOrder.push('watch')
- })
- const gfsReconciler = (watcher as any).gfsReconciler
- vi.spyOn(gfsReconciler, 'fullReconcile').mockImplementationOnce(async () => {
- gfsInitialStarted.resolve(undefined)
- await releaseGfsInitial.promise
- gfsOrder.push('initial')
- })
- const gfsReconcile = vi.spyOn(gfsReconciler, 'reconcile').mockImplementationOnce(async () => {
- gfsOrder.push('watch')
- })
-
- let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined
- let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined
- mocks.watch
- .mockImplementationOnce(async (_path, _options, callback) => {
- sfsWatchCallback = callback
- return { abort: vi.fn() }
- })
- .mockImplementationOnce(async (_path, _options, callback) => {
- gfsWatchCallback = callback
- return { abort: vi.fn() }
- })
- await (watcher as any).startSharedFileSystemWatch()
- await (watcher as any).startGlobalFileSystemWatch()
-
- const sfsInitial = (watcher as any).runInitialSharedFileSystemConvergence()
- const gfsInitial = (watcher as any).runInitialGlobalFileSystemConvergence()
- await Promise.all([sfsInitialStarted.promise, gfsInitialStarted.promise])
- const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)
- const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)
-
- await flushMicrotasks()
- expect(sfsReconciler.reconcile).not.toHaveBeenCalled()
- expect(gfsReconcile).not.toHaveBeenCalled()
-
- releaseSfsInitial.resolve(undefined)
- releaseGfsInitial.resolve(undefined)
- await Promise.all([sfsInitial, gfsInitial, sfsWatch, gfsWatch])
-
- expect(sfsOrder).toEqual(['initial', 'watch'])
- expect(gfsOrder).toEqual(['initial', 'watch'])
- expect(gfsReconcile).toHaveBeenCalledWith(
- expect.objectContaining({ status: currentGfs.status })
- )
- await watcher.stop()
- })
-
- it('does not let a periodic SFS or GFS inventory overwrite a newer watch cache entry', async () => {
- const staleSfs = { metadata: { name: 'stale-sfs', namespace: 'mcp-host' }, spec: {} }
- const currentSfs = { metadata: { name: 'current-sfs', namespace: 'mcp-host' }, spec: {} }
- const staleGfs = { metadata: { name: 'stale-gfs', namespace: 'mcp-host' }, spec: {} }
- const currentGfs = { metadata: { name: 'current-gfs', namespace: 'mcp-host' }, spec: {} }
- const sfsListStarted = deferred()
- const releaseSfsList = deferred<unknown>()
- const gfsListStarted = deferred()
- const releaseGfsList = deferred<unknown>()
- const watcher = new McpServerWatcher()
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'sharedfilesystems') {
- sfsListStarted.resolve(undefined)
- return releaseSfsList.promise
- }
- if (plural === 'globalfilesystems') {
- gfsListStarted.resolve(undefined)
- return releaseGfsList.promise
- }
- return { items: [] }
- })
- const gfsReconciler = (watcher as any).gfsReconciler
- vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)
-
- let sfsWatchCallback: ((type: string, apiObj: typeof currentSfs) => Promise<void>) | undefined
- let gfsWatchCallback: ((type: string, apiObj: typeof currentGfs) => Promise<void>) | undefined
- mocks.watch
- .mockImplementationOnce(async (_path, _options, callback) => {
- sfsWatchCallback = callback
- return { abort: vi.fn() }
- })
- .mockImplementationOnce(async (_path, _options, callback) => {
- gfsWatchCallback = callback
- return { abort: vi.fn() }
- })
- await (watcher as any).startSharedFileSystemWatch()
- await (watcher as any).startGlobalFileSystemWatch()
-
- const sfsResync = (watcher as any).runSfsResync()
- const gfsResync = (watcher as any).runGfsResync()
- await Promise.all([sfsListStarted.promise, gfsListStarted.promise])
- const sfsWatch = sfsWatchCallback!('MODIFIED', currentSfs)
- const gfsWatch = gfsWatchCallback!('MODIFIED', currentGfs)
- releaseSfsList.resolve({ items: [staleSfs] })
- releaseGfsList.resolve({ items: [staleGfs] })
- await Promise.all([sfsResync, gfsResync, sfsWatch, gfsWatch])
-
- expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([
- expect.objectContaining({ name: 'current-sfs' }),
- ])
- expect(gfsReconciler.fullReconcile).toHaveBeenCalledWith([
- expect.objectContaining({ name: 'current-gfs' }),
- ])
- expect((watcher as any).sharedFileSystems.get('current-sfs')).toEqual(
- expect.objectContaining({ name: 'current-sfs' })
- )
- expect((watcher as any).sharedFileSystems.has('stale-sfs')).toBe(false)
- expect((watcher as any).globalFileSystems.get('current-gfs')).toEqual(
- expect.objectContaining({ name: 'current-gfs' })
- )
- expect((watcher as any).globalFileSystems.has('stale-gfs')).toBe(false)
- await watcher.stop()
- })
-
- it('refreshes SFS and GFS caches and effects from an uncontested periodic inventory', async () => {
- const listedSfs = {
- metadata: { name: 'listed-sfs', namespace: 'mcp-host' },
- spec: { capacity: '1Gi' },
- }
- const listedGfs = {
- metadata: { name: 'listed-gfs', namespace: 'gfs' },
- spec: { capacity: '10Gi' },
- }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'sharedfilesystems') return { items: [listedSfs] }
- if (plural === 'globalfilesystems') return { items: [listedGfs] }
- return { items: [] }
- })
- const watcher = new McpServerWatcher()
- const gfsReconciler = (watcher as any).gfsReconciler
- const gfsFullReconcile = vi.spyOn(gfsReconciler, 'fullReconcile').mockResolvedValue(undefined)
-
- await Promise.all([(watcher as any).runSfsResync(), (watcher as any).runGfsResync()])
-
- expect((watcher as any).sharedFileSystems.get('listed-sfs')).toEqual(
- expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec })
- )
- expect((watcher as any).globalFileSystems.get('listed-gfs')).toEqual(
- expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec })
- )
- expect(mocks.sfsFullReconcile).toHaveBeenCalledWith([
- expect.objectContaining({ name: 'listed-sfs', spec: listedSfs.spec }),
- ])
- expect(gfsFullReconcile).toHaveBeenCalledWith([
- expect.objectContaining({ name: 'listed-gfs', spec: listedGfs.spec }),
- ])
- await watcher.stop()
- })
-
- it('does not run queued fleet effects after the watcher stops', async () => {
- const initialServer = {
- name: 'initial-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/initial-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'initial.example', port: 443 }],
- },
- }
- const queuedServer = {
- metadata: { name: 'queued-server', namespace: 'mcp-server' },
- spec: {
- contextRef: 'default',
- image: 'clerum/queued-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).servers.set(initialServer.name, initialServer)
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- const reconciler = (watcher as any).reconciler
- mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
- await Promise.all(
- servers.map((selected: { name: string }) =>
- options.runEffect(selected.name, () => reconciler.reconcile(selected))
- )
- )
- })
-
- const initial = (watcher as any).runInitialMcpServerConvergence()
- await egressStarted.promise
- const queuedWatch = (watcher as any).getMcpServerWatchCallback()('MODIFIED', queuedServer)
-
- await watcher.stop()
- releaseEgress.resolve(undefined)
- await Promise.all([initial, queuedWatch])
-
- expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).not.toHaveBeenCalled()
- })
-
- it('coalesces overlapping McpServer fleet passes into one trailing current-cache pass', async () => {
- const firstPassStarted = deferred()
- const releaseFirstPass = deferred()
- const snapshots: string[][] = []
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).servers.set('first-server', {
- name: 'first-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/first-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- })
- mocks.serverFullReconcile
- .mockImplementationOnce(async (servers: Array<{ name: string }>) => {
- snapshots.push(servers.map(server => server.name))
- firstPassStarted.resolve(undefined)
- await releaseFirstPass.promise
- })
- .mockImplementationOnce(async (servers: Array<{ name: string }>) => {
- snapshots.push(servers.map(server => server.name))
- })
-
- const first = (watcher as any).runInitialMcpServerConvergence()
- await firstPassStarted.promise
- ;(watcher as any).servers.set('second-server', {
- name: 'second-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/second-server:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- })
- const second = (watcher as any).runInitialMcpServerConvergence()
- const third = (watcher as any).runInitialMcpServerConvergence()
-
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
- releaseFirstPass.resolve(undefined)
- await Promise.all([first, second, third])
-
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
- expect(snapshots).toEqual([['first-server'], ['first-server', 'second-server']])
- await watcher.stop()
- })
-
- it('coalesces overlapping NetworkPolicy fleet passes into one trailing current-cache pass', async () => {
- const firstPassStarted = deferred()
- const releaseFirstPass = deferred()
- const snapshots: string[][] = []
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contexts.set('first-context', {
- name: 'first-context',
- namespace: 'mcp-server',
- spec: { contextId: 'first-context', mcpServers: [] },
- })
- mocks.netPolFullReconcile
- .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {
- snapshots.push(contexts.map(context => context.name))
- firstPassStarted.resolve(undefined)
- await releaseFirstPass.promise
- })
- .mockImplementationOnce(async (contexts: Array<{ name: string }>) => {
- snapshots.push(contexts.map(context => context.name))
- })
-
- const first = (watcher as any).runInitialNetworkPolicyConvergence()
- await firstPassStarted.promise
- ;(watcher as any).contexts.set('second-context', {
- name: 'second-context',
- namespace: 'mcp-server',
- spec: { contextId: 'second-context', mcpServers: [] },
- })
- const second = (watcher as any).runInitialNetworkPolicyConvergence()
- const third = (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
- releaseFirstPass.resolve(undefined)
- await Promise.all([first, second, third])
-
- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
- expect(snapshots).toEqual([['first-context'], ['first-context', 'second-context']])
- await watcher.stop()
- })
-
- it('does not lose an initial convergence request queued in the settlement microtask', async () => {
- const watcher = new McpServerWatcher()
- let trailing: Promise<void> | undefined
- const convergenceCore = vi
- .fn()
- .mockImplementationOnce(
- () =>
- ({
- then: (resolve: () => void) => {
- resolve()
- queueMicrotask(() => {
- trailing = (watcher as any).runInitialMcpServerConvergence()
- })
- },
- }) as PromiseLike<void>
- )
- .mockResolvedValueOnce(undefined)
- ;(watcher as any).runInitialMcpServerConvergenceCore = convergenceCore
-
- const first = (watcher as any).runInitialMcpServerConvergence()
- await first
- await vi.waitFor(() => expect(trailing).toBeDefined())
- await trailing
-
- expect(convergenceCore).toHaveBeenCalledTimes(2)
- await watcher.stop()
- })
-
- it('retains one keyed queue until every same-key waiter has drained', async () => {
- const watcher = new McpServerWatcher()
- const firstStarted = deferred()
- const releaseFirst = deferred()
- const effects: string[] = []
- const queues = (watcher as any).contextReconciliationQueues as Map<
- string,
- { references: number }
- >
-
- const first = (watcher as any).enqueueContextReconciliation('shared', async () => {
- effects.push('first')
- firstStarted.resolve(undefined)
- await releaseFirst.promise
- })
- await firstStarted.promise
- const second = (watcher as any).enqueueContextReconciliation('shared', async () => {
- effects.push('second')
- })
-
- expect(queues.get('shared')?.references).toBe(2)
- expect(effects).toEqual(['first'])
- releaseFirst.resolve(undefined)
- await Promise.all([first, second])
- expect(effects).toEqual(['first', 'second'])
- expect(queues.size).toBe(0)
- await watcher.stop()
- })
-
- it('does not execute newly enqueued keyed work after stop', async () => {
- const watcher = new McpServerWatcher()
- const work = vi.fn()
- ;(watcher as any).stopped = true
-
- await (watcher as any).enqueueContextReconciliation('stopped', work)
-
- expect(work).not.toHaveBeenCalled()
- expect((watcher as any).contextReconciliationQueues.size).toBe(0)
- })
-
- it('acquires multi-context identity keys in one deterministic sorted order', async () => {
- const watcher = new McpServerWatcher()
- const acquired: string[] = []
- vi.spyOn(watcher as any, 'enqueueContextReconciliation').mockImplementation(
- async (...args: unknown[]) => {
- const [key, work] = args as [string, () => Promise<void>]
- acquired.push(key)
- await work()
- }
- )
-
- await (watcher as any).enqueueContextIdentityReconciliation(
- ['z-context', 'a-context', 'm-context', 'z-context'],
- async () => undefined
- )
-
- expect(acquired).toEqual(['a-context', 'm-context', 'z-context'])
- await watcher.stop()
- })
-
- it('fences an McpServer fleet pass when its LIST-to-WATCH authority is retired', async () => {
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpWatchGeneration = 7
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 3)
- ;(watcher as any).servers.set('stale-server', {
- name: 'stale-server',
- namespace: 'mcp-server',
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/stale-server:v1',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'stale.example', port: 443 }],
- },
- })
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
- await Promise.all(
- servers.map((selected: { name: string }) =>
- options.runEffect(selected.name, () => reconciler.reconcile(selected))
- )
- )
- })
-
- const stalePass = (watcher as any).runInitialMcpServerConvergence()
- await egressStarted.promise
- ;(watcher as any).mcpServerCacheSynced = false
- ;(watcher as any).mcpWatchGeneration = 8
- const trailing = (watcher as any).runInitialMcpServerConvergence()
- releaseEgress.resolve(undefined)
- await Promise.all([stalePass, trailing])
-
- expect(reconciler.reconcile).not.toHaveBeenCalled()
- expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(3)
- ;(watcher as any).servers.set('stale-server', {
- name: 'stale-server',
- namespace: 'mcp-server',
- generation: 2,
- spec: {
- contextRef: 'default',
- image: 'clerum/stale-server:v2',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'current.example', port: 443 }],
- },
- })
- ;(watcher as any).mcpServerCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- await (watcher as any).runInitialMcpServerConvergence()
-
- expect(reconciler.reconcile).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({
- generation: 2,
- spec: expect.objectContaining({ image: 'clerum/stale-server:v2' }),
- })
- )
- expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)
- await watcher.stop()
- })
-
- it('fences NetworkPolicy positive effects when their inventory leases are retired', async () => {
- const firstPassStarted = deferred()
- const releaseFirstPass = deferred()
- const appliedContexts: string[] = []
- const appliedServers: string[] = []
- let pass = 0
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 11
- ;(watcher as any).mcpWatchGeneration = 13
- ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 4)
- ;(watcher as any).contexts.set('stale-context', {
- name: 'stale-context',
- namespace: 'mcp-server',
- spec: { contextId: 'stale-context', mcpServers: ['stale-server'] },
- })
- ;(watcher as any).servers.set('stale-server', {
- name: 'stale-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'stale-context',
- image: 'clerum/stale-server:v1',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- })
- mocks.netPolFullReconcile.mockImplementation(async (contexts, servers, options) => {
- pass += 1
- if (pass === 1) {
- firstPassStarted.resolve(undefined)
- await releaseFirstPass.promise
- }
- await Promise.all(
- contexts.map((context: { spec: { contextId: string } }) =>
- options.runContextEffect(context.spec.contextId, async () => {
- appliedContexts.push(context.spec.contextId)
- })
- )
- )
- await Promise.all(
- servers.map((server: { name: string }) =>
- options.runServerEffect(server.name, async () => {
- appliedServers.push(server.name)
- })
- )
- )
- })
-
- const stalePass = (watcher as any).runInitialNetworkPolicyConvergence()
- await firstPassStarted.promise
- ;(watcher as any).contextCacheSynced = false
- ;(watcher as any).mcpServerCacheSynced = false
- ;(watcher as any).contextWatchGeneration = 12
- ;(watcher as any).mcpWatchGeneration = 14
- const trailing = (watcher as any).runInitialNetworkPolicyConvergence()
- releaseFirstPass.resolve(undefined)
- await Promise.all([stalePass, trailing])
-
- expect(appliedContexts).toEqual([])
- expect(appliedServers).toEqual([])
- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(4)
- ;(watcher as any).contexts.clear()
- ;(watcher as any).servers.clear()
- ;(watcher as any).contexts.set('current-context', {
- name: 'current-context',
- namespace: 'mcp-server',
- spec: { contextId: 'current-context', mcpServers: ['current-server'] },
- })
- ;(watcher as any).servers.set('current-server', {
- name: 'current-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'current-context',
- image: 'clerum/current-server:v2',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- })
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- await (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect(appliedContexts).toEqual(['current-context'])
- expect(appliedServers).toEqual(['current-server'])
- expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
- await watcher.stop()
- })
-
- it('does not certify NetworkPolicy revocation after authority is lost at the callback boundary', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 21
- ;(watcher as any).mcpWatchGeneration = 34
- mocks.netPolFullReconcile.mockImplementationOnce(async (_contexts, _servers, options) => {
- ;(watcher as any).contextCacheSynced = false
- options.onAuthoritativeRevocationComplete()
- })
-
- await (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect((watcher as any).networkPolicyRevocationContextGeneration).not.toBe(21)
- expect((watcher as any).networkPolicyRevocationServerGeneration).not.toBe(34)
- await watcher.stop()
- })
-
- it('wires onExternalEgressRevoked to schedule a MODIFIED external-egress retry (B3)', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 21
- ;(watcher as any).mcpWatchGeneration = 34
- const server = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
- const retrySpy = vi
- .spyOn((watcher as any).externalEgressCoordinator, 'scheduleRetry')
- .mockImplementation(() => undefined)
- let captured: { onExternalEgressRevoked?: (s: unknown) => void } | undefined
- mocks.netPolFullReconcile.mockImplementationOnce(
- async (_contexts: unknown[], _servers: unknown[], options: typeof captured) => {
- captured = options
- }
- )
-
- await (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect(captured?.onExternalEgressRevoked).toBeTypeOf('function')
- captured!.onExternalEgressRevoked!(server)
- expect(retrySpy).toHaveBeenCalledWith('MODIFIED', server)
- await watcher.stop()
- })
-
- it('does not start runtime fleet convergence without a current NetworkPolicy safety certificate', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 4
- ;(watcher as any).mcpWatchGeneration = 7
-
- await (watcher as any).runInitialMcpServerConvergence()
- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
-
- await (watcher as any).runInitialNetworkPolicyConvergence()
- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
-
- await watcher.stop()
- })
-
- it('does not start runtime fleet convergence while the safety inventory is uncertified', async () => {
- // M1: with the revocation counters aligned, the ONLY thing that can null the
- // certificate is the lost safety fence. Fence lost -> cert null -> the fleet
- // lane must not run; fence restored -> it runs exactly once.
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 4
- ;(watcher as any).mcpWatchGeneration = 7
- markNetworkPolicyRevocationAuthoritative(watcher)
- try {
- mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
- await (watcher as any).runInitialMcpServerConvergence()
- expect(mocks.serverFullReconcile).not.toHaveBeenCalled()
-
- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
- await (watcher as any).runInitialMcpServerConvergence()
- expect(mocks.serverFullReconcile).toHaveBeenCalledOnce()
- } finally {
- // vi.clearAllMocks() clears calls, not implementations — restore explicitly
- // or every later test would re-gate on a stale false.
- mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
- await watcher.stop()
- }
- })
-
- it('invalidates readiness when either desired policy revision advances', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 4
- ;(watcher as any).mcpWatchGeneration = 7
- markNetworkPolicyRevocationAuthoritative(watcher)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- ;(watcher as any).mcpServerDesiredRevision += 1
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
-
- markNetworkPolicyRevocationAuthoritative(watcher)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- ;(watcher as any).contextDesiredRevision += 1
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
-
- await watcher.stop()
- })
-
- it('retires readiness and requests a safety pass before a changed McpServer effect settles', async () => {
- const previous = {
- metadata: {
- name: 'restrictive-change',
- namespace: 'mcp-server',
- uid: 'restrictive-change-uid',
- generation: 1,
- },
- spec: {
- contextRef: 'default',
- image: 'clerum/restrictive-change:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' as const }],
- },
- }
- const current = {
- ...previous,
- metadata: { ...previous.metadata, generation: 2 },
- spec: {
- ...previous.spec,
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' as const }],
- },
- }
- const watcher = new McpServerWatcher()
- ;(watcher as any).hostCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 4
- ;(watcher as any).mcpWatchGeneration = 7
- ;(watcher as any).servers.set(previous.metadata.name, {
- name: previous.metadata.name,
- namespace: previous.metadata.namespace,
- uid: previous.metadata.uid,
- generation: previous.metadata.generation,
- spec: previous.spec,
- })
- markNetworkPolicyRevocationAuthoritative(watcher)
- const safetyPass = vi
- .spyOn(watcher as any, 'runInitialNetworkPolicyConvergence')
- .mockResolvedValue(undefined)
- ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockRejectedValueOnce(
- new Error('replacement policy unavailable')
- )
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
-
- await (watcher as any).getMcpServerWatchCallback(7)('MODIFIED', current)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect(safetyPass).toHaveBeenCalled()
- await watcher.stop()
- errorSpy.mockRestore()
- })
-
- it('fences runtime effects when either generation retires the safety certificate', async () => {
- const selected = {
- name: 'certificate-fenced-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'clerum/certificate-fenced:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'certificate-fenced.example', port: 443 }],
- },
- }
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const runtimeEffect = vi.fn()
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 11
- ;(watcher as any).mcpWatchGeneration = 13
- markNetworkPolicyRevocationAuthoritative(watcher)
- ;(watcher as any).servers.set(selected.name, selected)
- ;(watcher as any).netPolReconciler.reconcileExternalEgress.mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- mocks.serverFullReconcile.mockImplementationOnce(async (servers, options) => {
- await Promise.all(
- servers.map((server: { name: string }) => options.runEffect(server.name, runtimeEffect))
- )
- })
-
- const convergence = (watcher as any).runInitialMcpServerConvergence()
- await egressStarted.promise
- ;(watcher as any).contextWatchGeneration = 12
- releaseEgress.resolve(undefined)
- await convergence
-
- expect(runtimeEffect).not.toHaveBeenCalled()
- await watcher.stop()
- })
-
- it.each([
- ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],
- ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],
- ] as const)(
- 'does not admit queued %s effects after the watcher stops',
- async (_, enqueueMethod) => {
- const firstStarted = deferred()
- const releaseFirst = deferred()
- const watcher = new McpServerWatcher()
- let queuedEffectRan = false
-
- const first = (watcher as any)[enqueueMethod]('same-object', async () => {
- firstStarted.resolve(undefined)
- await releaseFirst.promise
- }) as Promise<void>
- await firstStarted.promise
- const queued = (watcher as any)[enqueueMethod]('same-object', async () => {
- queuedEffectRan = true
- }) as Promise<void>
-
- await watcher.stop()
- releaseFirst.resolve(undefined)
- await Promise.all([first, queued])
-
- expect(queuedEffectRan).toBe(false)
- }
- )
-
- it.each([
- ['SharedFileSystem', 'enqueueSharedFileSystemReconciliation'],
- ['GlobalFileSystem', 'enqueueGlobalFileSystemReconciliation'],
- ] as const)(
- 'serializes the same %s identity without blocking an unrelated fleet member',
- async (_, enqueueMethod) => {
- const firstStarted = deferred()
- const releaseFirst = deferred()
- const secondStarted = deferred()
- const unrelatedStarted = deferred()
- const watcher = new McpServerWatcher()
- let secondRan = false
-
- const first = (watcher as any)[enqueueMethod]('slow', async () => {
- firstStarted.resolve(undefined)
- await releaseFirst.promise
- }) as Promise<void>
- await firstStarted.promise
- const second = (watcher as any)[enqueueMethod]('slow', async () => {
- secondRan = true
- secondStarted.resolve(undefined)
- }) as Promise<void>
- const unrelated = (watcher as any)[enqueueMethod]('independent', async () => {
- unrelatedStarted.resolve(undefined)
- }) as Promise<void>
-
- await unrelatedStarted.promise
- expect(secondRan).toBe(false)
- releaseFirst.resolve(undefined)
- await Promise.all([first, second, unrelated])
- await expect(secondStarted.promise).resolves.toBeUndefined()
- await watcher.stop()
- }
- )
-
- it('keeps a keyed lane alive until a third waiter has consumed the same queue entry', async () => {
- const firstStarted = deferred()
- const releaseFirst = deferred()
- const secondStarted = deferred()
- const releaseSecond = deferred()
- const thirdStarted = deferred()
- let thirdRan = false
- const watcher = new McpServerWatcher()
-
- const first = (watcher as any).enqueueContextReconciliation('shared', async () => {
- firstStarted.resolve(undefined)
- await releaseFirst.promise
- })
- await firstStarted.promise
- const second = (watcher as any).enqueueContextReconciliation('shared', async () => {
- secondStarted.resolve(undefined)
- await releaseSecond.promise
- })
- const third = (watcher as any).enqueueContextReconciliation('shared', async () => {
- thirdRan = true
- thirdStarted.resolve(undefined)
- })
-
- expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(3)
- releaseFirst.resolve(undefined)
- await secondStarted.promise
- expect(thirdRan).toBe(false)
- expect((watcher as any).contextReconciliationQueues.get('shared').references).toBe(2)
- releaseSecond.resolve(undefined)
- await Promise.all([first, second, third])
- expect((watcher as any).contextReconciliationQueues.has('shared')).toBe(false)
- await watcher.stop()
- })
-
- it('caps initial convergence retry delay after attempts exceed the schedule', async () => {
- vi.useFakeTimers()
- const watcher = new McpServerWatcher()
- const convergence = vi
- .spyOn(watcher as any, 'runInitialMcpServerConvergence')
- .mockResolvedValue(undefined)
- ;(watcher as any).initialConvergenceRetryAttempts.set('McpServer', 99)
- ;(watcher as any).scheduleInitialConvergenceRetry('McpServer')
- await vi.advanceTimersByTimeAsync(299_999)
- expect(convergence).not.toHaveBeenCalled()
- await vi.advanceTimersByTimeAsync(1)
- expect(convergence).toHaveBeenCalledOnce()
- await watcher.stop()
- })
-
- it('defers NetworkPolicy full convergence until both inventories are authoritative', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = false
- ;(watcher as any).initialConvergenceRetryAttempts.set('NetworkPolicy', 2)
-
- await (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- expect((watcher as any).initialConvergenceRetryAttempts.get('NetworkPolicy')).toBe(2)
- ;(watcher as any).mcpServerCacheSynced = true
- await (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
- expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
- [],
- [],
- expect.objectContaining({
- serverInventoryComplete: true,
- ensureDefaults: false,
- contextInventoryAuthoritative: expect.any(Function),
- serverInventoryAuthoritative: expect.any(Function),
- })
- )
- expect((watcher as any).initialConvergenceRetryAttempts.has('NetworkPolicy')).toBe(false)
- await watcher.stop()
- })
-
- it('fences NetworkPolicy orphan cleanup to the inventory generations captured by the pass', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).contextCacheSynced = true
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextWatchGeneration = 17
- ;(watcher as any).mcpWatchGeneration = 23
- let options:
- | {
- contextInventoryAuthoritative: () => boolean
- serverInventoryAuthoritative: () => boolean
- }
- | undefined
- mocks.netPolFullReconcile.mockImplementationOnce(
- async (
- _contexts: unknown[],
- _servers: unknown[],
- received: {
- contextInventoryAuthoritative: () => boolean
- serverInventoryAuthoritative: () => boolean
- }
- ) => {
- options = received
- }
- )
-
- await (watcher as any).runInitialNetworkPolicyConvergence()
-
- expect(options?.contextInventoryAuthoritative()).toBe(true)
- expect(options?.serverInventoryAuthoritative()).toBe(true)
-
- // A recovered watch may already be authoritative again, but it is a
- // different snapshot generation and cannot authorize cleanup by this pass.
- ;(watcher as any).contextWatchGeneration = 18
- ;(watcher as any).mcpWatchGeneration = 24
-
- expect(options?.contextInventoryAuthoritative()).toBe(false)
- expect(options?.serverInventoryAuthoritative()).toBe(false)
- await watcher.stop()
- })
-
- it('escalates and resets failed initial McpServer convergence without delaying bootstrap', async () => {
- vi.useFakeTimers()
- const server = {
- metadata: { name: 'initial-runtime', namespace: 'mcp-server' },
- spec: {
- contextRef: 'default',
- image: 'clerum/initial-runtime:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- },
- }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [server] }
- if (plural === 'contexts') return { items: [] }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'mcp-retry-rv' }, items: [] }
- }
- return { items: [] }
- })
- mocks.serverFullReconcile
- .mockRejectedValueOnce(new Error('initial runtime reconciliation failed'))
- .mockRejectedValueOnce(new Error('initial runtime reconciliation still unavailable'))
- .mockResolvedValueOnce(undefined)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const retryMetricBefore = await readInitialConvergenceMetric(
- 'clerum_hcc_initial_convergence_retries_total',
- 'McpServer'
- )
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
-
- await watcher.start()
- await vi.advanceTimersByTimeAsync(0)
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
-
- await vi.advanceTimersByTimeAsync(5000)
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
-
- await vi.advanceTimersByTimeAsync(14_999)
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(2)
- await vi.advanceTimersByTimeAsync(1)
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(3)
- expect((watcher as any).initialConvergenceRetryAttempts.has('McpServer')).toBe(false)
- expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(false)
- expect(
- await readInitialConvergenceMetric(
- 'clerum_hcc_initial_convergence_retries_total',
- 'McpServer'
- )
- ).toBe(retryMetricBefore + 2)
- expect(
- await readInitialConvergenceMetric(
- 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
- 'McpServer'
- )
- ).toBeGreaterThan(0)
-
- await watcher.stop()
- errorSpy.mockRestore()
- })
-
- it('cancels a pending initial convergence retry when the watcher stops', async () => {
- vi.useFakeTimers()
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- ;(watcher as any).contextCacheSynced = true
- markNetworkPolicyRevocationAuthoritative(watcher)
- mocks.serverFullReconcile.mockRejectedValueOnce(
- new Error('initial runtime reconciliation unavailable')
- )
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
-
- await (watcher as any).runInitialMcpServerConvergence()
-
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
- expect((watcher as any).initialConvergenceRetryAttempts.get('McpServer')).toBe(1)
- expect((watcher as any).initialConvergenceRetryTimers.has('McpServer')).toBe(true)
-
- await watcher.stop()
- await vi.advanceTimersByTimeAsync(300_000)
-
- expect(mocks.serverFullReconcile).toHaveBeenCalledTimes(1)
- expect((watcher as any).initialConvergenceRetryAttempts.size).toBe(0)
- expect((watcher as any).initialConvergenceRetryTimers.size).toBe(0)
- errorSpy.mockRestore()
- })
-
- it('retries failed initial NetworkPolicy convergence from the current caches', async () => {
- vi.useFakeTimers()
- const context = {
- metadata: { name: 'initial-context', namespace: 'mcp-server' },
- spec: { contextId: 'initial-context', mcpServers: [] },
- }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [] }
- if (plural === 'contexts') return { items: [context] }
- if (plural === 'communicationchannels') {
- return { metadata: { resourceVersion: 'netpol-retry-rv' }, items: [] }
- }
- return { items: [] }
- })
- mocks.netPolFullReconcile
- .mockRejectedValueOnce(new Error('initial NetworkPolicy reconciliation failed'))
- .mockImplementationOnce(
- async (
- _contexts: unknown,
- _servers: unknown,
- options?: { onAuthoritativeRevocationComplete?: () => void }
- ) => {
- options?.onAuthoritativeRevocationComplete?.()
- }
- )
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
-
- await watcher.start()
- await vi.advanceTimersByTimeAsync(0)
- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(1)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
-
- await vi.advanceTimersByTimeAsync(5000)
- expect(mocks.netPolFullReconcile).toHaveBeenCalledTimes(2)
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- expect(mocks.netPolFullReconcile).toHaveBeenLastCalledWith(
- [expect.objectContaining({ name: 'initial-context' })],
- [],
- expect.objectContaining({
- serverInventoryComplete: true,
- ensureDefaults: false,
- contextInventoryAuthoritative: expect.any(Function),
- serverInventoryAuthoritative: expect.any(Function),
- })
- )
-
- await watcher.stop()
- errorSpy.mockRestore()
- })
-
- it('does not report a post-certification additive failure as a readiness safety failure', async () => {
- vi.useFakeTimers()
- const additiveFailure = new AggregateError(
- [new Error('Context policy API unavailable')],
- 'One or more additive Context NetworkPolicy reconciliations failed'
- )
- mocks.netPolFullReconcile.mockImplementationOnce(
- async (
- _contexts: unknown,
- _servers: unknown,
- options?: { onAuthoritativeRevocationComplete?: () => void }
- ) => {
- options?.onAuthoritativeRevocationComplete?.()
- throw additiveFailure
- }
- )
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
-
- await watcher.start()
- await vi.advanceTimersByTimeAsync(0)
-
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- expect(errorSpy).toHaveBeenCalledWith(
- '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:',
- additiveFailure
- )
- expect(errorSpy).not.toHaveBeenCalledWith(
- '[K8s] Initial NetworkPolicy background reconciliation failed:',
- additiveFailure
- )
- expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(true)
-
- await watcher.stop()
- errorSpy.mockRestore()
- })
-
- it.each([
- ['McpServer', 'startMcpServerWatch'],
- ['Context', 'startContextWatch'],
- ] as const)(
- 'keeps safe bootstrap authority-gated when the %s watch cannot be established',
- async (_kind, method) => {
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- if (method === 'startMcpServerWatch') {
- vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))
- } else {
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- }
- if (method === 'startContextWatch') {
- vi.spyOn(watcher as any, method).mockRejectedValue(new Error(`${method} unavailable`))
- } else if (method !== 'startMcpServerWatch') {
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- }
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
- const recoveryTimerField =
- method === 'startMcpServerWatch'
- ? 'mcpServerCacheRecoveryTimer'
- : 'contextCacheRecoveryTimer'
-
- try {
- await expect(watcher.start()).resolves.toBeUndefined()
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(false)
- expect((watcher as any)[recoveryTimerField]).not.toBeNull()
- expect(mocks.netPolFullReconcile).not.toHaveBeenCalled()
- } finally {
- await watcher.stop()
- errorSpy.mockRestore()
- warnSpy.mockRestore()
- }
- }
- )
-
- it.each([
- ['SharedFileSystem', 'startSharedFileSystemWatch'],
- ['GlobalFileSystem', 'startGlobalFileSystemWatch'],
- ] as const)(
- 'isolates, logs, and retries an initial %s background watch rejection',
- async (kind, method) => {
- vi.useFakeTimers()
- const failure = new Error(`${method} unavailable`)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- const rejectedWatch = vi
- .spyOn(watcher as any, method)
- .mockRejectedValueOnce(failure)
- .mockResolvedValueOnce(undefined)
- const otherMethod =
- method === 'startSharedFileSystemWatch'
- ? 'startGlobalFileSystemWatch'
- : 'startSharedFileSystemWatch'
- vi.spyOn(watcher as any, otherMethod).mockResolvedValue(undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockResolvedValue(undefined)
-
- await expect(watcher.start()).resolves.toBeUndefined()
- await vi.advanceTimersByTimeAsync(0)
-
- expect(errorSpy).toHaveBeenCalledWith(
- `[K8s] ${kind} background watch failed to start:`,
- failure
- )
- expect(rejectedWatch).toHaveBeenCalledOnce()
- expect(mocks.netPolFullReconcile).toHaveBeenCalledOnce()
-
- await vi.advanceTimersByTimeAsync(4_999)
- expect(rejectedWatch).toHaveBeenCalledOnce()
- await vi.advanceTimersByTimeAsync(1)
- expect(rejectedWatch).toHaveBeenCalledTimes(2)
-
- await watcher.stop()
- errorSpy.mockRestore()
- }
- )
-
- it('applies the request deadline wrapper to the finite Host inventory LIST', async () => {
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') return { items: [] }
- return { items: [] }
- })
-
- await expect(listAllHosts()).resolves.toEqual([])
-
- expect(mocks.hostListCallOptions).toHaveBeenCalledWith(
- expect.objectContaining({
- middleware: [expect.objectContaining({ pre: expect.any(Function) })],
- middlewareMergeStrategy: 'append',
- })
- )
- })
-
- it('preserves Host UID from the Kubernetes inventory boundary', async () => {
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- return {
- items: [
- {
- metadata: {
- name: 'uid-host',
- namespace: 'mcp-host',
- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
- generation: 4,
- },
- spec: { host: 'uid-host', contextRef: 'context-a', secretRef: 'host-secret' },
- },
- ],
- }
- }
- return { items: [] }
- })
-
- await expect(listAllHosts()).resolves.toEqual([
- expect.objectContaining({
- name: 'uid-host',
- uid: '9f43826a-4031-4a31-93af-a3dd8fcfe805',
- generation: 4,
- }),
- ])
- })
-
- it('preserves observed GlobalFileSystem status from the Kubernetes inventory boundary', async () => {
- const status = {
- phase: 'Ready',
- pvcName: 'gfs-drive',
- serviceName: 'gfsc',
- serviceUrl: 'http://gfsc.gfs.svc.cluster.local:8087',
- }
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'globalfilesystems') {
- return {
- items: [
- {
- metadata: { name: 'gfs', namespace: 'gfs' },
- spec: {},
- status,
- },
- ],
- }
- }
- return { items: [] }
- })
-
- await expect(listAllGlobalFileSystems()).resolves.toEqual([
- expect.objectContaining({ name: 'gfs', namespace: 'gfs', status }),
- ])
- })
-
- it('reconciles startup external egress before runtime full reconciliation', async () => {
- const eventLog: string[] = []
- const server = {
- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
- spec: {
- contextRef: 'wf-research',
- image: 'clerum/web-search:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
- },
- }
-
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [server] }
- if (plural === 'contexts') return { items: [] }
- if (plural === 'hosts') return { items: [] }
- if (plural === 'sharedfilesystems') return { items: [] }
- if (plural === 'communicationchannels')
- return { metadata: { resourceVersion: '1' }, items: [] }
- return { items: [] }
- })
- mocks.ensureDefaultPolicies.mockImplementation(async () => {
- eventLog.push('defaults')
- })
- const watcher = new McpServerWatcher()
- const reconciler = (watcher as any).reconciler
- mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
- await Promise.all(
- servers.map((selected: { name: string }) =>
- options.runEffect(selected.name, async () => {
- eventLog.push('runtime')
- await reconciler.reconcile(selected)
- })
- )
- )
- })
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockImplementation(async () => {
- eventLog.push('egress')
- })
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
- async () => undefined
- )
-
- await watcher.start()
-
- await vi.waitFor(() => expect(eventLog).toContain('runtime'))
- expect(eventLog.slice(0, 3)).toEqual(['defaults', 'egress', 'runtime'])
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith(
- [expect.objectContaining({ name: 'web-search' })],
- { runEffect: expect.any(Function) }
- )
-
- watcher.stop()
- })
-
- it('skips startup runtime reconciliation for servers whose external egress is not ready', async () => {
- vi.useFakeTimers()
- const server = {
- metadata: { name: 'web-search', namespace: 'mcp-server', generation: 4 },
- spec: {
- contextRef: 'wf-research',
- image: 'clerum/web-search:test',
- transport: { type: 'streamableHttp' as const, port: 3000 },
- egressBindings: [{ dns: 'duckduckgo.com', port: 443 }],
- },
- }
-
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'mcpservers') return { items: [server] }
- if (plural === 'contexts') return { items: [] }
- if (plural === 'hosts') return { items: [] }
- if (plural === 'sharedfilesystems') return { items: [] }
- if (plural === 'communicationchannels')
- return { metadata: { resourceVersion: '1' }, items: [] }
- return { items: [] }
- })
-
- const watcher = new McpServerWatcher()
- const reconciler = (watcher as any).reconciler
- mocks.serverFullReconcile.mockImplementation(async (servers, options) => {
- await Promise.all(
- servers.map((selected: { name: string }) =>
- options.runEffect(selected.name, () => reconciler.reconcile(selected))
- )
- )
- })
- const netPol = (watcher as any).netPolReconciler
- const dnsFailure = new Error('dns resolution failed')
- netPol.reconcileExternalEgress.mockRejectedValueOnce(dnsFailure)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
- vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
- vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
- async () => undefined
- )
+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startContextWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockImplementation(async () => undefined)
+ vi.spyOn(watcher as any, 'startCommunicationChannelWatch').mockImplementation(
+ async () => undefined
+ )
await watcher.start()
- await vi.waitFor(() => expect(mocks.serverFullReconcile).toHaveBeenCalledOnce())
- expect(mocks.serverFullReconcile).toHaveBeenCalledWith(
- [expect.objectContaining({ name: 'web-search' })],
- { runEffect: expect.any(Function) }
- )
+ expect(mocks.serverFullReconcile).toHaveBeenCalledWith([])
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
- expect(reconciler.reconcile).not.toHaveBeenCalled()
- expect(watcher.isReadinessInventoryAuthoritative()).toBe(true)
- expect((watcher as any).initialConvergenceRetryTimers.has('NetworkPolicy')).toBe(false)
- expect(errorSpy).not.toHaveBeenCalledWith(
- '[K8s] Initial NetworkPolicy background reconciliation failed:',
- dnsFailure
- )
watcher.stop()
errorSpy.mockRestore()
@@ -4627,29 +448,6 @@ describe('McpServerWatcher startup', () => {
})
})
-describe('getContext error semantics', () => {
- beforeEach(() => {
- vi.clearAllMocks()
- })
-
- it('preserves not-found semantics only for an authoritative 404', async () => {
- mocks.getNamespacedCustomObject.mockRejectedValueOnce(
- Object.assign(new Error('context absent'), { code: 404 })
- )
-
- await expect(getContext('missing-context')).resolves.toBeNull()
- })
-
- it('propagates non-404 API failures instead of converting discovery to an empty fleet', async () => {
- const unavailable = Object.assign(new Error('apiserver unavailable'), {
- response: { statusCode: 503 },
- })
- mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)
-
- await expect(getContext('production-context')).rejects.toBe(unavailable)
- })
-})
-
describe('McpServerWatcher external egress retries', () => {
const serverObject = {
metadata: { name: 'redis-tools', namespace: 'mcp-server' },
@@ -4662,194 +460,41 @@ describe('McpServerWatcher external egress retries', () => {
}
beforeEach(() => {
- vi.clearAllMocks()
- vi.useFakeTimers()
- })
-
- afterEach(() => {
- vi.useRealTimers()
- vi.restoreAllMocks()
- })
-
- it('retries failed external egress reconciliation from an ADDED event', async () => {
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- const bindingReconciler = (watcher as any).bindingReconciler
- const serverWithBindings = {
- ...serverObject,
- metadata: {
- ...serverObject.metadata,
- annotations: {
- 'clerum.io/recipe-bindings': JSON.stringify([
- { source: 'redis-tools', target: 'downstream' },
- ]),
- },
- },
- }
- ;(watcher as any).contexts.set('context1', {
- name: 'context1',
- namespace: 'mcp-server',
- spec: { contextId: 'context1', mcpServers: ['redis-tools'] },
- })
- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
- netPol.reconcileExternalEgress.mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await watchCallback('ADDED', serverWithBindings)
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
- expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()
- expect(netPol.reconcileContext).not.toHaveBeenCalled()
-
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'redis-tools' }),
- { isCurrent: expect.any(Function) }
- )
- expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(1)
- expect(netPol.reconcileContext).toHaveBeenCalledWith(
- expect.objectContaining({
- spec: expect.objectContaining({ contextId: 'context1' }),
- }),
- expect.objectContaining({ isCurrent: expect.any(Function) })
- )
-
- watcher.stop()
- })
-
- it('retires a binding-policy lease when a same-watch MODIFIED supersedes the server', async () => {
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- const bindingReconciler = (watcher as any).bindingReconciler
- const bindingStarted = deferred()
- const releaseBinding = deferred()
- bindingReconciler.reconcileBindings.mockImplementationOnce(async () => {
- bindingStarted.resolve(undefined)
- await releaseBinding.promise
- })
- const annotations = {
- 'clerum.io/recipe-bindings': JSON.stringify([
- { from: 'redis-tools', to: 'downstream', port: 443 },
- ]),
- }
- const original = {
- ...serverObject,
- metadata: {
- ...serverObject.metadata,
- uid: 'redis-tools-uid',
- generation: 1,
- annotations,
- },
- }
- const replacement = {
- ...original,
- metadata: { ...original.metadata, generation: 2 },
- spec: { ...original.spec, image: 'redis-tools:v2' },
- }
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
-
- const originalEffect = watchCallback('ADDED', original)
- await bindingStarted.promise
- const replacementEffect = watchCallback('MODIFIED', replacement)
-
- const originalOptions = bindingReconciler.reconcileBindings.mock.calls[0][4]
- expect(originalOptions).toEqual({ isCurrent: expect.any(Function) })
- expect(originalOptions.isCurrent()).toBe(false)
-
- releaseBinding.resolve(undefined)
- await Promise.all([originalEffect, replacementEffect])
-
- expect(bindingReconciler.reconcileBindings).toHaveBeenCalledTimes(2)
- expect(bindingReconciler.reconcileBindings).toHaveBeenLastCalledWith(
- 'redis-tools',
- expect.any(Array),
- 'redis-tools',
- 'redis-tools',
- { isCurrent: expect.any(Function) }
- )
- await watcher.stop()
- })
-
- it('retires binding cleanup when a same-name McpServer is recreated during DELETE', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const bindingReconciler = (watcher as any).bindingReconciler
- const cleanupStarted = deferred()
- const releaseCleanup = deferred()
- bindingReconciler.cleanupBindings.mockImplementationOnce(async () => {
- cleanupStarted.resolve(undefined)
- await releaseCleanup.promise
- })
- const deleted = {
- ...serverObject,
- metadata: {
- ...serverObject.metadata,
- uid: 'redis-tools-old-uid',
- generation: 1,
- },
- }
- const recreated = {
- ...serverObject,
- metadata: {
- ...serverObject.metadata,
- uid: 'redis-tools-new-uid',
- generation: 1,
- },
- spec: { ...serverObject.spec, image: 'redis-tools:recreated' },
- }
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
-
- const deleteEffect = watchCallback('DELETED', deleted)
- await cleanupStarted.promise
- const recreateEffect = watchCallback('ADDED', recreated)
-
- const cleanupOptions = bindingReconciler.cleanupBindings.mock.calls[0][1]
- expect(cleanupOptions).toEqual({ deleteAllowed: expect.any(Function) })
- await expect(cleanupOptions.deleteAllowed()).resolves.toBe(false)
-
- releaseCleanup.resolve(undefined)
- await Promise.all([deleteEffect, recreateEffect])
+ vi.useFakeTimers()
+ })
- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledTimes(1)
- await watcher.stop()
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.restoreAllMocks()
})
- it('retries the complete pipeline when a no-egress runtime reconcile fails', async () => {
+ it('retries failed external egress reconciliation from an ADDED event', async () => {
const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
+ const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
- const noEgressServer = {
- ...serverObject,
- spec: {
- ...serverObject.spec,
- egressBindings: undefined,
- },
- }
- reconciler.reconcile
- .mockRejectedValueOnce(new Error('deployment temporarily unavailable'))
- .mockResolvedValue(undefined)
+ netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
+ netPol.reconcileExternalEgress.mockResolvedValue(undefined)
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(console, 'warn').mockImplementation(() => {})
- await (watcher as any).getMcpServerWatchCallback()('ADDED', noEgressServer)
- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
+
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5000)
- expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
+ expect(reconciler.reconcile).toHaveBeenCalledWith(
+ expect.objectContaining({ name: 'redis-tools' })
+ )
+
watcher.stop()
})
it('retries runtime reconciliation when external egress is ready but runtime reconcile fails', async () => {
const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockResolvedValue(undefined)
@@ -4869,8 +514,7 @@ describe('McpServerWatcher external egress retries', () => {
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenLastCalledWith(
- expect.objectContaining({ name: 'redis-tools' }),
- { isCurrent: expect.any(Function) }
+ expect.objectContaining({ name: 'redis-tools' })
)
watcher.stop()
@@ -4878,7 +522,6 @@ describe('McpServerWatcher external egress retries', () => {
it('keeps retry attempts bounded when runtime fails after an external egress retry succeeds', async () => {
const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
@@ -4904,16 +547,14 @@ describe('McpServerWatcher external egress retries', () => {
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(3)
expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenLastCalledWith(
- expect.objectContaining({ name: 'redis-tools' }),
- { isCurrent: expect.any(Function) }
+ expect.objectContaining({ name: 'redis-tools' })
)
watcher.stop()
})
- it('continues a pending runtime pipeline during periodic external egress resync', async () => {
+ it('does not let periodic external egress resync cancel a pending runtime retry', async () => {
const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockResolvedValue(undefined)
@@ -4922,7 +563,6 @@ describe('McpServerWatcher external egress retries', () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(console, 'warn').mockImplementation(() => {})
vi.spyOn(console, 'log').mockImplementation(() => {})
- vi.spyOn(Math, 'random').mockReturnValue(0)
const watchCallback = (watcher as any).getMcpServerWatchCallback()
await watchCallback('ADDED', serverObject)
@@ -4930,152 +570,18 @@ describe('McpServerWatcher external egress retries', () => {
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
- await (watcher as any).externalEgressCoordinator.runResync()
+ await (watcher as any).runExternalEgressResync()
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
- expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(5000)
+
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
expect(reconciler.reconcile).toHaveBeenCalledTimes(2)
watcher.stop()
})
- it('keeps retrying an ADDED pipeline at the capped delay until it recovers', async () => {
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- netPol.reconcileExternalEgress
- .mockRejectedValueOnce(new Error('initial outage'))
- .mockRejectedValueOnce(new Error('outage after 5s'))
- .mockRejectedValueOnce(new Error('outage after 15s'))
- .mockRejectedValueOnce(new Error('outage after 30s'))
- .mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- await (watcher as any).getMcpServerWatchCallback()('ADDED', serverObject)
- await vi.advanceTimersByTimeAsync(5000)
- await vi.advanceTimersByTimeAsync(15000)
- await vi.advanceTimersByTimeAsync(30000)
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(4)
- expect(reconciler.reconcile).not.toHaveBeenCalled()
-
- await vi.advanceTimersByTimeAsync(30000)
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(5)
- expect(reconciler.reconcile).toHaveBeenCalledTimes(1)
- watcher.stop()
- })
-
- it('keeps retrying a partial DELETE pipeline at the capped delay until it recovers', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- const bindingReconciler = (watcher as any).bindingReconciler
- netPol.cleanupExternalEgress
- .mockRejectedValueOnce(new Error('initial cleanup outage'))
- .mockRejectedValueOnce(new Error('cleanup outage after 5s'))
- .mockRejectedValueOnce(new Error('cleanup outage after 15s'))
- .mockRejectedValueOnce(new Error('cleanup outage after 30s'))
- .mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- await (watcher as any).getMcpServerWatchCallback()('DELETED', serverObject)
- await vi.advanceTimersByTimeAsync(5000)
- await vi.advanceTimersByTimeAsync(15000)
- await vi.advanceTimersByTimeAsync(30000)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(4)
- expect(reconciler.reconcileDelete).not.toHaveBeenCalled()
- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
-
- await vi.advanceTimersByTimeAsync(30000)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(5)
- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
- deleteAllowed: expect.any(Function),
- })
- watcher.stop()
- })
-
- it('bounds concurrent durable retries across McpServers', async () => {
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- const reconciler = (watcher as any).reconciler
- const release = deferred()
- let active = 0
- let maxActive = 0
- reconciler.reconcile.mockImplementation(async () => {
- active += 1
- maxActive = Math.max(maxActive, active)
- await release.promise
- active -= 1
- })
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- for (let index = 0; index < 11; index += 1) {
- const server = {
- name: `server-${index}`,
- namespace: 'mcp-server',
- spec: {
- ...serverObject.spec,
- egressBindings: undefined,
- },
- }
- ;(watcher as any).servers.set(server.name, server)
- ;(watcher as any).scheduleExternalEgressRetry('ADDED', server)
- }
-
- await vi.advanceTimersByTimeAsync(5000)
- await flushMicrotasks()
-
- expect(maxActive).toBe(10)
-
- release.resolve(undefined)
- await flushMicrotasks()
- watcher.stop()
- })
-
- it('replays a retry against the watcher generation current when the timer executes', async () => {
- const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
- const selected = {
- name: 'generation-fenced-server',
- namespace: 'mcp-server',
- spec: {
- ...serverObject.spec,
- egressBindings: undefined,
- },
- }
- ;(watcher as any).servers.set(selected.name, selected)
- ;(watcher as any).mcpWatchGeneration = 41
- const replay = vi
- .spyOn(watcher as any, 'reconcileMcpServerWatchEvent')
- .mockResolvedValue(undefined)
- vi.spyOn(console, 'warn').mockImplementation(() => {})
- ;(watcher as any).scheduleExternalEgressRetry('ADDED', selected)
- ;(watcher as any).mcpWatchGeneration = 42
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(replay).toHaveBeenCalledWith(
- 'MODIFIED',
- expect.objectContaining({ name: selected.name }),
- 42,
- expect.objectContaining({
- isCurrent: expect.any(Function),
- complete: expect.any(Function),
- })
- )
- watcher.stop()
- })
-
it('reconciles external egress before runtime reconciliation on ADDED events', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
@@ -5102,340 +608,39 @@ describe('McpServerWatcher external egress retries', () => {
const reconciler = (watcher as any).reconciler
netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('api temporarily unavailable'))
vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await watchCallback('ADDED', serverObject)
-
- expect(reconciler.reconcile).not.toHaveBeenCalled()
-
- watcher.stop()
- })
-
- it('retries failed external egress cleanup from a DELETED event', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- const bindingReconciler = (watcher as any).bindingReconciler
- netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))
- netPol.cleanupExternalEgress.mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await watchCallback('DELETED', serverObject)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
-
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
- expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith(
- 'redis-tools',
- 'mcp-server',
- undefined,
- expect.any(Function)
- )
- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
- deleteAllowed: expect.any(Function),
- })
-
- watcher.stop()
- })
-
- it('retries the full DELETE pipeline when authoritative absence is temporarily unavailable', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- const bindingReconciler = (watcher as any).bindingReconciler
- const unavailable = Object.assign(new Error('apiserver temporarily unavailable'), {
- response: { statusCode: 503 },
- })
- mocks.getNamespacedCustomObject.mockRejectedValueOnce(unavailable)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()
-
- expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
- expect(netPol.cleanupExternalEgress).not.toHaveBeenCalled()
- expect(reconciler.reconcileDelete).not.toHaveBeenCalled()
- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
-
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
- deleteAllowed: expect.any(Function),
- })
- expect(mocks.getNamespacedCustomObject).toHaveBeenCalledTimes(4)
-
- watcher.stop()
- })
-
- it('retries the full DELETE pipeline when inventory authority is lost between cleanup stages', async () => {
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- const bindingReconciler = (watcher as any).bindingReconciler
- reconciler.reconcileDelete.mockImplementationOnce(async () => {
- ;(watcher as any).mcpServerCacheSynced = false
- })
- vi.spyOn(console, 'warn').mockImplementation(() => {})
-
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
- await expect(watchCallback('DELETED', serverObject)).resolves.toBeUndefined()
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
- expect(reconciler.reconcileDelete).toHaveBeenCalledWith('redis-tools', 'mcp-server')
- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
-
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
- expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(1)
- expect(bindingReconciler.cleanupBindings).not.toHaveBeenCalled()
- ;(watcher as any).mcpServerCacheSynced = true
- await vi.advanceTimersByTimeAsync(14_999)
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
- await vi.advanceTimersByTimeAsync(1)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
- expect(reconciler.reconcileDelete).toHaveBeenCalledTimes(2)
- expect(bindingReconciler.cleanupBindings).toHaveBeenCalledWith('redis-tools', {
- deleteAllowed: expect.any(Function),
- })
-
- watcher.stop()
- })
-
- it('retires queued McpServer work immediately when its watch ends', async () => {
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- let watchCallback: ((type: string, apiObj: typeof serverObject) => Promise<void>) | undefined
- let watchDone: ((error: Error | null) => void) | undefined
- mocks.watch.mockImplementationOnce(async (_path, _options, callback, done) => {
- watchCallback = callback
- watchDone = done
- return { abort: vi.fn() }
- })
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- await (watcher as any).startMcpServerWatch('mcp-watch-rv')
- const blocker = (watcher as any).enqueueMcpServerReconciliation(
- { name: 'redis-tools', namespace: 'mcp-server' },
- async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }
- )
- await blockerStarted.promise
-
- const event = watchCallback!('ADDED', serverObject)
- watchDone!(null)
- releaseBlocker.resolve(undefined)
- await Promise.all([blocker, event])
-
- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
- await watcher.stop()
- })
-
- it('retries the recreated server when an older DELETE retry timer already owns the key', async () => {
- const deletedServer = {
- metadata: {
- name: 'redis-tools',
- namespace: 'mcp-server',
- uid: 'redis-tools-old-uid',
- generation: 1,
- },
- spec: serverObject.spec,
- }
- const recreatedServer = {
- metadata: {
- name: 'redis-tools',
- namespace: 'mcp-server',
- uid: 'redis-tools-new-uid',
- generation: 1,
- },
- spec: { ...serverObject.spec, image: 'redis-tools:recreated' },
- }
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('cleanup unavailable'))
- netPol.reconcileExternalEgress
- .mockRejectedValueOnce(new Error('recreated policy unavailable'))
- .mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
-
- await watchCallback('DELETED', deletedServer)
- await watchCallback('ADDED', recreatedServer)
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
- expect(reconciler.reconcile).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({
- uid: 'redis-tools-new-uid',
- generation: 1,
- spec: expect.objectContaining({ image: 'redis-tools:recreated' }),
- }),
- { isCurrent: expect.any(Function) }
- )
-
- watcher.stop()
- })
-
- it('retries the latest DELETE cleanup when an older ADD retry timer already owns the key', async () => {
- const server = {
- metadata: {
- name: 'redis-tools',
- namespace: 'mcp-server',
- uid: 'redis-tools-uid',
- generation: 3,
- },
- spec: serverObject.spec,
- }
- const watcher = new McpServerWatcher()
- ;(watcher as any).mcpServerCacheSynced = true
- const netPol = (watcher as any).netPolReconciler
- netPol.reconcileExternalEgress.mockRejectedValueOnce(new Error('policy unavailable'))
- netPol.cleanupExternalEgress
- .mockRejectedValueOnce(new Error('cleanup unavailable'))
- .mockResolvedValue(undefined)
- vi.spyOn(console, 'error').mockImplementation(() => {})
- vi.spyOn(console, 'warn').mockImplementation(() => {})
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
-
- await watchCallback('ADDED', server)
- await watchCallback('DELETED', server)
- await vi.advanceTimersByTimeAsync(5000)
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
- expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
-
- watcher.stop()
- })
-
- it('does not continue to runtime or bindings after stop releases a live DNS hold', async () => {
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const server = {
- metadata: {
- name: 'redis-tools',
- namespace: 'mcp-server',
- uid: 'redis-tools-uid',
- generation: 1,
- annotations: {
- 'clerum.io/recipe-bindings': JSON.stringify([
- { source: 'redis-tools', target: 'downstream' },
- ]),
- },
- },
- spec: serverObject.spec,
- }
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- const bindingReconciler = (watcher as any).bindingReconciler
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
-
- const reconcile = watchCallback('ADDED', server)
- await egressStarted.promise
- await watcher.stop()
- releaseEgress.resolve(undefined)
- await reconcile
-
- expect(reconciler.reconcile).not.toHaveBeenCalled()
- expect(bindingReconciler.reconcileBindings).not.toHaveBeenCalled()
- })
-
- it('does not run an older runtime revision after newer desired state arrives during egress', async () => {
- const egressStarted = deferred()
- const releaseEgress = deferred()
- const oldServer = {
- metadata: {
- name: 'redis-tools',
- namespace: 'mcp-server',
- uid: 'redis-tools-uid',
- generation: 1,
- },
- spec: { ...serverObject.spec, image: 'redis-tools:old' },
- }
- const newServer = {
- metadata: {
- name: 'redis-tools',
- namespace: 'mcp-server',
- uid: 'redis-tools-uid',
- generation: 2,
- },
- spec: { ...serverObject.spec, image: 'redis-tools:new' },
- }
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- const reconciler = (watcher as any).reconciler
- netPol.reconcileExternalEgress
- .mockImplementationOnce(async () => {
- egressStarted.resolve(undefined)
- await releaseEgress.promise
- })
- .mockResolvedValue(undefined)
- const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
- const oldReconcile = watchCallback('ADDED', oldServer)
- await egressStarted.promise
- const newReconcile = watchCallback('MODIFIED', newServer)
- releaseEgress.resolve(undefined)
- await Promise.all([oldReconcile, newReconcile])
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('ADDED', serverObject)
- expect(reconciler.reconcile).toHaveBeenCalledOnce()
- expect(reconciler.reconcile).toHaveBeenCalledWith(
- expect.objectContaining({
- generation: 2,
- spec: expect.objectContaining({ image: 'redis-tools:new' }),
- }),
- { isCurrent: expect.any(Function) }
- )
+ expect(reconciler.reconcile).not.toHaveBeenCalled()
watcher.stop()
})
- it('does not resume periodic external egress work from jitter after stop', async () => {
+ it('retries failed external egress cleanup from a DELETED event', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
- ;(watcher as any).servers.set('redis-tools', {
- name: 'redis-tools',
- namespace: 'mcp-server',
- spec: serverObject.spec,
- })
- vi.spyOn(Math, 'random').mockReturnValue(0.5)
+ netPol.cleanupExternalEgress.mockRejectedValueOnce(new Error('delete temporarily unavailable'))
+ netPol.cleanupExternalEgress.mockResolvedValue(undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'warn').mockImplementation(() => {})
- const resync = (watcher as any).externalEgressCoordinator.runResync()
- await flushMicrotasks()
- await watcher.stop()
- await vi.advanceTimersByTimeAsync(2500)
- await resync
+ const watchCallback = (watcher as any).getMcpServerWatchCallback()
+ await watchCallback('DELETED', serverObject)
- expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(1)
+
+ await vi.advanceTimersByTimeAsync(5000)
+
+ expect(netPol.cleanupExternalEgress).toHaveBeenCalledTimes(2)
+ expect(netPol.cleanupExternalEgress).toHaveBeenLastCalledWith('redis-tools', 'mcp-server')
+
+ watcher.stop()
})
it('periodic resync reuses external egress reconciliation for cached servers with bindings', async () => {
const watcher = new McpServerWatcher()
- markMcpServerInventoryAuthoritative(watcher)
const netPol = (watcher as any).netPolReconciler
netPol.reconcileExternalEgress.mockResolvedValue(undefined)
vi.spyOn(Math, 'random').mockReturnValue(0)
@@ -5445,7 +650,7 @@ describe('McpServerWatcher external egress retries', () => {
spec: serverObject.spec,
})
- await (watcher as any).externalEgressCoordinator.runResync()
+ await (watcher as any).runExternalEgressResync()
expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
@@ -5455,92 +660,55 @@ describe('McpServerWatcher external egress retries', () => {
it('does not run parallel external egress reconciles for the same McpServer', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
- const mutationStarted = deferred()
- const releaseMutation = deferred()
const server = {
name: 'redis-tools',
namespace: 'mcp-server',
spec: serverObject.spec,
}
;(watcher as any).servers.set('redis-tools', server)
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- mutationStarted.resolve(undefined)
- await releaseMutation.promise
- })
+ ;(watcher as any).externalEgressInFlight.set(
+ 'mcp-server/redis-tools',
+ new Promise(() => undefined)
+ )
vi.spyOn(Math, 'random').mockReturnValue(0)
- const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
- await mutationStarted.promise
- await (watcher as any).externalEgressCoordinator.runResync()
+ await (watcher as any).runExternalEgressResync()
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
- releaseMutation.resolve(undefined)
- await active
watcher.stop()
})
it('waits for already in-flight external egress before reconciling the current server', async () => {
const watcher = new McpServerWatcher()
const netPol = (watcher as any).netPolReconciler
- const mutationStarted = deferred()
- const releaseMutation = deferred()
const server = {
name: 'redis-tools',
namespace: 'mcp-server',
spec: serverObject.spec,
}
- ;(watcher as any).servers.set(server.name, server)
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- mutationStarted.resolve(undefined)
- await releaseMutation.promise
+ let releaseExisting!: () => void
+ const existing = new Promise<void>(resolve => {
+ releaseExisting = resolve
})
+ ;(watcher as any).externalEgressInFlight.set('mcp-server/redis-tools', existing)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
- const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
- await mutationStarted.promise
- const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)
- await flushMicrotasks()
+ const run = (watcher as any).runExternalEgressOnce('MODIFIED', server)
+ await Promise.resolve()
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
+ expect(netPol.reconcileExternalEgress).not.toHaveBeenCalled()
expect(warn).toHaveBeenCalledWith(
'[K8s] Waiting for external egress reconcile for mcp-server/redis-tools; already in flight'
)
- releaseMutation.resolve(undefined)
- await Promise.all([active, waiting])
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(2)
+ releaseExisting()
+ await expect(run).resolves.toBeUndefined()
+ expect(netPol.reconcileExternalEgress).toHaveBeenCalledTimes(1)
warn.mockRestore()
watcher.stop()
})
-
- it('does not start another external egress mutation after stop releases an in-flight wait', async () => {
- const watcher = new McpServerWatcher()
- const netPol = (watcher as any).netPolReconciler
- const mutationStarted = deferred()
- const releaseMutation = deferred()
- const server = {
- name: 'redis-tools',
- namespace: 'mcp-server',
- spec: serverObject.spec,
- }
- ;(watcher as any).servers.set(server.name, server)
- netPol.reconcileExternalEgress.mockImplementationOnce(async () => {
- mutationStarted.resolve(undefined)
- await releaseMutation.promise
- })
-
- const active = (watcher as any).runExternalEgressOnce('MODIFIED', server)
- await mutationStarted.promise
- const waiting = (watcher as any).runExternalEgressOnce('MODIFIED', server)
- await flushMicrotasks()
- await watcher.stop()
- releaseMutation.resolve(undefined)
- await Promise.all([active, waiting])
-
- expect(netPol.reconcileExternalEgress).toHaveBeenCalledOnce()
- })
})
describe('McpServerWatcher.countCommunicationChannels', () => {
@@ -5611,139 +779,9 @@ describe('McpServerWatcher wires CC counter into HostReconciler', () => {
})
})
-describe('McpServerWatcher Host mutation dependency wiring', () => {
- it('H2: wires a uid-guarded cache reflector into HostReconciler', () => {
- const watcher = new McpServerWatcher()
- const reflect = (
- watcher.getHostReconciler() as unknown as {
- _reflectHostOutcomeFn:
- | ((name: string, uid: string | undefined, apply: (target: HostCRD) => void) => void)
- | null
- }
- )._reflectHostOutcomeFn
- if (!reflect) {
- throw new Error('setReflectHostOutcome was never wired by the McpServerWatcher constructor')
- }
- const entry = {
- name: 'h2-host',
- namespace: 'mcp-host',
- uid: 'uid-1',
- spec: { host: 'h2-host', contextRef: 'c', secretRef: 's' },
- } as unknown as HostCRD
- ;(watcher as unknown as { hosts: Map<string, HostCRD> }).hosts.set('h2-host', entry)
- const apply = vi.fn()
- reflect('h2-host', 'uid-1', apply) // positive: current entry, matching uid
- expect(apply).toHaveBeenCalledTimes(1)
- expect(apply).toHaveBeenCalledWith(entry)
- apply.mockClear()
- reflect('h2-host', 'uid-2', apply) // recreation: same name, different uid → skip
- expect(apply).not.toHaveBeenCalled()
- reflect('h2-host', undefined, apply) // fail-closed on missing uid
- expect(apply).not.toHaveBeenCalled()
- reflect('absent-host', 'uid-1', apply) // absent entry → skip
- expect(apply).not.toHaveBeenCalled()
- })
-
- it('tracks only the selected Context, referenced SFS state, and Host-scoped channels', () => {
- const watcher = new McpServerWatcher()
- const host = {
- name: 'alpha-host',
- namespace: 'mcp-host',
- spec: { contextRef: 'alpha-context' },
- } as HostCRD
- const context = {
- name: 'alpha-context',
- namespace: 'mcp-server',
- spec: {
- contextId: 'alpha-context',
- sharedFileSystems: [{ name: 'documents', mountPath: '/contexts/documents' }],
- },
- }
- const documentsV1 = {
- name: 'documents',
- namespace: 'mcp-host',
- spec: { storageClassName: 'fast-v1' },
- }
- const alphaChannelV1 = {
- name: 'telegram-alpha',
- namespace: 'channels',
- spec: { hostRef: 'alpha-host' },
- }
- ;(watcher as any).contexts.set(context.name, context)
- ;(watcher as any).sharedFileSystems.set(documentsV1.name, documentsV1)
- ;(watcher as any).communicationChannels.set(alphaChannelV1.name, alphaChannelV1)
- ;(watcher as any).ccCacheSynced = true
- ;(watcher as any).ccWatchGeneration = 7
- const sfsReconciler = (watcher as any).sharedFileSystemReconciler
- sfsReconciler.isMountable.mockReturnValue(true)
- const hostReconciler = watcher.getHostReconciler() as any
- const resolveDependencies = hostReconciler._hostMutationDependenciesFn
-
- const original = resolveDependencies(host)
- expect(original).toEqual([
- { name: context.name, namespace: context.namespace, spec: context.spec },
- true,
- 7,
- { name: documentsV1.name, namespace: documentsV1.namespace, spec: documentsV1.spec },
- true,
- {
- name: alphaChannelV1.name,
- namespace: alphaChannelV1.namespace,
- spec: alphaChannelV1.spec,
- },
- ])
- ;(watcher as any).sharedFileSystems.set('unrelated', {
- name: 'unrelated',
- namespace: 'mcp-host',
- spec: {},
- })
- ;(watcher as any).communicationChannels.set('telegram-beta', {
- name: 'telegram-beta',
- namespace: 'channels',
- spec: { hostRef: 'beta-host' },
- })
- const afterUnrelatedChanges = resolveDependencies(host)
- expect(afterUnrelatedChanges).toEqual(original)
-
- const documentsV2 = {
- ...documentsV1,
- spec: { storageClassName: 'fast-v2' },
- }
- ;(watcher as any).sharedFileSystems.set(documentsV2.name, documentsV2)
- const afterSfsRevision = resolveDependencies(host)
- expect(afterSfsRevision[3]).toEqual({
- name: documentsV2.name,
- namespace: documentsV2.namespace,
- spec: documentsV2.spec,
- })
- expect(afterSfsRevision[3]).not.toEqual(original[3])
-
- sfsReconciler.isMountable.mockReturnValue(false)
- const afterMountabilityChange = resolveDependencies(host)
- expect(afterMountabilityChange[4]).toBe(false)
-
- const alphaChannelV2 = {
- ...alphaChannelV1,
- spec: {
- ...alphaChannelV1.spec,
- credentialsSecretRef: { name: 'telegram-alpha-v2' },
- },
- }
- ;(watcher as any).communicationChannels.set(alphaChannelV2.name, alphaChannelV2)
- const afterChannelRevision = resolveDependencies(host)
- expect(afterChannelRevision.at(-1)).toEqual({
- name: alphaChannelV2.name,
- namespace: alphaChannelV2.namespace,
- spec: alphaChannelV2.spec,
- })
- expect(afterChannelRevision.at(-1)).not.toEqual(original.at(-1))
- })
-})
-
describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('re-reconciles the Host whose name matches the CC hostRef', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>
hosts.set('marketing', {
name: 'marketing',
@@ -5762,7 +800,6 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('is a no-op when the hostRef does not match any cached Host', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const reconcileSpy = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -5774,7 +811,6 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('reports reconcile errors without throwing so the watch can schedule recovery', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, { name: string; spec: any }>
hosts.set('marketing', {
name: 'marketing',
@@ -5792,7 +828,6 @@ describe('McpServerWatcher.reconcileHostsReferencingCC', () => {
it('schedules lifecycle convergence when a targeted CC event reconcile fails', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
;(watcher as any).hosts.set('marketing', {
name: 'marketing',
namespace: 'mcp-host',
@@ -5861,8 +896,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {
})
const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
const ccWatchSpy = vi.spyOn(watcher as any, 'startCommunicationChannelWatch')
@@ -5901,8 +936,8 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {
const watcher = new McpServerWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
const ccWatchSpy = vi
@@ -5970,7 +1005,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('partial-channel', {
name: 'partial-channel',
@@ -6010,7 +1045,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
if (plural === 'hosts') return { items: [host] }
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('watch-observed-channel', {
name: 'watch-observed-channel',
@@ -6051,7 +1086,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostReconcileHosts
.mockImplementationOnce(() => firstFleetPass.promise)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const failClosedPass = (watcher as any).requestHostFleetReconcile(
@@ -6127,7 +1162,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
})
const fleetFailure = new Error('fleet did not converge')
mocks.hostReconcileHosts.mockRejectedValueOnce(fleetFailure).mockResolvedValueOnce(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -6169,7 +1204,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
if (hostListAttempts === 1) throw inventoryFailure
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -6179,12 +1214,14 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
lifecycleGeneration
)
expect((watcher as any).ccAppliedLifecycleGeneration).not.toBe(lifecycleGeneration)
+ expect((watcher as any).ccFleetRetryTimer).not.toBeNull()
expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(5000)
await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
expect(hostListAttempts).toBe(2)
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
+ expect((watcher as any).ccFleetRetryTimer).toBeNull()
} finally {
errorSpy.mockRestore()
watcher.stop()
@@ -6200,7 +1237,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
mocks.hostFullReconcile.mockRejectedValueOnce(new HostFleetReconcileError([], [cleanupFailure]))
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -6212,6 +1249,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
)
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
+ expect((watcher as any).ccFleetRetryTimer).toBeNull()
await vi.advanceTimersByTimeAsync(300000)
expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
} finally {
@@ -6232,7 +1270,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostFullReconcile
.mockRejectedValueOnce(new HostFleetReconcileError([hostFailure], [cleanupFailure]))
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
;(watcher as any).ccCacheSynced = true
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -6249,6 +1287,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
await vi.advanceTimersByTimeAsync(5000)
await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
expect((watcher as any).ccAppliedLifecycleGeneration).toBe(lifecycleGeneration)
+ expect((watcher as any).ccFleetRetryTimer).toBeNull()
await (watcher as any).performHostResync()
expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2)
@@ -6273,7 +1312,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostReconcileHosts
.mockRejectedValueOnce(new Error('lifecycle pass failed'))
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
;(watcher as any).ccCacheSynced = true
@@ -6318,7 +1357,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
mocks.hostReconcileHosts.mockRejectedValue(new Error('persistent fleet failure'))
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const lifecycleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
@@ -6364,13 +1403,15 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
mocks.hostReconcileHosts
.mockRejectedValueOnce(new Error('old generation failed'))
.mockResolvedValueOnce(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
await (watcher as any).requestHostFleetReconcile('fail-closed', staleGeneration)
+ expect((watcher as any).ccFleetRetryTimer).not.toBeNull()
const currentGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
+ expect((watcher as any).ccFleetRetryTimer).toBeNull()
await (watcher as any).requestHostFleetReconcile('recovered', currentGeneration)
expect(mocks.hostReconcileHosts).toHaveBeenCalledTimes(2)
@@ -6398,7 +1439,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
}
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('previous-channel', {
name: 'previous-channel',
@@ -6439,7 +1480,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const scheduleSpy = vi
.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery')
.mockImplementation(() => {})
@@ -6497,7 +1538,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
.mockImplementationOnce(() => firstHostConvergence.promise)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const firstRecovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>
try {
@@ -6555,7 +1596,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
}
)
.mockResolvedValue({ abort: vi.fn() })
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
try {
await expect((watcher as any).recoverCommunicationChannelCache()).resolves.toBe(false)
@@ -6588,7 +1629,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { items: [] }
})
mocks.watch.mockImplementationOnce(() => pendingWatch.promise)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const recovery = (watcher as any).recoverCommunicationChannelCache() as Promise<boolean>
try {
@@ -6633,7 +1674,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const scheduleSpy = vi
.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery')
.mockImplementation(() => {})
@@ -6695,9 +1736,9 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
maxActivePasses = Math.max(maxActivePasses, activePasses)
activePasses -= 1
})
- const watcher = newContextAuthoritativeWatcher()
- stubAuthoritativeInventoryWatch(watcher, 'McpServer')
- stubAuthoritativeInventoryWatch(watcher, 'Context')
+ const watcher = new McpServerWatcher()
+ vi.spyOn(watcher as any, 'startMcpServerWatch').mockResolvedValue(undefined)
+ vi.spyOn(watcher as any, 'startContextWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startSharedFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'startGlobalFileSystemWatch').mockResolvedValue(undefined)
vi.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery').mockImplementation(
@@ -6717,11 +1758,8 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
expect(mocks.hostReconcileHosts).not.toHaveBeenCalled()
expect(maxActivePasses).toBe(1)
- // Startup now resolves after safe bootstrap; the serialized fleet tail
- // keeps running independently and must still drain the queued lifecycle pass.
- await start
initialPass.resolve(undefined)
- await vi.waitFor(() => expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce())
+ await start
expect(mocks.hostFullReconcile).toHaveBeenCalledOnce()
expect(mocks.hostReconcileHosts).toHaveBeenCalledOnce()
@@ -6748,7 +1786,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return { abort }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
;(watcher as any).ccCacheSynced = true
await (watcher as any).startCommunicationChannelWatch('504')
@@ -6782,7 +1820,7 @@ describe('McpServerWatcher CommunicationChannel cache recovery', () => {
return requests[callbacks.length - 1]
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const scheduleSpy = vi
.spyOn(watcher as any, 'scheduleCommunicationChannelCacheRecovery')
.mockImplementation(() => {})
@@ -6833,7 +1871,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
const abort = vi.fn()
const watchRequest = { abort }
mocks.watch.mockReset().mockResolvedValue(watchRequest)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
await (watcher as any).startHostWatch('stream-rv')
await vi.advanceTimersByTimeAsync(30_001)
@@ -6868,11 +1906,9 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => activePass)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
;(watcher as any).ccCacheSynced = true
- ;(watcher as any).hostFleetScheduler.markLifecycleApplied(
- (watcher as any).ccLifecycleGeneration
- )
+ ;(watcher as any).ccAppliedLifecycleGeneration = (watcher as any).ccLifecycleGeneration
const first = (watcher as any).runHostResync() as Promise<void>
await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))
@@ -6891,53 +1927,9 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)
})
- it('queues a trailing full Host pass when Context recovery lands during an active pass', async () => {
- vi.clearAllMocks()
- const activePass = deferred()
- mocks.hostFullReconcile
- .mockReset()
- .mockImplementationOnce(() => activePass.promise)
- .mockResolvedValue(undefined)
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'contexts') {
- return {
- metadata: { resourceVersion: 'recovered-context-rv' },
- items: [
- {
- metadata: { name: 'recovered-context', namespace: 'mcp-server' },
- spec: {
- contextId: 'recovered-context',
- mcpServers: [],
- sharedFileSystems: [{ name: 'recovered-sfs' }],
- },
- },
- ],
- }
- }
- return { items: [] }
- })
- mocks.watch.mockReset().mockResolvedValue({ abort: vi.fn() })
- const watcher = newContextAuthoritativeWatcher()
- ;(watcher as any).hostCacheSynced = true
-
- const first = (watcher as any).requestHostFleetReconcile(
- 'before Context recovery'
- ) as Promise<void>
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1))
-
- await expect((watcher as any).recoverContextInventoryAndWatch()).resolves.toBe(true)
- expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(1)
-
- activePass.resolve(undefined)
- await first
- await vi.waitFor(() => expect(mocks.hostFullReconcile).toHaveBeenCalledTimes(2))
-
- await watcher.stop()
- })
-
it('does not invent another fleet transition during a failed periodic cache recovery', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
;(watcher as any).ccCacheSynced = false
const recover = vi
.spyOn(watcher as any, 'recoverCommunicationChannelCache')
@@ -6953,33 +1945,6 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
watcher.stop()
})
- it('coalesces the newest Host input revision into the serialized trailing pass', async () => {
- vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
- const activePass = deferred()
- const requests: Array<{ inputRevision: number }> = []
- vi.spyOn(watcher as any, 'performHostFleetReconcile').mockImplementation(
- async (...args: unknown[]) => {
- const request = args[0] as { inputRevision: number }
- requests.push({ ...request })
- if (requests.length === 1) await activePass.promise
- }
- )
- ;(watcher as any).hostFleetScheduler.bumpInputRevision()
- const first = (watcher as any).requestHostFleetReconcile('active') as Promise<void>
- ;(watcher as any).hostFleetScheduler.bumpInputRevision()
- const second = (watcher as any).requestHostFleetReconcile('pending-v2') as Promise<void>
- ;(watcher as any).hostFleetScheduler.bumpInputRevision()
- const third = (watcher as any).requestHostFleetReconcile('pending-v3') as Promise<void>
-
- activePass.resolve(undefined)
- await Promise.all([first, second, third])
-
- expect(requests).toHaveLength(2)
- expect(requests[1].inputRevision).toBe(3)
- watcher.stop()
- })
-
it('collapses overlapping fleet requests into one serialized trailing pass', async () => {
vi.clearAllMocks()
const fleetV1 = [
@@ -7023,7 +1988,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
maxActivePasses = Math.max(maxActivePasses, activePasses)
activePasses -= 1
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const first = (watcher as any).requestHostFleetReconcile(
'first',
@@ -7076,7 +2041,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
it('uses fresh lifecycle inventory without replacing newer Host watch-cache entries', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const initialHost = {
name: 'stateless-primary',
namespace: 'mcp-host',
@@ -7141,7 +2106,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => lifecyclePass)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()
const lifecycle = (watcher as any).requestHostFleetReconcile(
'CommunicationChannel interruption',
@@ -7186,7 +2151,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => lifecyclePass)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const generation = (watcher as any).beginCommunicationChannelLifecycleTransition()
const lifecycle = (watcher as any).requestHostFleetReconcile(
'CommunicationChannel recovery',
@@ -7231,7 +2196,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
.mockReset()
.mockImplementationOnce(() => lifecyclePass)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const active = (watcher as any).requestHostFleetReconcile(
'CommunicationChannel interruption',
@@ -7278,7 +2243,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
})
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const firstGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const full = (watcher as any).requestHostFleetReconcile(
'initial Host reconciliation',
@@ -7330,7 +2295,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return { items: [] }
})
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
@@ -7415,14 +2380,13 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return { items: [] }
})
mocks.hostFullReconcile.mockReset().mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('transient event failure'))
.mockResolvedValue(undefined)
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- markHostInventoryAuthoritative(watcher)
await hostWatchCallback('MODIFIED', {
metadata: { name: 'retry-before-list', namespace: 'mcp-host', generation: 1 },
@@ -7431,9 +2395,6 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
expect(reconcile).toHaveBeenCalledTimes(1)
expect((watcher as any).hostWatchRetryTimers.size).toBe(1)
- // The active Host watch is then lost. A full request must recover a fresh
- // snapshot and retire the failed event's pending retry.
- ;(watcher as any).hostCacheSynced = false
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
expect((watcher as any).hostWatchRetryTimers.size).toBe(0)
@@ -7486,7 +2447,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
await fullReconcile.promise
order.push('full:end')
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const hostReconciler = watcher.getHostReconciler()
const reconcile = vi.spyOn(hostReconciler, 'reconcile').mockImplementation(async host => {
order.push(`watch:${host.name}:${host.generation}`)
@@ -7539,7 +2500,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
}
)
mocks.listNamespacedCustomObject.mockImplementation(async () => ({ items: [] }))
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const hostReconciler = watcher.getHostReconciler()
// The reconcile()/reconcileDelete() spies stand in for the per-Host chain;
// the actual serializeByHost non-overlap guarantee is proven in
@@ -7590,7 +2551,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const order: string[] = []
const timeout = new HostK8sRequestTimeoutError('AppsV1Api.readNamespacedDeployment', 30_000)
const reconcile = vi
@@ -7601,7 +2562,6 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
})
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- markHostInventoryAuthoritative(watcher)
const first = hostWatchCallback('MODIFIED', {
metadata: { name: 'host-a', namespace: 'mcp-host', generation: 1 },
@@ -7637,7 +2597,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
if (plural === 'hosts') return { items: [] }
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -7686,7 +2646,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
// wins). Block that full pass so it stays running until shutdown unblocks
// the promoted trailing waiter — the semantic this test protects.
mocks.hostFullReconcile.mockReset().mockImplementation(() => trailingPass)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const activeRequest = (watcher as any).requestHostFleetReconcile(
'startup',
@@ -7732,7 +2692,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
resolveActivePass = resolve
})
mocks.hostReconcileHosts.mockReset().mockImplementationOnce(() => activePass)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const activeGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const activeRequest = (watcher as any).requestHostFleetReconcile(
'active',
@@ -7771,7 +2731,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
return inventory
})
mocks.hostReconcileHosts.mockReset().mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const staleGeneration = (watcher as any).beginCommunicationChannelLifecycleTransition()
const staleRequest = (watcher as any).requestHostFleetReconcile(
'stale transition',
@@ -7825,7 +2785,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
if (plural === 'hosts') return hostInventory
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const run = (watcher as any).requestHostFleetReconcile('shutdown race') as Promise<void>
await vi.waitFor(() => {
@@ -7850,7 +2810,7 @@ describe('McpServerWatcher Host periodic resync serialization', () => {
it('releases the periodic single-flight guard after a rejected pass', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const failure = new Error('transient resync failure')
const performSpy = vi
.spyOn(watcher as any, 'performHostResync')
@@ -7883,189 +2843,12 @@ describe('McpServerWatcher Host watch generation', () => {
vi.useRealTimers()
})
- it('folds a newer lifecycle generation into an in-flight Host inventory recovery', async () => {
- const hostListStarted = deferred()
- const hostList = deferred<{
- metadata: { resourceVersion: string }
- items: unknown[]
- }>()
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- hostListStarted.resolve(undefined)
- return hostList.promise
- }
- return { items: [] }
- })
- const watcher = newContextAuthoritativeWatcher()
- const fleetRequest = vi
- .spyOn(watcher as any, 'requestHostFleetReconcile')
- .mockResolvedValue(undefined)
-
- const recovery = (watcher as any).recoverHostInventoryAndWatch(
- 'Host watch recovery convergence',
- 1,
- 'watch-recovery'
- ) as Promise<unknown>
- await hostListStarted.promise
- const joined = (watcher as any).recoverHostInventoryAndWatch(
- 'initial Host reconciliation',
- 2,
- 'cold-start'
- ) as Promise<unknown>
-
- expect(joined).toBe(recovery)
- hostList.resolve({ metadata: { resourceVersion: 'joined-host-rv' }, items: [] })
- await Promise.all([recovery, joined])
-
- expect(fleetRequest).toHaveBeenCalledOnce()
- expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')
- await watcher.stop()
- })
-
- it('adopts a pending cold-start intent when a direct Host recovery starts first', async () => {
- vi.useFakeTimers()
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- return { metadata: { resourceVersion: 'adopted-host-rv' }, items: [] }
- }
- return { items: [] }
- })
- const watcher = newContextAuthoritativeWatcher()
- const fleetRequest = vi
- .spyOn(watcher as any, 'requestHostFleetReconcile')
- .mockResolvedValue(undefined)
-
- ;(watcher as any).scheduleHostCacheRecovery({
- convergenceReason: 'initial Host reconciliation',
- ccLifecycleGeneration: 2,
- cause: 'cold-start',
- })
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
-
- await (watcher as any).recoverHostInventoryAndWatch(
- 'Host watch recovery convergence',
- 1,
- 'watch-recovery'
- )
-
- expect(mocks.listNamespacedCustomObject).toHaveBeenCalledTimes(1)
- expect(fleetRequest).toHaveBeenCalledOnce()
- expect(fleetRequest).toHaveBeenCalledWith('initial Host reconciliation', 2, 'full')
- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
- await watcher.stop()
- })
-
- it('folds a recovery intent scheduled while Host LIST is in flight into the successful pass', async () => {
- vi.useFakeTimers()
- const hostListStarted = deferred()
- const hostList = deferred<{
- metadata: { resourceVersion: string }
- items: unknown[]
- }>()
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural === 'hosts') {
- hostListStarted.resolve(undefined)
- return hostList.promise
- }
- return { items: [] }
- })
- const watcher = newContextAuthoritativeWatcher()
- const fleetRequest = vi
- .spyOn(watcher as any, 'requestHostFleetReconcile')
- .mockResolvedValue(undefined)
-
- const recovery = (watcher as any).recoverHostInventoryAndWatch(
- 'Host watch recovery convergence',
- 1,
- 'watch-recovery'
- ) as Promise<unknown>
- await hostListStarted.promise
- ;(watcher as any).scheduleHostCacheRecovery({
- convergenceReason: 'CommunicationChannel event Host convergence',
- ccLifecycleGeneration: 3,
- cause: 'cold-start',
- })
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
- hostList.resolve({ metadata: { resourceVersion: 'in-flight-intent-rv' }, items: [] })
- await recovery
-
- expect(fleetRequest).toHaveBeenCalledOnce()
- expect(fleetRequest).toHaveBeenCalledWith(
- 'CommunicationChannel event Host convergence',
- 3,
- 'full'
- )
- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
- await watcher.stop()
- })
-
- it('preserves an in-flight recovery intent across failure and the scheduled retry', async () => {
- vi.useFakeTimers()
- const firstHostListStarted = deferred()
- const firstHostList = deferred<{
- metadata: { resourceVersion: string }
- items: unknown[]
- }>()
- let hostListCalls = 0
- mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
- if (plural !== 'hosts') return { items: [] }
- hostListCalls += 1
- if (hostListCalls === 1) {
- firstHostListStarted.resolve(undefined)
- return firstHostList.promise
- }
- return { metadata: { resourceVersion: 'retry-preserved-rv' }, items: [] }
- })
- const watcher = newContextAuthoritativeWatcher()
- const fleetRequest = vi
- .spyOn(watcher as any, 'requestHostFleetReconcile')
- .mockResolvedValue(undefined)
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
-
- const recovery = (watcher as any).recoverHostInventoryAndWatch(
- 'Host watch recovery convergence',
- 1,
- 'watch-recovery'
- ) as Promise<unknown>
- await firstHostListStarted.promise
- ;(watcher as any).scheduleHostCacheRecovery({
- convergenceReason: 'CommunicationChannel event Host convergence',
- ccLifecycleGeneration: 3,
- cause: 'cold-start',
- })
- firstHostList.reject(new Error('transient Host LIST failure'))
- await expect(recovery).rejects.toThrow('transient Host LIST failure')
-
- expect((watcher as any).hostCacheRecoveryIntent).toEqual({
- convergenceReason: 'CommunicationChannel event Host convergence',
- ccLifecycleGeneration: 3,
- cause: 'cold-start',
- })
- expect((watcher as any).hostCacheRecoveryTimer).not.toBeNull()
-
- await vi.advanceTimersByTimeAsync(5_000)
- await vi.waitFor(() => expect(fleetRequest).toHaveBeenCalledOnce())
-
- expect(hostListCalls).toBe(2)
- expect(fleetRequest).toHaveBeenCalledWith(
- 'CommunicationChannel event Host convergence',
- 3,
- 'full'
- )
- expect((watcher as any).hostCacheRecoveryIntent).toBeNull()
- expect((watcher as any).hostCacheRecoveryTimer).toBeNull()
- errorSpy.mockRestore()
- await watcher.stop()
- })
-
it('rejects a Host snapshot that cannot anchor a continuing watch', async () => {
mocks.listNamespacedCustomObject.mockImplementation(async ({ plural }: { plural: string }) => {
if (plural === 'hosts') return { metadata: {}, items: [] }
return { items: [] }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
@@ -8101,7 +2884,7 @@ describe('McpServerWatcher Host watch generation', () => {
if (watchQueries.length === 1) throw new Error('watch connection failed')
return { abort: vi.fn() }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
@@ -8145,7 +2928,7 @@ describe('McpServerWatcher Host watch generation', () => {
if (path.endsWith('/hosts')) return lateRequest.promise
return { abort: vi.fn() }
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const requested = (watcher as any).requestHostFleetReconcile(
'Periodic resync',
@@ -8154,8 +2937,7 @@ describe('McpServerWatcher Host watch generation', () => {
) as Promise<void>
await vi.waitFor(() => expect(mocks.watch).toHaveBeenCalledOnce())
- const internal = (watcher as any).hostFleetScheduler.hostFleetReconcileInFlight
- .promise as Promise<void>
+ const internal = (watcher as any).hostFleetReconcileInFlight.promise as Promise<void>
expect((watcher as any).hosts.has('snapshot-host')).toBe(true)
watcher.stop()
@@ -8188,7 +2970,7 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
await (watcher as any).startHostWatch('initial-rv')
;(watcher as any).hostCacheSynced = true
@@ -8228,7 +3010,7 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('transient event failure'))
@@ -8305,7 +3087,7 @@ describe('McpServerWatcher Host watch generation', () => {
mocks.hostFullReconcile
.mockImplementationOnce(() => firstFullReconcile.promise)
.mockResolvedValue(undefined)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const full = (watcher as any).requestHostFleetReconcile(
'Periodic resync',
undefined,
@@ -8371,7 +3153,7 @@ describe('McpServerWatcher Host watch generation', () => {
],
}
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
await (watcher as any).startHostWatch('test-host-rv')
await (watcher as any).requestHostFleetReconcile('Periodic resync', undefined, 'full')
@@ -8412,7 +3194,7 @@ describe('McpServerWatcher Host watch generation', () => {
],
}
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -8466,7 +3248,7 @@ describe('McpServerWatcher Host watch generation', () => {
],
}
})
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcileDelete = vi
.spyOn(watcher.getHostReconciler(), 'reconcileDelete')
.mockResolvedValue(undefined)
@@ -8488,8 +3270,7 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery diff enqueues an immediate per-Host delete for a disappeared Host (#827 item 1)', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
- markHostInventoryAuthoritative(watcher)
+ const watcher = new McpServerWatcher()
const reconcileDelete = vi
.spyOn(watcher.getHostReconciler(), 'reconcileDelete')
.mockResolvedValue(undefined)
@@ -8520,8 +3301,7 @@ describe('McpServerWatcher Host watch generation', () => {
// short-circuits on snapshotNames.has(name) and never dispatches at all.
it('recovery delete carries a live-cache fence for a Host recreated during admission (#827 F2 parity)', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
- markHostInventoryAuthoritative(watcher)
+ const watcher = new McpServerWatcher()
const reconcileDelete = vi
.spyOn(watcher.getHostReconciler(), 'reconcileDelete')
.mockResolvedValue(undefined)
@@ -8555,8 +3335,7 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery diff reconciles (does NOT delete) a same-name Host recreated with a new uid (#827 items 1+2)', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
- markHostInventoryAuthoritative(watcher)
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -8592,7 +3371,7 @@ describe('McpServerWatcher Host watch generation', () => {
})
it('suppresses a watch event whose identity (uid) differs from the cached Host (#827 item 2)', () => {
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
;(watcher as any).hosts.set('x', {
name: 'x',
namespace: 'mcp-host',
@@ -8621,8 +3400,7 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery urgently admits a wake-pending Host that is otherwise unchanged (§10.2 step 7 wake branch)', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
- markHostInventoryAuthoritative(watcher)
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -8664,7 +3442,7 @@ describe('McpServerWatcher Host watch generation', () => {
it('recovery does NOT urgently dispatch a fully-unchanged Host with no wake annotation', async () => {
vi.clearAllMocks()
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockResolvedValue(undefined)
@@ -8712,14 +3490,13 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('transient'))
.mockResolvedValue(undefined)
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- markHostInventoryAuthoritative(watcher)
await hostWatchCallback('MODIFIED', {
metadata: { name: 'retry-lane-host', namespace: 'mcp-host', generation: 2 },
@@ -8755,7 +3532,7 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const unrelatedHost = {
name: 'unrelated-host',
namespace: 'mcp-host',
@@ -8770,7 +3547,6 @@ describe('McpServerWatcher Host watch generation', () => {
await (watcher as any).startHostWatch('test-host-rv')
const watchGeneration = (watcher as any).hostWatchGeneration
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- markHostInventoryAuthoritative(watcher)
await hostWatchCallback('ADDED', {
metadata: { name: 'retry-host', namespace: 'mcp-host', generation: 1 },
@@ -8806,13 +3582,12 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValue(new Error('persistent failure'))
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- markHostInventoryAuthoritative(watcher)
await hostWatchCallback('ADDED', {
metadata: { name: 'bounded-host', namespace: 'mcp-host', generation: 1 },
@@ -8842,14 +3617,13 @@ describe('McpServerWatcher Host watch generation', () => {
return { abort: vi.fn() }
}
)
- const watcher = newContextAuthoritativeWatcher()
+ const watcher = new McpServerWatcher()
const reconcile = vi
.spyOn(watcher.getHostReconciler(), 'reconcile')
.mockRejectedValueOnce(new Error('stale event failed'))
.mockResolvedValue(undefined)
await (watcher as any).startHostWatch('test-host-rv')
if (!hostWatchCallback) throw new Error('Host watch callback was not installed')
- markHostInventoryAuthoritative(watcher)
await hostWatchCallback('ADDED', {
metadata: { name: 'superseded-host', namespace: 'mcp-host', generation: 1 },
@@ -8871,7 +3645,6 @@ describe('McpServerWatcher Host watch generation', () => {
describe('McpServerWatcher.startCommunicationChannelWatch', () => {
it('caches ADDED events and triggers Host reconcile', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
name: 'marketing',
@@ -8895,7 +3668,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
it('removes from cache on DELETED and triggers Host reconcile', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-marketing', {
name: 'cc-marketing',
@@ -8924,7 +3696,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
it('on MODIFIED with hostRef change, reconciles BOTH old and new hosts', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-1', {
name: 'cc-1',
@@ -8989,7 +3760,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch', () => {
describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader revision rolls', () => {
it('on CC ADDED, patches the new host channel-reader revision annotation', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
name: 'marketing',
@@ -9012,7 +3782,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('preserves credentialsSecretRef from the watch event into the cache', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
@@ -9043,7 +3812,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('on CC MODIFIED with hostRef change, patches BOTH old and new hosts', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-1', {
name: 'cc-1',
@@ -9078,7 +3846,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('on CC DELETED, patches the (former) host channel-reader revision annotation', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const cache = (watcher as any).communicationChannels as Map<string, any>
cache.set('cc-marketing', {
name: 'cc-marketing',
@@ -9107,7 +3874,6 @@ describe('McpServerWatcher.startCommunicationChannelWatch — channel-reader rev
it('swallows patchChannelReaderRevisionAnnotation errors without breaking the watch loop', async () => {
const watcher = new McpServerWatcher()
- markHostInventoryAuthoritative(watcher)
const hosts = (watcher as any).hosts as Map<string, any>
hosts.set('marketing', {
name: 'marketing',
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index 1dc96adcf..7fe7ceaec 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -14,19 +14,9 @@ import {
} from './administrativeOutcomeReporter'
import { BindingDef, BindingPolicyReconciler } from './bindingPolicyReconciler'
import { config } from './config'
-import {
- ExternalEgressConvergenceCoordinator,
- type ExternalEgressRetryHandle,
- type ExternalEgressWatchEventType,
-} from './externalEgressConvergenceCoordinator'
import { gfsDefaultFactoryConfig } from './gfsConfig'
import { GfsReconciler } from './gfsReconciler'
import { ControlApiGfsSeedClient } from './gfsSeedClient'
-import {
- type HostFleetReconcileMode,
- type HostFleetReconcileRequest,
- HostFleetScheduler,
-} from './hostFleetScheduler'
import {
HostFleetReconcileError,
type HostReconcileSource,
@@ -40,19 +30,8 @@ import {
import { K8sGfsApi } from './k8s/gfsK8sApi'
import { makeHostK8sApiClient } from './k8s/hostK8sApiClient'
import { pvcName as sfsPvcName } from './k8s/sharedFileSystemFactory'
-import {
- confirmAuthoritativeMcpServerAbsence,
- isMcpServerStatusOnlyUpdate,
- sameMcpServerDesiredRevision,
-} from './mcpServerSafety'
-import {
- hostDeleteCleanupTotal,
- hostFleetRequestsTotal,
- hostWatchRecoverySeconds,
- initialConvergenceLastSuccessTimestampSeconds,
- initialConvergenceRetriesTotal,
-} from './metrics'
-import { NetworkPolicyReconciler, sameContextDesiredRevision } from './networkPolicyReconciler'
+import { hostDeleteCleanupTotal, hostFleetRequestsTotal, hostWatchRecoverySeconds } from './metrics'
+import { NetworkPolicyReconciler } from './networkPolicyReconciler'
import { McpServerReconciler } from './reconciler'
import { SharedFileSystemReconciler } from './sharedFileSystemReconciler'
import {
@@ -69,7 +48,6 @@ import {
SharedFileSystemCRD,
SharedFileSystemSpec,
} from './types'
-import { getErrorCode } from './utils'
// Only initialize K8s client if not in dev mode
let customObjectsApi: k8s.CustomObjectsApi | null = null
@@ -105,13 +83,30 @@ const PLURAL_HOSTS = 'hosts'
const PLURAL_SHAREDFILESYSTEMS = 'sharedfilesystems'
const PLURAL_GLOBALFILESYSTEMS = 'globalfilesystems'
const PLURAL_COMMUNICATIONCHANNELS = 'communicationchannels'
-// Before readiness was decoupled, a failed initial McpServer or NetworkPolicy
-// sweep made provider.start() fail and Kubernetes restarted HCC. These retries
-// retain that convergence guarantee now that the sweeps run in the background.
-const INITIAL_CONVERGENCE_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
-const INVENTORY_CACHE_RECOVERY_RETRY_MS = 5000
+const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000]
+const EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY = 10
+const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000
+/**
+ * External-egress resync delay (ms), H2 (issue #299). The configured interval is
+ * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a
+ * rotating low-TTL host is sampled every rotation. The floor is the hard lower
+ * bound to avoid a hot-loop. Pure and unit-testable.
+ */
+export function externalEgressResyncDelayMs(
+ intervalSec: number,
+ floorSec: number,
+ minObservedTtlMs = Infinity
+): number {
+ const floorMs = floorSec * 1000
+ const configuredMs = intervalSec * 1000
+ const ttlAwareMs = Number.isFinite(minObservedTtlMs)
+ ? Math.min(configuredMs, minObservedTtlMs / 2)
+ : configuredMs
+ return Math.max(ttlAwareMs, floorMs)
+}
const COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS = 5000
+const COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS = [5000, 15000, 30000, 60000, 300000]
const HOST_CACHE_RECOVERY_RETRY_MS = 5000
const HOST_WATCH_RECONCILE_RETRY_DELAYS_MS = [5000, 15000, 30000]
// Wake-pending Hosts get immediate per-Host admission after watch recovery
@@ -123,92 +118,57 @@ type CommunicationChannelSnapshot = {
resourceVersion?: string
}
-type McpServerSnapshot = {
- servers: McpServerCRD[]
- resourceVersion?: string
-}
-
-type ContextSnapshot = {
- contexts: ContextCRD[]
- resourceVersion?: string
-}
-
type HostSnapshot = {
hosts: HostCRD[]
resourceVersion?: string
}
-type HostInventoryRecoveryCause = 'cold-start' | 'watch-recovery'
+type HostFleetReconcileMode = 'full' | 'lifecycle'
type HostWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'
-type InitialConvergenceLane = 'McpServer' | 'NetworkPolicy'
-type NetworkPolicySafetyCertificate = {
- contextGeneration: number
- serverGeneration: number
- contextRevision: number
- serverRevision: number
+type HostFleetReconcileRequest = {
+ reason: string
+ mode: HostFleetReconcileMode
+ ccLifecycleGeneration?: number
}
-type ActiveInitialConvergenceRun = {
- trailingRequested: boolean
+type PendingHostFleetReconcile = HostFleetReconcileRequest & {
promise: Promise<void>
+ resolve: () => void
}
-type HostInventoryRecoveryRequest = {
- convergenceReason: string
- ccLifecycleGeneration?: number
- cause: HostInventoryRecoveryCause
-}
-
-type ActiveHostInventoryRecovery = {
- request: HostInventoryRecoveryRequest
- promise: Promise<HostCRD[]>
-}
+type ActiveHostFleetReconcile = HostFleetReconcileRequest & { promise: Promise<void> }
-function mergeHostInventoryRecoveryRequest(
- target: HostInventoryRecoveryRequest,
- requested: HostInventoryRecoveryRequest
-): void {
- // Cold start is the stronger admission mode: its snapshot is consumed only
- // by the bounded full pass, whereas watch recovery also dispatches urgent
- // per-Host work. Preserve that mode when either joined caller requires it.
- if (requested.cause === 'cold-start' || target.cause !== 'cold-start') {
- target.convergenceReason = requested.convergenceReason
- }
- if (target.cause === 'cold-start' || requested.cause === 'cold-start') {
- target.cause = 'cold-start'
- }
- if (requested.ccLifecycleGeneration !== undefined) {
- target.ccLifecycleGeneration =
- target.ccLifecycleGeneration === undefined
- ? requested.ccLifecycleGeneration
- : Math.max(target.ccLifecycleGeneration, requested.ccLifecycleGeneration)
- }
+function hostFleetRequestCovers(
+ active: HostFleetReconcileRequest,
+ requested: HostFleetReconcileRequest
+): boolean {
+ const modeCovered = active.mode === 'full' || requested.mode === 'lifecycle'
+ if (!modeCovered) return false
+ if (requested.ccLifecycleGeneration === undefined) return true
+ return active.ccLifecycleGeneration === requested.ccLifecycleGeneration
}
-/**
- * Serializes a reconciliation domain without turning a rejected operation into
- * a permanent queue failure. Kubernetes Watch invokes async callbacks without
- * awaiting the preceding callback, while a full reconciliation can delete
- * orphaned resources owned by the same domain. A single ordered effect stream
- * therefore makes a later watch event the final writer after an older sweep.
- */
-class SerializedReconciliationQueue {
- private tail: Promise<void> = Promise.resolve()
-
- enqueue(work: () => Promise<void>): Promise<void> {
- const result = this.tail.then(work)
- this.tail = result.then(
- () => undefined,
- () => undefined
- )
- return result
+function mergeHostFleetRequests(
+ pending: HostFleetReconcileRequest,
+ requested: HostFleetReconcileRequest
+): HostFleetReconcileRequest {
+ const pendingGeneration = pending.ccLifecycleGeneration
+ const requestedGeneration = requested.ccLifecycleGeneration
+ return {
+ reason: requested.reason,
+ mode: pending.mode === 'full' || requested.mode === 'full' ? 'full' : 'lifecycle',
+ ccLifecycleGeneration:
+ pendingGeneration === undefined
+ ? requestedGeneration
+ : requestedGeneration === undefined
+ ? pendingGeneration
+ : Math.max(pendingGeneration, requestedGeneration),
}
}
-type KeyedReconciliationQueue = {
- queue: SerializedReconciliationQueue
- references: number
+function delay(ms: number): Promise<void> {
+ return new Promise(resolve => setTimeout(resolve, ms))
}
type McpServerWatchObject = {
@@ -245,7 +205,7 @@ export interface McpServerProvider {
/**
* List all McpServer CRDs in the namespace.
*/
-async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {
+export async function listAllMcpServers(): Promise<McpServerCRD[]> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
@@ -261,7 +221,6 @@ async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {
})
const list = response as {
- metadata?: { resourceVersion?: string }
items: Array<{
metadata: {
name: string
@@ -288,21 +247,17 @@ async function listMcpServerSnapshot(): Promise<McpServerSnapshot> {
}))
console.log(`[K8s] Found ${servers.length} McpServer(s)`)
- return { servers, resourceVersion: list.metadata?.resourceVersion }
+ return servers
} catch (error) {
console.error('[K8s] Failed to list McpServers:', error)
throw error
}
}
-export async function listAllMcpServers(): Promise<McpServerCRD[]> {
- return (await listMcpServerSnapshot()).servers
-}
-
/**
* List all Context CRDs in the namespace.
*/
-async function listContextSnapshot(): Promise<ContextSnapshot> {
+export async function listAllContexts(): Promise<ContextCRD[]> {
if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
@@ -318,14 +273,8 @@ async function listContextSnapshot(): Promise<ContextSnapshot> {
})
const list = response as {
- metadata?: { resourceVersion?: string }
items: Array<{
- metadata: {
- name: string
- namespace?: string
- uid?: string
- generation?: number
- }
+ metadata: { name: string; namespace?: string }
spec: ContextSpec
}>
}
@@ -333,23 +282,17 @@ async function listContextSnapshot(): Promise<ContextSnapshot> {
const contexts = list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || config.namespace,
- uid: item.metadata.uid,
- generation: item.metadata.generation,
spec: item.spec,
}))
console.log(`[K8s] Found ${contexts.length} Context(s)`)
- return { contexts, resourceVersion: list.metadata?.resourceVersion }
+ return contexts
} catch (error) {
console.error('[K8s] Failed to list Contexts:', error)
throw error
}
}
-export async function listAllContexts(): Promise<ContextCRD[]> {
- return (await listContextSnapshot()).contexts
-}
-
/**
* List all Host CRDs in host namespace.
*/
@@ -456,16 +399,12 @@ export async function getAuthToken(
* SharedFileSystems are allowed to live in, per CRD validation).
*/
export async function listAllSharedFileSystems(): Promise<SharedFileSystemCRD[]> {
- if (!hostCustomObjectsApi) {
+ if (!customObjectsApi) {
throw new Error('K8s client not initialized - are you in dev mode?')
}
try {
console.log(`[K8s] Listing all SharedFileSystems in namespace ${config.hostNamespace}`)
- // Deadline-bearing client, like listHostSnapshot: the cold-start Host fleet
- // pass waits on this inventory, so an apiserver that never answers would
- // otherwise strand the fleet behind an already-certified readiness — and
- // leave the SharedFileSystem watch, which starts after this await, dead.
- const response = await hostCustomObjectsApi.listNamespacedCustomObject({
+ const response = await customObjectsApi.listNamespacedCustomObject({
group: GROUP,
version: VERSION,
namespace: config.hostNamespace,
@@ -507,17 +446,12 @@ export async function listAllGlobalFileSystems(): Promise<GlobalFileSystemCRD[]>
plural: PLURAL_GLOBALFILESYSTEMS,
})
const list = response as {
- items: Array<{
- metadata: { name: string; namespace?: string }
- spec: GlobalFileSystemSpec
- status?: GlobalFileSystemCRD['status']
- }>
+ items: Array<{ metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }>
}
return list.items.map(item => ({
name: item.metadata.name,
namespace: item.metadata.namespace || namespace,
spec: item.spec,
- status: item.status,
}))
} catch (error) {
console.error('[K8s] Failed to list GlobalFileSystems:', error)
@@ -618,7 +552,7 @@ export async function getContext(contextId: string): Promise<ContextCRD | null>
return null
}
console.error(`[K8s] Failed to read Context CRD:`, error)
- throw error
+ return null
}
}
@@ -638,26 +572,7 @@ export class McpServerWatcher implements McpServerProvider {
private hosts: Map<string, HostCRD> = new Map()
private contexts: Map<string, ContextCRD> = new Map()
private sharedFileSystems: Map<string, SharedFileSystemCRD> = new Map()
- private globalFileSystems: Map<string, GlobalFileSystemCRD> = new Map()
private communicationChannels: Map<string, CommunicationChannelCRD> = new Map()
- private mcpWatchGeneration = 0
- private contextWatchGeneration = 0
- private mcpServerDesiredRevision = 0
- private contextDesiredRevision = 0
- // Readiness covers authoritative revocation, not additive fleet completion.
- // Generations bind the completed safety sweep to the exact LIST -> WATCH
- // pair that supplied its absence decisions. A recovered watch invalidates
- // this marker until the new inventory's orphan-allow sweep completes.
- private networkPolicyRevocationContextGeneration = 0
- private networkPolicyRevocationServerGeneration = 0
- private networkPolicyRevocationContextRevision = 0
- private networkPolicyRevocationServerRevision = 0
- private mcpServerCacheSynced = false
- private contextCacheSynced = false
- private mcpServerCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
- private contextCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
- private mcpServerCacheRecoveryInFlight: Promise<boolean> | null = null
- private contextCacheRecoveryInFlight: Promise<boolean> | null = null
// A watch callback can still settle after its stream reports completion.
// Host reconciles change the mcp-host pod template, so stale callbacks must
// never replay an older Host spec after a replacement watch is active.
@@ -670,18 +585,11 @@ export class McpServerWatcher implements McpServerProvider {
private readonly hostWatchRetryAttempts = new Map<string, number>()
private hostCacheSynced = false
private hostCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
- private sfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null
- private gfsWatchRestartTimer: ReturnType<typeof setTimeout> | null = null
// Host watch LIST-to-WATCH recovery is a dedicated, deduplicated operation
// (§10.2). Concurrent recovery signals reuse the in-flight promise so exactly
// one LIST + WATCH is installed. Independent per-Host events no longer share a
// process-wide convergence tail — they enter the per-Host serializer directly.
- private hostRecoveryInFlight: ActiveHostInventoryRecovery | null = null
- private hostCacheRecoveryIntent: HostInventoryRecoveryRequest | null = null
- // Only the cold-start Host fleet pass waits for the initial SFS inventory.
- // The inventory itself remains asynchronous with respect to provider.start
- // and NetworkPolicy readiness certification.
- private initialHostFleetSfsInventory: Promise<boolean> | null = null
+ private hostRecoveryInFlight: Promise<HostCRD[]> | null = null
// B2: tracks whether the CC snapshot is paired with a continuing watch.
// Set to true only while a complete snapshot is paired with a live watch;
// used by HostReconciler to make fail-closed lifecycle decisions.
@@ -689,16 +597,16 @@ export class McpServerWatcher implements McpServerProvider {
private ccWatchGeneration = 0
private ccCacheRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private ccCacheRecoveryInFlight: Promise<boolean> | null = null
- private readonly hostFleetScheduler = new HostFleetScheduler({
- perform: request => this.performHostFleetReconcile(request),
- recordRequest: result => hostFleetRequestsTotal.inc({ result }),
+ private ccLifecycleGeneration = 0
+ private ccAppliedLifecycleGeneration = -1
+ private ccFleetRetryTimer: ReturnType<typeof setTimeout> | null = null
+ private ccFleetRetryAttempt = 0
+ private hostFleetReconcileInFlight: ActiveHostFleetReconcile | null = null
+ private hostFleetReconcilePending: PendingHostFleetReconcile | null = null
+ private resolveHostFleetShutdown: () => void = () => {}
+ private readonly hostFleetShutdown = new Promise<void>(resolve => {
+ this.resolveHostFleetShutdown = resolve
})
- private get ccLifecycleGeneration(): number {
- return this.hostFleetScheduler.currentLifecycleGeneration
- }
- private get ccAppliedLifecycleGeneration(): number {
- return this.hostFleetScheduler.appliedLifecycleGeneration
- }
private hostResyncInFlight: Promise<void> | null = null
private changeCallback?: () => void
private stopped = false
@@ -710,46 +618,15 @@ export class McpServerWatcher implements McpServerProvider {
private gfsReconciler: GfsReconciler
private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
private readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter
- private readonly externalEgressCoordinator = new ExternalEgressConvergenceCoordinator({
- listServers: () => [...this.servers.values()],
- getCurrentServer: name => this.servers.get(name),
- inventoryAuthoritative: () => !this.stopped && this.mcpServerCacheSynced,
- sameDesiredRevision: sameMcpServerDesiredRevision,
- enqueue: (server, work) => this.enqueueMcpServerReconciliation(server, work),
- mutate: (type, server, options) => this.performExternalEgressMutation(type, server, options),
- replay: (type, server, retry) =>
- this.reconcileMcpServerWatchEvent(type, server, this.mcpWatchGeneration, retry),
- externalEgressRefreshMinTtlMs: () => this.netPolReconciler.externalEgressRefreshMinTtlMs,
- })
- private readonly initialConvergenceRetryTimers = new Map<
- InitialConvergenceLane,
- ReturnType<typeof setTimeout>
- >()
- private readonly initialConvergenceRetryAttempts = new Map<InitialConvergenceLane, number>()
- private readonly initialConvergenceRuns = new Map<
- InitialConvergenceLane,
- ActiveInitialConvergenceRun
- >()
- // Every resource effect serializes by resource identity, so one slow fleet
- // member cannot block unrelated live events while same-resource updates
- // still retain last-writer ordering.
- private readonly mcpServerReconciliationQueues = new Map<string, KeyedReconciliationQueue>()
- private readonly contextReconciliationQueues = new Map<string, KeyedReconciliationQueue>()
- private readonly sharedFileSystemReconciliationQueues = new Map<
- string,
- KeyedReconciliationQueue
- >()
- private readonly globalFileSystemReconciliationQueues = new Map<
- string,
- KeyedReconciliationQueue
- >()
- private sharedFileSystemCacheRevision = 0
- private globalFileSystemCacheRevision = 0
+ private readonly externalEgressRetryTimers = new Map<string, ReturnType<typeof setTimeout>>()
+ private readonly externalEgressRetryAttempts = new Map<string, number>()
+ private readonly externalEgressInFlight = new Map<string, Promise<void>>()
// Periodic resync timer: K8s watches can drop events on long disconnects;
// running fullReconcile every N minutes guarantees mcp-host-runtime-token Secret
// rotation eventually catches up even after a missed MODIFIED event.
// Disabled when interval <= 0 (tests).
private resyncTimer: ReturnType<typeof setInterval> | null = null
+ private externalEgressResyncTimer: ReturnType<typeof setTimeout> | null = null
// Periodic SharedFileSystem resync (#592): the SFS watch fires only on SFS CRD
// changes, not on PVC binding / wfc pod readiness, so a SharedFileSystem that
// reported Initializing/Degraded needs a periodic re-reconcile to converge to a
@@ -767,11 +644,6 @@ export class McpServerWatcher implements McpServerProvider {
}
this.watch = new k8s.Watch(kc)
this.reconciler = new McpServerReconciler(kc)
- this.reconciler.setInventoryAuthority(() => ({
- known: this.mcpServerCacheSynced,
- generation: this.mcpWatchGeneration,
- }))
- this.reconciler.setResolveCurrentServer(name => this.servers.get(name))
const infrastructureTelemetryReporter = config.controlApiBaseUrl
? createInfrastructureTelemetryReporter(config.governedTracingEnabled, {
baseUrl: config.controlApiBaseUrl,
@@ -821,14 +693,6 @@ export class McpServerWatcher implements McpServerProvider {
// entries a delete/newer event has since removed), and orphan cleanup
// compares candidates with the CURRENT inventory rather than a pass snapshot.
this.hostReconciler.setResolveCurrentHost(name => this.hosts.get(name))
- // H2: reflect committed lifecycle outcomes onto the CURRENT cache entry —
- // but never onto a same-name recreation. uid === undefined fails closed
- // (production watch objects always carry uid).
- this.hostReconciler.setReflectHostOutcome((name, uid, apply) => {
- const cached = this.hosts.get(name)
- if (!cached || uid === undefined || cached.uid !== uid) return
- apply(cached)
- })
// Orphan cleanup requires known watch authority and a stable watch
// generation; expose both so cleanup fail-closes while authority is unknown
// or the watch has been retired mid-pass.
@@ -836,40 +700,6 @@ export class McpServerWatcher implements McpServerProvider {
known: this.hostCacheSynced,
generation: this.hostWatchGeneration,
}))
- this.hostReconciler.setHostMutationAuthority(() => ({
- known: this.hostCacheSynced && this.contextCacheSynced,
- hostGeneration: this.hostWatchGeneration,
- contextGeneration: this.contextWatchGeneration,
- }))
- this.hostReconciler.setResolveHostMutationDependencies(host => {
- const context = this.contexts.get(host.spec.contextRef)
- // Lease desired semantics, not cache-object identity or observed status.
- // Kubernetes watch events replace objects and status-only changes may
- // legitimately advance resourceVersion without changing Host inputs.
- const desiredRevision = <T extends { name: string; namespace: string; spec: unknown }>(
- resource: T | undefined
- ) =>
- resource
- ? { name: resource.name, namespace: resource.namespace, spec: resource.spec }
- : undefined
- const dependencies: unknown[] = [
- desiredRevision(context),
- this.ccCacheSynced,
- this.ccWatchGeneration,
- ]
- for (const ref of context?.spec.sharedFileSystems ?? []) {
- const sfs = this.sharedFileSystems.get(ref.name)
- dependencies.push(
- desiredRevision(sfs),
- sfs ? this.sharedFileSystemReconciler.isMountable(sfs) : false
- )
- }
- const channels = this.findCommunicationChannelsByHostRef(host.name).sort((a, b) =>
- a.name.localeCompare(b.name)
- )
- dependencies.push(...channels.map(channel => desiredRevision(channel)))
- return dependencies
- })
// Per-CC credentials Secret migration: HostReconciler computes the
// channel-reader credentials-revision annotation by hashing the Secret(s)
// referenced by all CCs of a host. Wire both finders so
@@ -898,487 +728,65 @@ export class McpServerWatcher implements McpServerProvider {
return this.ccCacheSynced
}
- /**
- * Readiness authority covers the inventories that back HCC discovery,
- * Context policy decisions, and Host API/runtime ownership. Communication
- * Channel loss is intentionally excluded: that subsystem already fails safe
- * by preserving replicas and disabling stateless suspension until recovery.
- * SFS/GFS retain their established eventual-resync contract.
- */
- isReadinessInventoryAuthoritative(): boolean {
- return (
- !this.stopped &&
- this.mcpServerCacheSynced &&
- this.contextCacheSynced &&
- this.hostCacheSynced &&
- // The generation/revision equalities below prove no newer desired state
- // is outstanding; they cannot prove the last pass actually revoked what
- // it set out to revoke. Losing a delete fence bumps no generation and
- // moves no revision, so it is invisible to every other term here while
- // leaving live an allow the pass classified as stale. The reconciler
- // degrades this fence at the point of loss; readiness reads it so the
- // gate re-closes for the same reason it opened.
- this.netPolReconciler.hasCertifiedSafetyInventory() &&
- this.networkPolicyRevocationContextGeneration === this.contextWatchGeneration &&
- this.networkPolicyRevocationServerGeneration === this.mcpWatchGeneration &&
- this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&
- this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision
- )
- }
-
- /**
- * Host-managed runtime mutation requires a LIST snapshot paired with its
- * continuing WATCH. This narrower predicate is also used by non-HTTP effect
- * producers while the broader readiness predicate additionally covers
- * McpServer and Context discovery authority.
- */
- isHostInventoryAuthoritative(): boolean {
- return !this.stopped && this.hostCacheSynced
- }
-
- /**
- * Host runtime state is derived from both the Host and Context inventories:
- * Context owns the SharedFileSystem mount set rendered into each Host pod.
- * Mutation must therefore stop when either LIST -> WATCH pair is unavailable.
- */
- private isHostEffectInventoryAuthoritative(): boolean {
- return this.isHostInventoryAuthoritative() && this.contextCacheSynced
- }
-
- /**
- * Fail closed at every Host-effect admission boundary while recovery owns
- * inventory authority. Preserve the reason/generation in the retry intent so
- * the successful LIST -> WATCH recovery's full pass applies the deferred
- * effect from current state instead of acting on the stale cache.
- */
- private admitHostDependentEffects(
- convergenceReason: string,
- ccLifecycleGeneration?: number
- ): boolean {
- if (this.isHostEffectInventoryAuthoritative()) return true
- if (!this.stopped && !this.hostCacheSynced) {
- this.scheduleHostCacheRecovery({
- convergenceReason,
- ccLifecycleGeneration,
- cause: 'watch-recovery',
- })
- }
- return false
- }
-
- private retireMcpServerWatch(expectedGeneration?: number): boolean {
- if (expectedGeneration !== undefined && expectedGeneration !== this.mcpWatchGeneration) {
- return false
- }
- this.mcpServerCacheSynced = false
- this.mcpWatchGeneration += 1
- this.mcpWatchRequest?.abort()
- this.mcpWatchRequest = null
- return true
- }
-
- private retireContextWatch(expectedGeneration?: number): boolean {
- if (expectedGeneration !== undefined && expectedGeneration !== this.contextWatchGeneration) {
- return false
- }
- this.contextCacheSynced = false
- this.contextWatchGeneration += 1
- this.ctxWatchRequest?.abort()
- this.ctxWatchRequest = null
- return true
- }
-
- private async enqueueKeyedReconciliation(
- queues: Map<string, KeyedReconciliationQueue>,
- key: string,
- work: () => Promise<void>
- ): Promise<void> {
- let entry = queues.get(key)
- if (!entry) {
- entry = { queue: new SerializedReconciliationQueue(), references: 0 }
- queues.set(key, entry)
- }
- entry.references++
- try {
- await entry.queue.enqueue(async () => {
- if (this.stopped) return
- await work()
- })
- } finally {
- entry.references--
- if (entry.references === 0 && queues.get(key) === entry) queues.delete(key)
- }
- }
-
- private enqueueMcpServerReconciliation(
- server: Pick<McpServerCRD, 'name' | 'namespace'>,
- work: () => Promise<void>
- ): Promise<void> {
- return this.enqueueKeyedReconciliation(
- this.mcpServerReconciliationQueues,
- `${server.namespace}/${server.name}`,
- work
- )
- }
-
- private enqueueContextReconciliation(
- contextId: string,
- work: () => Promise<void>
- ): Promise<void> {
- return this.enqueueKeyedReconciliation(this.contextReconciliationQueues, contextId, work)
- }
-
- private enqueueContextIdentityReconciliation(
- contextIds: string[],
- work: () => Promise<void>
- ): Promise<void> {
- const keys = [...new Set(contextIds)].sort()
- const enter = (index: number): Promise<void> => {
- const key = keys[index]
- if (!key) return work()
- return this.enqueueContextReconciliation(key, () => enter(index + 1))
- }
- return enter(0)
- }
-
- private hasMcpServerInventoryAuthority(watchGeneration: number): boolean {
- return this.mcpServerCacheSynced && this.mcpWatchGeneration === watchGeneration
- }
-
- private hasContextInventoryAuthority(watchGeneration: number): boolean {
- return this.contextCacheSynced && this.contextWatchGeneration === watchGeneration
- }
-
- private async mcpServerAbsentForDelete(
- name: string,
- namespace: string,
- watchGeneration: number
- ): Promise<boolean> {
- if (!customObjectsApi) return false
- return confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => this.hasMcpServerInventoryAuthority(watchGeneration),
- resolveCurrent: () => this.servers.get(name),
- readCurrent: () =>
- customObjectsApi.getNamespacedCustomObject({
- group: GROUP,
- version: VERSION,
- namespace,
- plural: PLURAL_MCPSERVERS,
- name,
- }),
- })
- }
-
- private async authorizeMcpServerDeleteOrRetry(
- server: McpServerCRD,
- watchGeneration: number
- ): Promise<boolean> {
- try {
- const authorized = await this.mcpServerAbsentForDelete(
- server.name,
- server.namespace,
- watchGeneration
- )
- if (!authorized && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
- console.warn(
- `[K8s] McpServer inventory authority changed while deleting ${server.name}; ` +
- 'full cleanup deferred until retry'
- )
- this.scheduleExternalEgressRetry('DELETED', server)
- this.changeCallback?.()
- }
- return authorized
- } catch (error) {
- console.error(
- `[K8s] Authoritative absence check failed for deleted McpServer ${server.name}; ` +
- 'cleanup blocked until retry:',
- error
- )
- this.scheduleExternalEgressRetry('DELETED', server)
- this.changeCallback?.()
- return false
- }
- }
-
- private async contextAbsentForDelete(
- name: string,
- namespace: string,
- watchGeneration: number
- ): Promise<boolean> {
- if (
- !this.contextCacheSynced ||
- this.contextWatchGeneration !== watchGeneration ||
- this.contexts.has(name) ||
- !customObjectsApi
- ) {
- return false
- }
- try {
- await customObjectsApi.getNamespacedCustomObject({
- group: GROUP,
- version: VERSION,
- namespace,
- plural: PLURAL_CONTEXTS,
- name,
- })
- return false
- } catch (error: unknown) {
- if (getErrorCode(error) !== 404) throw error
- }
- return (
- this.contextCacheSynced &&
- this.contextWatchGeneration === watchGeneration &&
- !this.contexts.has(name)
- )
- }
-
- private async contextIdAbsentForDelete(
- contextId: string,
- watchGeneration: number
- ): Promise<boolean> {
- const inventoryAuthoritative = () =>
- this.contextCacheSynced && this.contextWatchGeneration === watchGeneration
- const resolveCurrent = () =>
- [...this.contexts.values()].find(context => context.spec.contextId === contextId)
- if (!customObjectsApi || !inventoryAuthoritative() || resolveCurrent()) return false
-
- const response = (await customObjectsApi.listNamespacedCustomObject({
- group: GROUP,
- version: VERSION,
- namespace: config.namespace,
- plural: PLURAL_CONTEXTS,
- })) as { items?: Array<{ spec?: { contextId?: string } }> }
- if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false
-
- return inventoryAuthoritative() && !resolveCurrent()
- }
-
- private enqueueSharedFileSystemReconciliation(
- name: string,
- work: () => Promise<void>
- ): Promise<void> {
- return this.enqueueKeyedReconciliation(this.sharedFileSystemReconciliationQueues, name, work)
- }
-
- private enqueueGlobalFileSystemReconciliation(
- name: string,
- work: () => Promise<void>
- ): Promise<void> {
- return this.enqueueKeyedReconciliation(this.globalFileSystemReconciliationQueues, name, work)
- }
-
- /**
- * Fleet passes acquire every selected object lane in stable order. This
- * preserves their snapshot-wide ordering without making two unrelated live
- * watch events wait behind one another.
- */
- private enqueueKeyedFleetReconciliation(
- queues: Map<string, KeyedReconciliationQueue>,
- keys: string[],
- work: () => Promise<void>
- ): Promise<void> {
- const orderedKeys = [...new Set(keys)].sort()
- const enter = (index: number): Promise<void> => {
- const key = orderedKeys[index]
- if (!key) return work()
- return this.enqueueKeyedReconciliation(queues, key, () => enter(index + 1))
- }
- return enter(0)
- }
-
- private enqueueSharedFileSystemFleetReconciliation(
- names: string[],
- work: () => Promise<void>
- ): Promise<void> {
- return this.enqueueKeyedFleetReconciliation(
- this.sharedFileSystemReconciliationQueues,
- names,
- work
- )
- }
-
- private enqueueGlobalFileSystemFleetReconciliation(
- names: string[],
- work: () => Promise<void>
- ): Promise<void> {
- return this.enqueueKeyedFleetReconciliation(
- this.globalFileSystemReconciliationQueues,
- names,
- work
- )
- }
-
- private requireInventoryResourceVersion(
- kind: 'McpServer' | 'Context',
- resourceVersion: string | undefined
- ): string {
- if (!resourceVersion) {
- throw new Error(`${kind} snapshot missing resourceVersion`)
+ private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {
+ this.communicationChannels.clear()
+ for (const channel of snapshot.channels) {
+ this.communicationChannels.set(channel.name, channel)
}
- return resourceVersion
}
- private installMcpServerSnapshot(snapshot: McpServerSnapshot): void {
- this.servers.clear()
- for (const server of snapshot.servers) {
- this.servers.set(server.name, server)
+ private beginCommunicationChannelLifecycleTransition(): number {
+ this.ccLifecycleGeneration += 1
+ this.ccFleetRetryAttempt = 0
+ if (this.ccFleetRetryTimer) {
+ clearTimeout(this.ccFleetRetryTimer)
+ this.ccFleetRetryTimer = null
}
+ return this.ccLifecycleGeneration
}
- private installContextSnapshot(snapshot: ContextSnapshot): void {
- this.contexts.clear()
- for (const context of snapshot.contexts) {
- this.contexts.set(context.name, context)
+ private markCommunicationChannelLifecycleApplied(generation: number): void {
+ if (generation !== this.ccLifecycleGeneration) return
+ this.ccAppliedLifecycleGeneration = generation
+ this.ccFleetRetryAttempt = 0
+ if (this.ccFleetRetryTimer) {
+ clearTimeout(this.ccFleetRetryTimer)
+ this.ccFleetRetryTimer = null
}
}
- private async restartMcpServerWatch(snapshot: McpServerSnapshot): Promise<number> {
- const resourceVersion = this.requireInventoryResourceVersion(
- 'McpServer',
- snapshot.resourceVersion
- )
- this.mcpServerCacheSynced = false
- this.installMcpServerSnapshot(snapshot)
- const watchGeneration = await this.startMcpServerWatch(resourceVersion)
+ private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {
+ const generation = request.ccLifecycleGeneration
+ if (generation === undefined) return
if (
this.stopped ||
- watchGeneration !== this.mcpWatchGeneration ||
- this.mcpWatchRequest === null
+ generation !== this.ccLifecycleGeneration ||
+ generation === this.ccAppliedLifecycleGeneration ||
+ this.ccFleetRetryTimer
) {
- throw new Error('McpServer snapshot could not be paired with an active watch')
+ return
}
- this.mcpServerCacheSynced = true
- return watchGeneration
- }
-
- private async restartContextWatch(snapshot: ContextSnapshot): Promise<number> {
- const resourceVersion = this.requireInventoryResourceVersion(
- 'Context',
- snapshot.resourceVersion
+ const delayIndex = Math.min(
+ this.ccFleetRetryAttempt,
+ COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS.length - 1
)
- this.contextCacheSynced = false
- this.installContextSnapshot(snapshot)
- const watchGeneration = await this.startContextWatch(resourceVersion)
- if (
- this.stopped ||
- watchGeneration !== this.contextWatchGeneration ||
- this.ctxWatchRequest === null
- ) {
- throw new Error('Context snapshot could not be paired with an active watch')
- }
- this.contextCacheSynced = true
- return watchGeneration
- }
-
- private recoverMcpServerInventoryAndWatch(): Promise<boolean> {
- if (this.stopped) return Promise.resolve(false)
- if (this.mcpServerCacheRecoveryInFlight) return this.mcpServerCacheRecoveryInFlight
-
- const recovery = (async () => {
- try {
- const snapshot = await listMcpServerSnapshot()
- if (this.stopped) return false
- await this.restartMcpServerWatch(snapshot)
- if (this.stopped) return false
- this.changeCallback?.()
- if (this.contextCacheSynced) {
- void this.runInitialNetworkPolicyConvergence()
- }
- return true
- } catch (error) {
- this.mcpServerCacheSynced = false
- console.error('[K8s] McpServer cache recovery failed:', error)
- return false
- }
- })()
-
- this.mcpServerCacheRecoveryInFlight = recovery
- void recovery.finally(() => {
- if (this.mcpServerCacheRecoveryInFlight === recovery) {
- this.mcpServerCacheRecoveryInFlight = null
- }
- })
- return recovery
- }
-
- private recoverContextInventoryAndWatch(): Promise<boolean> {
- if (this.stopped) return Promise.resolve(false)
- if (this.contextCacheRecoveryInFlight) return this.contextCacheRecoveryInFlight
-
- const recovery = (async () => {
- try {
- const snapshot = await listContextSnapshot()
- if (this.stopped) return false
- await this.restartContextWatch(snapshot)
- if (this.stopped) return false
- this.hostFleetScheduler.bumpInputRevision()
- void this.runInitialNetworkPolicyConvergence()
- void this.runInitialSharedFileSystemConvergence()
- if (this.hostCacheSynced) {
- const pendingCcLifecycleGeneration =
- this.ccLifecycleGeneration > this.ccAppliedLifecycleGeneration
- ? this.ccLifecycleGeneration
- : undefined
- void this.requestHostFleetReconcile(
- 'Context cache recovery',
- pendingCcLifecycleGeneration,
- 'full'
- )
- }
- return true
- } catch (error) {
- this.contextCacheSynced = false
- console.error('[K8s] Context cache recovery failed:', error)
- return false
- }
- })()
-
- this.contextCacheRecoveryInFlight = recovery
- void recovery.finally(() => {
- if (this.contextCacheRecoveryInFlight === recovery) {
- this.contextCacheRecoveryInFlight = null
+ const retryDelay = COMMUNICATION_CHANNEL_FLEET_RETRY_DELAYS_MS[delayIndex]
+ this.ccFleetRetryAttempt += 1
+ this.ccFleetRetryTimer = setTimeout(() => {
+ this.ccFleetRetryTimer = null
+ if (
+ this.stopped ||
+ generation !== this.ccLifecycleGeneration ||
+ generation === this.ccAppliedLifecycleGeneration
+ ) {
+ return
}
- })
- return recovery
- }
-
- private scheduleMcpServerCacheRecovery(): void {
- if (this.stopped || this.mcpServerCacheRecoveryTimer) return
- this.mcpServerCacheRecoveryTimer = setTimeout(() => {
- this.mcpServerCacheRecoveryTimer = null
- void this.recoverMcpServerInventoryAndWatch().then(recovered => {
- if (!recovered) this.scheduleMcpServerCacheRecovery()
- })
- }, INVENTORY_CACHE_RECOVERY_RETRY_MS)
- }
-
- private scheduleContextCacheRecovery(): void {
- if (this.stopped || this.contextCacheRecoveryTimer) return
- this.contextCacheRecoveryTimer = setTimeout(() => {
- this.contextCacheRecoveryTimer = null
- void this.recoverContextInventoryAndWatch().then(recovered => {
- if (!recovered) this.scheduleContextCacheRecovery()
- })
- }, INVENTORY_CACHE_RECOVERY_RETRY_MS)
- }
-
- private installCommunicationChannelSnapshot(snapshot: CommunicationChannelSnapshot): void {
- this.communicationChannels.clear()
- for (const channel of snapshot.channels) {
- this.communicationChannels.set(channel.name, channel)
- }
- }
-
- private beginCommunicationChannelLifecycleTransition(): number {
- return this.hostFleetScheduler.beginLifecycleTransition()
- }
-
- private markCommunicationChannelLifecycleApplied(generation: number): void {
- this.hostFleetScheduler.markLifecycleApplied(generation)
- }
-
- private scheduleCommunicationChannelFleetRetry(request: HostFleetReconcileRequest): void {
- this.hostFleetScheduler.scheduleLifecycleRetry(request)
+ void this.requestHostFleetReconcile(
+ 'CommunicationChannel lifecycle convergence retry',
+ generation,
+ request.mode
+ )
+ }, retryDelay)
}
/**
@@ -1476,45 +884,19 @@ export class McpServerWatcher implements McpServerProvider {
* Hosts through their per-Host chains (step 7) and requests — without awaiting
* — a coalesced background full pass (step 8).
*/
- private recoverHostInventoryAndWatch(
- convergenceReason = 'Host watch recovery convergence',
- ccLifecycleGeneration?: number,
- cause: HostInventoryRecoveryCause = 'watch-recovery'
- ): Promise<HostCRD[]> {
+ private recoverHostInventoryAndWatch(): Promise<HostCRD[]> {
if (this.stopped) return Promise.resolve([])
- const requested = { convergenceReason, ccLifecycleGeneration, cause }
- // A scheduled retry and an immediate caller are two entrances to the same
- // monotonic recovery intent. Adopt the pending request before choosing or
- // joining the active operation so a direct recovery cannot erase a stronger
- // cold-start cause or newer lifecycle generation when it cancels the timer.
- const request = this.hostCacheRecoveryIntent
- ? { ...this.hostCacheRecoveryIntent }
- : { ...requested }
- if (this.hostCacheRecoveryIntent) {
- mergeHostInventoryRecoveryRequest(request, requested)
- this.hostCacheRecoveryIntent = null
- if (this.hostCacheRecoveryTimer) {
- clearTimeout(this.hostCacheRecoveryTimer)
- this.hostCacheRecoveryTimer = null
- }
- }
- if (this.hostRecoveryInFlight) {
- mergeHostInventoryRecoveryRequest(this.hostRecoveryInFlight.request, request)
- return this.hostRecoveryInFlight.promise
- }
- const recovery = this.performHostInventoryRecovery(request)
- const active = { request, promise: recovery }
- this.hostRecoveryInFlight = active
+ if (this.hostRecoveryInFlight) return this.hostRecoveryInFlight
+ const recovery = this.performHostInventoryRecovery()
+ this.hostRecoveryInFlight = recovery
const clear = (): void => {
- if (this.hostRecoveryInFlight === active) this.hostRecoveryInFlight = null
+ if (this.hostRecoveryInFlight === recovery) this.hostRecoveryInFlight = null
}
void recovery.then(clear, clear)
return recovery
}
- private async performHostInventoryRecovery(
- request: HostInventoryRecoveryRequest
- ): Promise<HostCRD[]> {
+ private async performHostInventoryRecovery(): Promise<HostCRD[]> {
const startedAt = Date.now()
// Capture the pre-recovery inventory so step 7 can distinguish genuinely
// new/changed Hosts from ones the fleet pass will converge anyway.
@@ -1559,58 +941,24 @@ export class McpServerWatcher implements McpServerProvider {
{ phase: 'total', outcome: 'success' },
(Date.now() - startedAt) / 1000
)
- // Effect producers can discover newer work while LIST/WATCH recovery is
- // already in flight. Fold that scheduled intent into this successful
- // operation before cancelling its timer so no lifecycle generation,
- // convergence reason, or stronger cold-start cause is discarded.
- if (this.hostCacheRecoveryIntent) {
- mergeHostInventoryRecoveryRequest(request, this.hostCacheRecoveryIntent)
- }
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
- this.hostCacheRecoveryIntent = null
- if (request.cause === 'watch-recovery') {
- this.enqueueRecoveredUrgentHosts(
- snapshot.hosts,
- previousNames,
- previousGenerations,
- previousUids
- )
- // Addendum 4 (#827): a Host present before recovery and absent from the
- // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate
- // per-Host delete cleanup so a DELETE lost to watch retirement is never
- // silently dropped.
- this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)
- }
+ this.enqueueRecoveredUrgentHosts(
+ snapshot.hosts,
+ previousNames,
+ previousGenerations,
+ previousUids
+ )
+ // Addendum 4 (#827): a Host present before recovery and absent from the
+ // fresh authoritative snapshot genuinely disappeared. Enqueue an immediate
+ // per-Host delete cleanup so a DELETE lost to watch retirement is never
+ // silently dropped.
+ this.enqueueRecoveredHostDeletes(snapshot.hosts, previousNames)
// Request a coalesced background full pass, but do NOT await it before
- // declaring watch recovery complete. The cold-start pass alone waits for
- // the concurrent SFS LIST: otherwise it can write every Host template
- // without its declared mounts, then immediately roll the fleet again
- // when SFS discovery finishes. NetworkPolicy readiness and start() stay
- // independent from that finite inventory request.
- const requestFleetPass = () => {
- if (this.stopped) return
- void this.requestHostFleetReconcile(
- request.convergenceReason,
- request.ccLifecycleGeneration,
- 'full'
- )
- }
- const initialSfsInventory =
- request.cause === 'cold-start' ? this.initialHostFleetSfsInventory : null
- if (initialSfsInventory) {
- // Waiting for the inventory is deliberate: a Host fleet pass without it
- // writes mount-less templates and immediately rerolls the whole fleet.
- // The wait cannot strand the fleet, because the LIST underneath runs on
- // the deadline-bearing client: an apiserver that never answers aborts
- // the request, `startSharedFileSystemInventoryAndWatch` logs and
- // resolves false, and this settles. One path, one pass.
- void initialSfsInventory.then(() => requestFleetPass())
- } else {
- requestFleetPass()
- }
+ // declaring watch recovery complete.
+ void this.requestHostFleetReconcile('Host watch recovery convergence', undefined, 'full')
return snapshot.hosts
} catch (error) {
// Keep authority unknown, record the failure, and use the existing
@@ -1622,7 +970,7 @@ export class McpServerWatcher implements McpServerProvider {
)
if (!this.stopped) {
console.error('[K8s] Host watch recovery failed:', error)
- this.scheduleHostCacheRecovery(request)
+ this.scheduleHostCacheRecovery()
}
throw error
}
@@ -1662,7 +1010,6 @@ export class McpServerWatcher implements McpServerProvider {
private async dispatchUrgentHostReconcile(name: string): Promise<void> {
if (this.stopped) return
- if (!this.admitHostDependentEffects('Recovered urgent Host convergence')) return
const host = this.hosts.get(name)
if (!host) return
try {
@@ -1694,7 +1041,6 @@ export class McpServerWatcher implements McpServerProvider {
private async dispatchRecoveredHostDelete(name: string): Promise<void> {
if (this.stopped) return
- if (!this.admitHostDependentEffects('Recovered Host deletion convergence')) return
// The fresh authoritative LIST already confirmed absence and the cache no
// longer holds this Host, so cleanup is authorized. Route it through the
// per-Host serializer (reconcileDelete → serializeByHost) so an older
@@ -1726,30 +1072,13 @@ export class McpServerWatcher implements McpServerProvider {
}
}
- private scheduleHostCacheRecovery(
- requested: HostInventoryRecoveryRequest = {
- convergenceReason: 'Host watch recovery convergence',
- cause: 'watch-recovery',
- }
- ): void {
- if (this.stopped) return
- if (this.hostCacheRecoveryIntent) {
- mergeHostInventoryRecoveryRequest(this.hostCacheRecoveryIntent, requested)
- } else {
- this.hostCacheRecoveryIntent = { ...requested }
- }
- if (this.hostCacheRecoveryTimer) return
+ private scheduleHostCacheRecovery(): void {
+ if (this.stopped || this.hostCacheRecoveryTimer) return
this.hostCacheRecoveryTimer = setTimeout(() => {
this.hostCacheRecoveryTimer = null
- const request = this.hostCacheRecoveryIntent ?? requested
- this.hostCacheRecoveryIntent = null
// Watch recovery is an independent operation, no longer coupled to a full
// fleet pass. Recovery itself requests the background convergence pass.
- void this.recoverHostInventoryAndWatch(
- request.convergenceReason,
- request.ccLifecycleGeneration,
- request.cause
- ).catch(() => undefined)
+ void this.recoverHostInventoryAndWatch().catch(() => undefined)
}, HOST_CACHE_RECOVERY_RETRY_MS)
}
@@ -1794,7 +1123,6 @@ export class McpServerWatcher implements McpServerProvider {
source: HostReconcileSource = 'urgent'
): Promise<void> {
if (this.stopped) return
- if (!this.admitHostDependentEffects('Host watch event convergence')) return
if (!this.isCurrentHostWatchEvent(type, host, eventRevision)) {
// #827: a DELETE that is no longer current because the cache now holds a
// (recreated) same-name Host is a superseded delete — record it and do
@@ -1889,7 +1217,6 @@ export class McpServerWatcher implements McpServerProvider {
await this.recoverHostInventoryAndWatch()
if (this.stopped) return
}
- if (!this.isHostEffectInventoryAuthoritative()) return
// Reconcile the CURRENT cache. Fleet workers dispatch Host keys and
// resolve the freshest cached spec at execution (see collectHostReconcile
// Failures); only fullReconcile runs the authority-gated orphan cleanup.
@@ -1952,7 +1279,78 @@ export class McpServerWatcher implements McpServerProvider {
ccLifecycleGeneration?: number,
mode?: HostFleetReconcileMode
): Promise<void> {
- return this.hostFleetScheduler.request(reason, ccLifecycleGeneration, mode)
+ if (this.stopped) return Promise.resolve()
+ const requestMode = mode ?? (ccLifecycleGeneration === undefined ? 'full' : 'lifecycle')
+ if (
+ ccLifecycleGeneration !== undefined &&
+ (ccLifecycleGeneration !== this.ccLifecycleGeneration ||
+ (requestMode === 'lifecycle' &&
+ ccLifecycleGeneration === this.ccAppliedLifecycleGeneration))
+ ) {
+ return Promise.resolve()
+ }
+
+ const request: HostFleetReconcileRequest = {
+ reason,
+ mode: requestMode,
+ ccLifecycleGeneration,
+ }
+ const active = this.hostFleetReconcileInFlight
+ if (!active) {
+ hostFleetRequestsTotal.inc({ result: 'started' })
+ return this.waitForHostFleetOrShutdown(this.startHostFleetReconcile(request))
+ }
+ if (hostFleetRequestCovers(active, request)) {
+ hostFleetRequestsTotal.inc({ result: 'coalesced' })
+ return this.waitForHostFleetOrShutdown(active.promise)
+ }
+
+ if (!this.hostFleetReconcilePending) {
+ let resolve!: () => void
+ const promise = new Promise<void>(resolvePromise => {
+ resolve = resolvePromise
+ })
+ this.hostFleetReconcilePending = {
+ ...request,
+ promise,
+ resolve,
+ }
+ } else {
+ const merged = mergeHostFleetRequests(this.hostFleetReconcilePending, request)
+ this.hostFleetReconcilePending.reason = merged.reason
+ this.hostFleetReconcilePending.mode = merged.mode
+ this.hostFleetReconcilePending.ccLifecycleGeneration = merged.ccLifecycleGeneration
+ }
+ // The request did not start its own pass; it is queued behind the active one.
+ hostFleetRequestsTotal.inc({ result: 'coalesced' })
+ return this.waitForHostFleetOrShutdown(this.hostFleetReconcilePending.promise)
+ }
+
+ private waitForHostFleetOrShutdown(coverage: Promise<void>): Promise<void> {
+ return Promise.race([coverage, this.hostFleetShutdown])
+ }
+
+ private startHostFleetReconcile(request: HostFleetReconcileRequest): Promise<void> {
+ const promise = this.performHostFleetReconcile(request)
+ const active: ActiveHostFleetReconcile = { ...request, promise }
+ this.hostFleetReconcileInFlight = active
+
+ const settle = () => {
+ if (this.hostFleetReconcileInFlight !== active) return
+ this.hostFleetReconcileInFlight = null
+ const pending = this.hostFleetReconcilePending
+ this.hostFleetReconcilePending = null
+ if (!pending) return
+ if (this.stopped) {
+ pending.resolve()
+ return
+ }
+ hostFleetRequestsTotal.inc({ result: 'trailing' })
+ const trailing = this.startHostFleetReconcile(pending)
+ void trailing.then(pending.resolve, pending.resolve)
+ }
+ void promise.then(settle, settle)
+ return promise
}
private contextsReferencingSfs(sfsName: string): Array<{ namespace: string; name: string }> {
@@ -2012,35 +1410,14 @@ export class McpServerWatcher implements McpServerProvider {
return out
}
- private reconcileSharedFileSystemsReferencedByContext(
- prevContext: ContextCRD | undefined,
- nextContext: ContextCRD | undefined,
- isCurrent: () => boolean = () => true
- ): Promise<void> {
- const names = [
- ...(prevContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),
- ...(nextContext?.spec.sharedFileSystems ?? []).map(ref => ref.name),
- ]
- return this.enqueueSharedFileSystemFleetReconciliation(names, async () => {
- if (!isCurrent()) return
- await this.reconcileSharedFileSystemsReferencedByContextCore(
- prevContext,
- nextContext,
- isCurrent
- )
- })
- }
-
- private async reconcileSharedFileSystemsReferencedByContextCore(
+ private async reconcileSharedFileSystemsReferencedByContext(
prevContext: ContextCRD | undefined,
- nextContext: ContextCRD | undefined,
- isCurrent: () => boolean
+ nextContext: ContextCRD | undefined
): Promise<void> {
const names = new Set<string>()
for (const r of prevContext?.spec.sharedFileSystems ?? []) names.add(r.name)
for (const r of nextContext?.spec.sharedFileSystems ?? []) names.add(r.name)
for (const name of names) {
- if (!isCurrent()) return
const sfs = this.sharedFileSystems.get(name)
if (!sfs) continue
try {
@@ -2057,9 +1434,6 @@ export class McpServerWatcher implements McpServerProvider {
* Deployment template picks up (or drops) the volume + volumeMount.
*/
private async reconcileHostsReferencingSfs(sfsName: string): Promise<void> {
- if (!this.admitHostDependentEffects(`SharedFileSystem "${sfsName}" Host convergence`)) {
- return
- }
const affectedContexts = new Set<string>()
for (const ctx of this.contexts.values()) {
if (ctx.spec.sharedFileSystems?.some(r => r.name === sfsName)) {
@@ -2077,24 +1451,6 @@ export class McpServerWatcher implements McpServerProvider {
}
}
- /**
- * Reconcile Hosts affected by a Context change only while the Host cache is
- * authoritative. A recovery full pass consumes the current Context cache, so
- * deferral loses no effect and never acts on an unpaired Host LIST.
- */
- private async reconcileHostsReferencingContext(
- contextName: string,
- isCurrent: () => boolean = () => true
- ): Promise<void> {
- if (!isCurrent()) return
- if (!this.admitHostDependentEffects(`Context "${contextName}" Host convergence`)) return
- for (const host of this.hosts.values()) {
- if (!isCurrent()) return
- if (host.spec.contextRef !== contextName) continue
- await this.hostReconciler.reconcile(host)
- }
- }
-
/**
* Re-reconcile the Host whose name matches the given hostRef. Called by the
* CommunicationChannel watch handler so per-Host channel-reader Deployment
@@ -2107,9 +1463,6 @@ export class McpServerWatcher implements McpServerProvider {
* its caller schedules bounded fleet convergence.
*/
private async reconcileHostsReferencingCC(hostRef: string): Promise<boolean> {
- if (!this.admitHostDependentEffects(`CommunicationChannel Host "${hostRef}" convergence`)) {
- return false
- }
const host = this.hosts.get(hostRef)
if (!host) return true
try {
@@ -2201,28 +1554,9 @@ export class McpServerWatcher implements McpServerProvider {
* tracker resolves the heartbeat's hostRef through this).
*/
getHost(name: string): HostCRD | undefined {
- if (!this.isHostInventoryAuthoritative()) return undefined
return this.hosts.get(name)
}
- /**
- * Route channel credentials-Secret events through the same Host authority
- * boundary as watch, Context, SFS, and heartbeat effects.
- */
- async reconcileChannelReaderRevision(secretName: string, secretNamespace: string): Promise<void> {
- const convergenceReason = `CommunicationChannel Secret "${secretNamespace}/${secretName}" Host convergence`
- if (!this.ccCacheSynced) {
- console.warn(
- `[K8s] Deferring ${convergenceReason}; CommunicationChannel cache is not authoritative`
- )
- return
- }
- if (!this.admitHostDependentEffects(convergenceReason)) {
- return
- }
- await this.hostReconciler.reconcileChannelReaderRevision(secretName, secretNamespace)
- }
-
/**
* Trigger a reconcile for any cached McpServer whose `envSecret.name`
* matches the given Secret. Used by SecretInformer to react to Secret
@@ -2230,45 +1564,18 @@ export class McpServerWatcher implements McpServerProvider {
* event.
*/
async reconcileByEnvSecret(secretName: string, secretNamespace: string): Promise<void> {
- const inventoryGeneration = this.mcpWatchGeneration
- if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
- const selected = [...this.servers.values()].filter(
- server => server.namespace === secretNamespace && server.spec.envSecret?.name === secretName
- )
- await Promise.all(
- selected.map(server =>
- this.enqueueMcpServerReconciliation(server, async () => {
- if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return
- const current = this.servers.get(server.name)
- if (
- !current ||
- current.namespace !== secretNamespace ||
- current.spec.envSecret?.name !== secretName
- ) {
- return
- }
- try {
- console.log(
- `[K8s] Re-reconciling McpServer "${current.name}" after Secret "${secretName}" change`
- )
- await this.reconciler.reconcile(current, {
- isCurrent: () => {
- if (!this.hasMcpServerInventoryAuthority(inventoryGeneration)) return false
- const latest = this.servers.get(current.name)
- return (
- latest !== undefined &&
- latest.namespace === secretNamespace &&
- latest.spec.envSecret?.name === secretName &&
- sameMcpServerDesiredRevision(current, latest)
- )
- },
- })
- } catch (err) {
- console.error(`[K8s] Secret-triggered reconcile failed for "${current.name}":`, err)
- }
- })
- )
- )
+ for (const server of this.servers.values()) {
+ if (server.namespace !== secretNamespace) continue
+ if (server.spec.envSecret?.name !== secretName) continue
+ try {
+ console.log(
+ `[K8s] Re-reconciling McpServer "${server.name}" after Secret "${secretName}" change`
+ )
+ await this.reconciler.reconcile(server)
+ } catch (err) {
+ console.error(`[K8s] Secret-triggered reconcile failed for "${server.name}":`, err)
+ }
+ }
}
/**
@@ -2296,7 +1603,7 @@ export class McpServerWatcher implements McpServerProvider {
transport,
auth: server.spec.auth,
enabled: server.spec.enabled !== false,
- status: this.reconciler.getStatus(server),
+ status: this.reconciler.getStatus(server.name),
}
}
@@ -2351,55 +1658,61 @@ export class McpServerWatcher implements McpServerProvider {
* Start watching for changes and run initial reconciliation.
*/
async start(): Promise<void> {
- // L0/L1 isolation and removal of legacy permissive policies are mandatory
- // bootstrap barriers. They are a fixed-size safety operation, independent
- // of fleet size, and must succeed before HCC can watch or reconcile runtimes.
- await this.netPolReconciler.ensureDefaultPolicies()
-
- // Discovery and policy caches become authoritative only when each complete
- // LIST snapshot is paired with a watch continuing from its opaque collection
- // resourceVersion. Full fleet convergence happens later in the background.
- // A transient LIST/WATCH failure on either inventory must not kill the
- // retry that restart*Watch already armed (it sets *CacheSynced=false before
- // it can throw). Mirror the Host lane below: log, keep the flag false, and
- // schedule in-process recovery. The HTTP server stays live, dynamic
- // readiness stays 503, and recovery promotes readiness only after a fresh
- // LIST is paired with its continuing WATCH.
+ let initialServers: McpServerCRD[] = []
+ let serverInventoryComplete = false
try {
- const initialServerSnapshot = await listMcpServerSnapshot()
- await this.restartMcpServerWatch(initialServerSnapshot)
+ initialServers = await listAllMcpServers()
+ serverInventoryComplete = true
+ for (const server of initialServers) {
+ this.servers.set(server.name, server)
+ }
} catch (error) {
- this.mcpServerCacheSynced = false
console.error(
- '[K8s] Initial McpServer inventory is unavailable; HCC remains unready while in-process recovery continues:',
+ '[K8s] Skipping initial McpServer full reconciliation because server discovery failed:',
error
)
}
- if (!this.mcpServerCacheSynced) {
- this.scheduleMcpServerCacheRecovery()
+
+ // Ensure baseline network policies exist even if initial Context discovery fails.
+ try {
+ await this.netPolReconciler.ensureDefaultPolicies()
+ } catch (error) {
+ console.error('[K8s] Failed to ensure default NetworkPolicies before startup:', error)
+ }
+
+ let initialExternalEgressFailures = new Set<string>()
+ if (serverInventoryComplete) {
+ initialExternalEgressFailures =
+ await this.reconcileInitialExternalEgressBeforeRuntime(initialServers)
+ }
+
+ // Full McpServer reconciliation (Deployments + Services). Servers with
+ // egressBindings are reconciled only after their external egress policies
+ // have converged, matching the ADDED/MODIFIED watch-event contract.
+ if (serverInventoryComplete) {
+ const runtimeServers = initialServers.filter(
+ server => !initialExternalEgressFailures.has(this.externalEgressRetryKey(server))
+ )
+ console.log('[K8s] Running initial full reconciliation...')
+ await this.reconciler.fullReconcile(runtimeServers)
}
try {
- const initialContextSnapshot = await listContextSnapshot()
- await this.restartContextWatch(initialContextSnapshot)
+ const initialContexts = await listAllContexts()
+ for (const context of initialContexts) {
+ this.contexts.set(context.name, context)
+ }
+ console.log('[K8s] Running initial NetworkPolicy reconciliation...')
+ await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
+ serverInventoryComplete,
+ ensureDefaults: false,
+ })
} catch (error) {
- this.contextCacheSynced = false
console.error(
- '[K8s] Initial Context inventory is unavailable; HCC remains unready while in-process recovery continues:',
+ '[K8s] Skipping initial NetworkPolicy reconciliation because context discovery failed:',
error
)
}
- if (!this.contextCacheSynced) {
- this.scheduleContextCacheRecovery()
- }
-
- // Host templates resolve Context SharedFileSystem references from this
- // cache. Inventory is therefore a fixed startup boundary, not part of the
- // unbounded SFS reconciliation fleet: letting the first Host pass run
- // without it writes mount-less templates and immediately rolls the entire
- // Host fleet a second time once discovery catches up.
- const initialSharedFileSystemInventory = this.startSharedFileSystemInventoryAndWatch()
- this.initialHostFleetSfsInventory = initialSharedFileSystemInventory
// ── CommunicationChannel snapshot + watch — MUST complete before Host fullReconcile ──
// A stateless Host may suspend only after this controller has a complete
@@ -2448,30 +1761,54 @@ export class McpServerWatcher implements McpServerProvider {
}
console.log(
- `[K8s] Starting initial Host background convergence... (ccCacheSynced=${this.ccCacheSynced})`
+ `[K8s] Running initial Host reconciliation... (ccCacheSynced=${this.ccCacheSynced})`
)
const initialLifecycleGeneration =
this.ccLifecycleGeneration === 0
? this.beginCommunicationChannelLifecycleTransition()
: this.ccLifecycleGeneration
+ await this.requestHostFleetReconcile(
+ 'initial Host reconciliation',
+ initialLifecycleGeneration,
+ 'full'
+ )
+
+ // ── SharedFileSystem initial load + reconciliation ──
try {
- await this.recoverHostInventoryAndWatch(
- 'initial Host reconciliation',
- initialLifecycleGeneration,
- 'cold-start'
+ const initialSfses = await listAllSharedFileSystems()
+ for (const sfs of initialSfses) {
+ this.sharedFileSystems.set(sfs.name, sfs)
+ }
+ console.log('[K8s] Running initial SharedFileSystem reconciliation...')
+ await this.sharedFileSystemReconciler.fullReconcile(initialSfses)
+ } catch (error) {
+ console.error(
+ '[K8s] Skipping initial SharedFileSystem reconciliation because discovery failed:',
+ error
)
+ }
+
+ // ── GlobalFileSystem (gfs) initial load + reconciliation ──
+ // gfs is a cluster singleton (DISTINCT from SharedFileSystem). A failed
+ // discovery (e.g. the CRD not installed yet) is skipped, not fatal — the
+ // next HCC restart / resync retries. Live updates land via the watch
+ // (follow-up); the deploy applies the singleton before HCC reconciles it.
+ try {
+ const initialGfses = await listAllGlobalFileSystems()
+ console.log('[K8s] Running initial GlobalFileSystem reconciliation...')
+ await this.gfsReconciler.fullReconcile(initialGfses)
} catch (error) {
- // Host inventory is an essential readiness authority, but a transient
- // LIST/WATCH failure must not kill the retry that recovery just armed.
- // Continue startup with hostCacheSynced=false: the HTTP server stays live,
- // dynamic readiness remains 503, and recovery promotes readiness only
- // after a fresh LIST is paired with its continuing WATCH.
- console.warn(
- '[K8s] Initial Host inventory is unavailable; HCC remains unready while in-process recovery continues:',
+ console.error(
+ '[K8s] Skipping initial GlobalFileSystem reconciliation because discovery failed:',
error
)
}
+ await this.startMcpServerWatch()
+ await this.startContextWatch()
+ await this.startSharedFileSystemWatch()
+ await this.startGlobalFileSystemWatch()
+
const resyncSec = config.hostResyncIntervalSec
if (resyncSec > 0) {
this.resyncTimer = setInterval(() => {
@@ -2509,375 +1846,35 @@ export class McpServerWatcher implements McpServerProvider {
}
const externalEgressResyncSec = config.externalEgressResyncIntervalSec
- // #205 delegates external-egress periodic resync to the convergence
- // coordinator. Its runResyncCore drives reconcileExternalEgress, so the
- // #299 sliding-window accumulation still converges on DNS rotation between
- // McpServer events. The coordinator owns cadence with a self-rescheduling
- // timer that advances to <= observed TTL/2 (H2, issue #299), bounded by the
- // configured floor.
- this.externalEgressCoordinator.startPeriodicResync(
- externalEgressResyncSec,
- config.externalEgressRefreshFloorSec
- )
-
- // Full convergence is observable and retains each lane's existing retry or
- // periodic-resync contract, but it no longer extends provider.start().
- // NetworkPolicy safety starts first and launches McpServer convergence from
- // its certification callback, before additive policy convergence completes.
- // This prevents the startup egress refresh from racing or undoing the safety
- // pass while keeping readiness independent from both additive fleets.
- void this.runInitialNetworkPolicyConvergence()
- void initialSharedFileSystemInventory.then(inventoryComplete => {
- if (inventoryComplete && !this.stopped) {
- void this.runInitialSharedFileSystemConvergence()
- }
- })
- void this.startGlobalFileSystemBackgroundLane()
- }
-
- private async startSharedFileSystemInventoryAndWatch(): Promise<boolean> {
- let inventoryComplete = false
- try {
- const initialSfses = await listAllSharedFileSystems()
- inventoryComplete = true
- for (const sfs of initialSfses) this.sharedFileSystems.set(sfs.name, sfs)
- } catch (error) {
- console.error(
- '[K8s] Skipping initial SharedFileSystem background convergence because discovery failed:',
- error
- )
- }
- // The cold-start Host fleet pass gates on this promise, so it must settle on
- // the inventory alone. `k8s.Watch` carries no transport deadline — the
- // client builds `AbortSignal.any([controller, timeout])` and then overwrites
- // it with the bare controller — so awaiting the watch start here would
- // re-strand the fleet behind an already-certified readiness, which is the
- // exact failure bounding the LIST was meant to close.
- void this.startSharedFileSystemWatch().catch(error => {
- console.error('[K8s] SharedFileSystem background watch failed to start:', error)
- this.scheduleSharedFileSystemWatchRestart(5000)
- })
- if (inventoryComplete && !this.stopped) {
- return true
- }
- return false
- }
-
- private async startGlobalFileSystemBackgroundLane(): Promise<void> {
- let inventoryComplete = false
- try {
- const initialGfses = await listAllGlobalFileSystems()
- inventoryComplete = true
- for (const gfs of initialGfses) this.globalFileSystems.set(gfs.name, gfs)
- } catch (error) {
- console.error(
- '[K8s] Skipping initial GlobalFileSystem background convergence because discovery failed:',
- error
- )
- }
- try {
- await this.startGlobalFileSystemWatch()
- } catch (error) {
- console.error('[K8s] GlobalFileSystem background watch failed to start:', error)
- this.scheduleGlobalFileSystemWatchRestart(5000)
- }
- if (inventoryComplete && !this.stopped) {
- void this.runInitialGlobalFileSystemConvergence()
- }
- }
-
- private runInitialMcpServerConvergence(): Promise<void> {
- return this.runInitialConvergence('McpServer')
- }
-
- private currentNetworkPolicySafetyCertificate(): NetworkPolicySafetyCertificate | null {
- if (
- this.stopped ||
- !this.contextCacheSynced ||
- !this.mcpServerCacheSynced ||
- !this.netPolReconciler.hasCertifiedSafetyInventory() ||
- this.networkPolicyRevocationContextGeneration !== this.contextWatchGeneration ||
- this.networkPolicyRevocationServerGeneration !== this.mcpWatchGeneration ||
- this.networkPolicyRevocationContextRevision !== this.contextDesiredRevision ||
- this.networkPolicyRevocationServerRevision !== this.mcpServerDesiredRevision
- ) {
- return null
- }
- return {
- contextGeneration: this.contextWatchGeneration,
- serverGeneration: this.mcpWatchGeneration,
- contextRevision: this.contextDesiredRevision,
- serverRevision: this.mcpServerDesiredRevision,
- }
- }
-
- private recordNetworkPolicySafetyCertificate(
- certificate: NetworkPolicySafetyCertificate
- ): boolean {
- if (
- this.stopped ||
- !this.contextCacheSynced ||
- !this.mcpServerCacheSynced ||
- this.contextWatchGeneration !== certificate.contextGeneration ||
- this.mcpWatchGeneration !== certificate.serverGeneration ||
- this.contextDesiredRevision !== certificate.contextRevision ||
- this.mcpServerDesiredRevision !== certificate.serverRevision
- ) {
- return false
- }
- this.networkPolicyRevocationContextGeneration = certificate.contextGeneration
- this.networkPolicyRevocationServerGeneration = certificate.serverGeneration
- this.networkPolicyRevocationContextRevision = certificate.contextRevision
- this.networkPolicyRevocationServerRevision = certificate.serverRevision
- return true
- }
-
- private isNetworkPolicySafetyCertificateCurrent(
- certificate: NetworkPolicySafetyCertificate
- ): boolean {
- const current = this.currentNetworkPolicySafetyCertificate()
- return (
- current !== null &&
- current.contextGeneration === certificate.contextGeneration &&
- current.serverGeneration === certificate.serverGeneration &&
- current.contextRevision === certificate.contextRevision &&
- current.serverRevision === certificate.serverRevision
- )
- }
-
- private async runInitialMcpServerConvergenceCore(): Promise<void> {
- const safetyCertificate = this.currentNetworkPolicySafetyCertificate()
- if (safetyCertificate === null) return
- const inventoryGeneration = this.mcpWatchGeneration
- const inventoryAuthoritative = () =>
- this.hasMcpServerInventoryAuthority(inventoryGeneration) &&
- this.isNetworkPolicySafetyCertificateCurrent(safetyCertificate)
- try {
- const initialServers = [...this.servers.values()]
- const initialExternalEgressGates = this.externalEgressCoordinator.prepareStartupGates(
- initialServers,
- inventoryAuthoritative
- )
- if (this.stopped) return
- console.log('[K8s] Running initial McpServer background reconciliation...')
- await this.reconciler.fullReconcile(initialServers, {
- runEffect: async (serverName, work) => {
- const selected = this.servers.get(serverName)
- const laneOwner = selected ?? {
- name: serverName,
- namespace: config.namespace,
- }
- if (!(await initialExternalEgressGates.waitFor(laneOwner))) return
- if (!inventoryAuthoritative()) return
- await this.enqueueMcpServerReconciliation(laneOwner, async () => {
- if (!inventoryAuthoritative()) return
- await work()
- })
- },
- })
- if (!inventoryAuthoritative()) return
- initialConvergenceLastSuccessTimestampSeconds.set({ lane: 'McpServer' }, Date.now() / 1000)
- this.clearInitialConvergenceRetry('McpServer')
- } catch (error) {
- console.error('[K8s] Initial McpServer background reconciliation failed:', error)
- this.scheduleInitialConvergenceRetry('McpServer')
- }
- }
-
- private runInitialNetworkPolicyConvergence(): Promise<void> {
- return this.runInitialConvergence('NetworkPolicy')
- }
-
- private runInitialConvergence(lane: InitialConvergenceLane): Promise<void> {
- const active = this.initialConvergenceRuns.get(lane)
- if (active) {
- active.trailingRequested = true
- return active.promise
- }
-
- const run: ActiveInitialConvergenceRun = {
- trailingRequested: false,
- promise: Promise.resolve(),
- }
- this.initialConvergenceRuns.set(lane, run)
- run.promise = Promise.resolve()
- .then(async () => {
- do {
- run.trailingRequested = false
- if (lane === 'McpServer') {
- await this.runInitialMcpServerConvergenceCore()
- } else {
- await this.runInitialNetworkPolicyConvergenceCore()
- }
- } while (!this.stopped && run.trailingRequested)
- // Retire the active slot synchronously before this promise settles.
- // A request arriving in the following settlement microtask must start
- // a new run instead of attaching to a pass whose loop already exited.
- if (this.initialConvergenceRuns.get(lane) === run) {
- this.initialConvergenceRuns.delete(lane)
- }
- })
- .finally(() => {
- // Errors bypass the normal retirement above.
- if (this.initialConvergenceRuns.get(lane) === run) {
- this.initialConvergenceRuns.delete(lane)
- }
- })
- return run.promise
- }
-
- private async runInitialNetworkPolicyConvergenceCore(): Promise<void> {
- // A NetworkPolicy full pass combines authoritative Context/McpServer allow
- // revocation with additive Context policy effects. External egress
- // creation and DNS refresh have one separate owner: the startup/resync
- // coordinator. Running safety from a partial inventory would be neither a
- // full convergence nor safe evidence of success. Recovery of either
- // missing LIST -> WATCH pair schedules a fresh current-cache pass.
- if (!this.contextCacheSynced || !this.mcpServerCacheSynced) return
- let authoritativeRevocationCompleted = false
- try {
- const initialContexts = [...this.contexts.values()]
- const initialServers = [...this.servers.values()]
- const contextInventoryGeneration = this.contextWatchGeneration
- const serverInventoryGeneration = this.mcpWatchGeneration
- const safetyCertificate: NetworkPolicySafetyCertificate = {
- contextGeneration: contextInventoryGeneration,
- serverGeneration: serverInventoryGeneration,
- contextRevision: this.contextDesiredRevision,
- serverRevision: this.mcpServerDesiredRevision,
- }
- const serverInventoryComplete = this.mcpServerCacheSynced
- const contextInventoryAuthoritative = () =>
- this.hasContextInventoryAuthority(contextInventoryGeneration)
- const serverInventoryAuthoritative = () =>
- this.hasMcpServerInventoryAuthority(serverInventoryGeneration)
- console.log('[K8s] Running initial NetworkPolicy background reconciliation...')
- await this.netPolReconciler.fullReconcile(initialContexts, initialServers, {
- serverInventoryComplete,
- ensureDefaults: false,
- contextInventoryAuthoritative,
- serverInventoryAuthoritative,
- runContextEffect: (contextId, work) =>
- this.enqueueContextReconciliation(contextId, async () => {
- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return
- await work()
- }),
- runServerEffect: (serverName, work) => {
- const selected = this.servers.get(serverName)
- return this.enqueueMcpServerReconciliation(
- selected ?? { name: serverName, namespace: config.namespace },
- async () => {
- if (!serverInventoryAuthoritative()) return
- await work()
- }
- )
- },
- resolveCurrentContext: name => this.contexts.get(name),
- resolveCurrentContextById: contextId =>
- [...this.contexts.values()].find(context => context.spec.contextId === contextId),
- resolveCurrentServer: name => this.servers.get(name),
- contextDesiredRevision: () => this.contextDesiredRevision,
- serverDesiredRevision: () => this.mcpServerDesiredRevision,
- // B3 piece-2: queue an additive recreation for every server whose
- // external-egress allow the authoritative pass revoked — fired at the
- // deletion, so an abort/throw cannot strand a revoked-but-desired allow.
- // The coordinator maps against the current cache (absent → DELETED, no
- // recreate) and rebuilds from the current spec with fresh DNS.
- onExternalEgressRevoked: server =>
- this.externalEgressCoordinator.scheduleRetry('MODIFIED', server),
- onAuthoritativeRevocationComplete: () => {
- if (!this.recordNetworkPolicySafetyCertificate(safetyCertificate)) return
- authoritativeRevocationCompleted = true
- // Start runtime convergence at the safety boundary, before this full
- // pass enters additive Context work. Startup egress gates may now
- // preserve only policies certified by this exact authoritative
- // inventory generation.
- void this.runInitialMcpServerConvergence()
- },
- })
- if (!contextInventoryAuthoritative() || !serverInventoryAuthoritative()) return
- initialConvergenceLastSuccessTimestampSeconds.set(
- { lane: 'NetworkPolicy' },
- Date.now() / 1000
+ if (externalEgressResyncSec > 0) {
+ // Self-rescheduling (not a fixed setInterval) so the delay can advance to
+ // <= observed TTL/2 (H2, issue #299) and rescheduling AFTER each pass
+ // completes guarantees no overlapping resyncs.
+ const scheduleNext = (): void => {
+ if (this.stopped) return
+ const delayMs = externalEgressResyncDelayMs(
+ externalEgressResyncSec,
+ config.externalEgressRefreshFloorSec,
+ this.netPolReconciler.externalEgressRefreshMinTtlMs
+ )
+ this.externalEgressResyncTimer = setTimeout(() => {
+ void this.runExternalEgressResync().finally(() => scheduleNext())
+ }, delayMs)
+ this.externalEgressResyncTimer.unref?.()
+ }
+ // Idempotency guard (audit F5): never arm a second self-rescheduling chain
+ // if one is already live (start() runs once today, but a future restart
+ // path must not double the cadence).
+ if (!this.externalEgressResyncTimer) scheduleNext()
+ console.log(
+ `[K8s] External egress periodic DNS resync enabled (every ${externalEgressResyncSec}s, advancing to <= TTL/2)`
)
- this.clearInitialConvergenceRetry('NetworkPolicy')
- } catch (error) {
- console.error(
- authoritativeRevocationCompleted
- ? '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed:'
- : '[K8s] Initial NetworkPolicy background reconciliation failed:',
- error
+ } else {
+ console.warn(
+ '[K8s] External egress periodic DNS resync DISABLED (HCC_EXTERNAL_EGRESS_RESYNC_SEC=0); ' +
+ 'exact-host FQDN egress will NOT converge on DNS rotation between McpServer events — ' +
+ 'accumulated IPs can go stale and block workloads (issue #299). Enable it unless you have a specific reason.'
)
- this.scheduleInitialConvergenceRetry('NetworkPolicy')
- }
- }
-
- private clearInitialConvergenceRetry(lane: InitialConvergenceLane): void {
- const timer = this.initialConvergenceRetryTimers.get(lane)
- if (timer) {
- clearTimeout(timer)
- this.initialConvergenceRetryTimers.delete(lane)
- }
- this.initialConvergenceRetryAttempts.delete(lane)
- }
-
- private scheduleInitialConvergenceRetry(lane: InitialConvergenceLane): void {
- if (this.stopped || this.initialConvergenceRetryTimers.has(lane)) return
-
- const attempt = (this.initialConvergenceRetryAttempts.get(lane) ?? 0) + 1
- const delayMs =
- INITIAL_CONVERGENCE_RETRY_DELAYS_MS[
- Math.min(attempt - 1, INITIAL_CONVERGENCE_RETRY_DELAYS_MS.length - 1)
- ]
- this.initialConvergenceRetryAttempts.set(lane, attempt)
- initialConvergenceRetriesTotal.inc({ lane })
- console.warn(
- `[K8s] Scheduling initial ${lane} background convergence retry ${attempt} in ${delayMs}ms`
- )
-
- const timer = setTimeout(() => {
- this.initialConvergenceRetryTimers.delete(lane)
- if (this.stopped) return
- if (lane === 'McpServer') {
- void this.runInitialMcpServerConvergence()
- return
- }
- void this.runInitialNetworkPolicyConvergence()
- }, delayMs)
- this.initialConvergenceRetryTimers.set(lane, timer)
- }
-
- private runInitialSharedFileSystemConvergence(): Promise<void> {
- const names = [...this.sharedFileSystems.keys()]
- return this.enqueueSharedFileSystemFleetReconciliation(names, () =>
- this.runInitialSharedFileSystemConvergenceCore()
- )
- }
-
- private async runInitialSharedFileSystemConvergenceCore(): Promise<void> {
- try {
- const initialSfses = [...this.sharedFileSystems.values()]
- console.log('[K8s] Running initial SharedFileSystem background reconciliation...')
- await this.sharedFileSystemReconciler.fullReconcile(initialSfses)
- } catch (error) {
- console.error('[K8s] Initial SharedFileSystem background reconciliation failed:', error)
- }
- }
-
- private runInitialGlobalFileSystemConvergence(): Promise<void> {
- const names = [...this.globalFileSystems.keys()]
- return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>
- this.runInitialGlobalFileSystemConvergenceCore()
- )
- }
-
- private async runInitialGlobalFileSystemConvergenceCore(): Promise<void> {
- try {
- const initialGfses = [...this.globalFileSystems.values()]
- console.log('[K8s] Running initial GlobalFileSystem background reconciliation...')
- await this.gfsReconciler.fullReconcile(initialGfses)
- } catch (error) {
- console.error('[K8s] Initial GlobalFileSystem background reconciliation failed:', error)
}
}
@@ -2905,58 +1902,42 @@ export class McpServerWatcher implements McpServerProvider {
} else if (type === 'DELETED') {
this.sharedFileSystems.delete(sfs.name)
}
- this.sharedFileSystemCacheRevision += 1
- await this.enqueueSharedFileSystemReconciliation(sfs.name, async () => {
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- await this.sharedFileSystemReconciler.reconcile(sfs)
- } else if (type === 'DELETED') {
- await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)
- }
- } catch (error) {
- console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.sharedFileSystemReconciler.reconcile(sfs)
+ } else if (type === 'DELETED') {
+ await this.sharedFileSystemReconciler.reconcileDelete(sfs.name, sfs.namespace, sfs.spec)
}
+ } catch (error) {
+ console.error(`[K8s] SharedFileSystem reconciliation failed for ${sfs.name}:`, error)
+ }
- // Re-reconcile any Host whose Context references this SharedFileSystem.
- // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on
- // the next mcp-host pod template diff so the pod stops failing to mount.
- try {
- await this.reconcileHostsReferencingSfs(sfs.name)
- } catch (error) {
- console.error(
- `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,
- error
- )
- }
- })
+ // Re-reconcile any Host whose Context references this SharedFileSystem.
+ // ADDED/MODIFIED: pick up the now-resolvable mount; DELETED: drop it on
+ // the next mcp-host pod template diff so the pod stops failing to mount.
+ try {
+ await this.reconcileHostsReferencingSfs(sfs.name)
+ } catch (error) {
+ console.error(
+ `[K8s] Failed to re-reconcile Hosts referencing SharedFileSystem ${sfs.name}:`,
+ error
+ )
+ }
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
- this.sfsWatchRequest = null
if (err) {
console.error('[K8s] SharedFileSystem watch error:', err)
}
console.log('[K8s] SharedFileSystem watch ended, restarting...')
- this.scheduleSharedFileSystemWatchRestart(err ? 5000 : 1000)
+ setTimeout(() => this.startSharedFileSystemWatch(), err ? 5000 : 1000)
}
this.sfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
- private scheduleSharedFileSystemWatchRestart(delayMs: number): void {
- if (this.stopped || this.sfsWatchRestartTimer) return
- this.sfsWatchRestartTimer = setTimeout(() => {
- this.sfsWatchRestartTimer = null
- if (this.stopped) return
- void this.startSharedFileSystemWatch().catch(error => {
- console.error('[K8s] SharedFileSystem background watch restart failed:', error)
- this.scheduleSharedFileSystemWatchRestart(5000)
- })
- }, delayMs)
- }
-
/**
* Watch the GlobalFileSystem singleton (gfs). ADDED/MODIFIED → reconcile the
* drive stack; DELETED → tear it down (respecting retainOnDelete). DISTINCT
@@ -2970,63 +1951,37 @@ export class McpServerWatcher implements McpServerProvider {
const watchCallback = async (
type: string,
- apiObj: {
- metadata: { name: string; namespace?: string }
- spec: GlobalFileSystemSpec
- status?: GlobalFileSystemCRD['status']
- }
+ apiObj: { metadata: { name: string; namespace?: string }; spec: GlobalFileSystemSpec }
) => {
const gfs: GlobalFileSystemCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || namespace,
spec: apiObj.spec,
- status: apiObj.status,
}
console.log(`[K8s] GlobalFileSystem watch event: ${type} for ${gfs.name}`)
- if (type === 'ADDED' || type === 'MODIFIED') {
- this.globalFileSystems.set(gfs.name, gfs)
- } else if (type === 'DELETED') {
- this.globalFileSystems.delete(gfs.name)
- }
- this.globalFileSystemCacheRevision += 1
- await this.enqueueGlobalFileSystemReconciliation(gfs.name, async () => {
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- await this.gfsReconciler.reconcile(gfs)
- } else if (type === 'DELETED') {
- await this.gfsReconciler.reconcileDelete(gfs)
- }
- } catch (error) {
- console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.gfsReconciler.reconcile(gfs)
+ } else if (type === 'DELETED') {
+ await this.gfsReconciler.reconcileDelete(gfs)
}
- })
+ } catch (error) {
+ console.error(`[K8s] GlobalFileSystem reconciliation failed for ${gfs.name}:`, error)
+ }
}
const doneCallback = (err: Error | null) => {
if (this.stopped) return
- this.gfsWatchRequest = null
if (err) {
console.error('[K8s] GlobalFileSystem watch error:', err)
}
console.log('[K8s] GlobalFileSystem watch ended, restarting...')
- this.scheduleGlobalFileSystemWatchRestart(err ? 5000 : 1000)
+ setTimeout(() => this.startGlobalFileSystemWatch(), err ? 5000 : 1000)
}
this.gfsWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
- private scheduleGlobalFileSystemWatchRestart(delayMs: number): void {
- if (this.stopped || this.gfsWatchRestartTimer) return
- this.gfsWatchRestartTimer = setTimeout(() => {
- this.gfsWatchRestartTimer = null
- if (this.stopped) return
- void this.startGlobalFileSystemWatch().catch(error => {
- console.error('[K8s] GlobalFileSystem background watch restart failed:', error)
- this.scheduleGlobalFileSystemWatchRestart(5000)
- })
- }, delayMs)
- }
-
/**
* Extracted so unit tests can invoke the handler without spinning up a real
* K8s watch. Production calls this from `startCommunicationChannelWatch` and
@@ -3081,14 +2036,6 @@ export class McpServerWatcher implements McpServerProvider {
if (previousHostRef && previousHostRef !== cc.spec.hostRef) {
affectedHostRefs.add(previousHostRef)
}
- if (!this.isHostInventoryAuthoritative()) {
- const lifecycleGeneration = this.beginCommunicationChannelLifecycleTransition()
- this.admitHostDependentEffects(
- 'CommunicationChannel event Host convergence',
- lifecycleGeneration
- )
- return
- }
let needsFleetRetry = false
for (const hostRef of affectedHostRefs) {
if (!(await this.reconcileHostsReferencingCC(hostRef))) {
@@ -3250,41 +2197,8 @@ export class McpServerWatcher implements McpServerProvider {
private async runGfsResync(): Promise<void> {
if (this.stopped) return
try {
- const cacheRevisionAtListStart = this.globalFileSystemCacheRevision
const gfses = await listAllGlobalFileSystems()
- const names = [
- ...this.globalFileSystems.keys(),
- ...gfses.map(globalFileSystem => globalFileSystem.name),
- ]
- return this.enqueueGlobalFileSystemFleetReconciliation(names, () =>
- this.runGfsResyncCore(gfses, cacheRevisionAtListStart)
- )
- } catch (err) {
- console.error(
- `[K8s] GlobalFileSystem periodic resync failed: ${
- err instanceof Error ? err.message : String(err)
- }`
- )
- }
- }
-
- private async runGfsResyncCore(
- listedGfses?: GlobalFileSystemCRD[],
- listedAtCacheRevision?: number
- ): Promise<void> {
- if (this.stopped) return
- try {
- const cacheRevisionAtListStart = listedAtCacheRevision ?? this.globalFileSystemCacheRevision
- const gfses = listedGfses ?? (await listAllGlobalFileSystems())
- let reconcileInventory: GlobalFileSystemCRD[]
- if (cacheRevisionAtListStart === this.globalFileSystemCacheRevision) {
- this.globalFileSystems.clear()
- for (const gfs of gfses) this.globalFileSystems.set(gfs.name, gfs)
- reconcileInventory = gfses
- } else {
- reconcileInventory = [...this.globalFileSystems.values()]
- }
- await this.gfsReconciler.fullReconcile(reconcileInventory)
+ await this.gfsReconciler.fullReconcile(gfses)
} catch (err) {
console.error(
`[K8s] GlobalFileSystem periodic resync failed: ${
@@ -3297,36 +2211,9 @@ export class McpServerWatcher implements McpServerProvider {
private async runSfsResync(): Promise<void> {
if (this.stopped) return
try {
- const cacheRevisionAtListStart = this.sharedFileSystemCacheRevision
const sfses = await listAllSharedFileSystems()
- const names = [
- ...this.sharedFileSystems.keys(),
- ...sfses.map(sharedFileSystem => sharedFileSystem.name),
- ]
- return this.enqueueSharedFileSystemFleetReconciliation(names, () =>
- this.runSfsResyncCore(sfses, cacheRevisionAtListStart)
- )
- } catch (error) {
- console.error('[K8s] Periodic SharedFileSystem resync failed:', error)
- }
- }
-
- private async runSfsResyncCore(
- listedSfses?: SharedFileSystemCRD[],
- listedAtCacheRevision?: number
- ): Promise<void> {
- if (this.stopped) return
- try {
- const cacheRevisionAtListStart = listedAtCacheRevision ?? this.sharedFileSystemCacheRevision
- const sfses = listedSfses ?? (await listAllSharedFileSystems())
- let reconcileInventory: SharedFileSystemCRD[]
- if (cacheRevisionAtListStart === this.sharedFileSystemCacheRevision) {
- this.sharedFileSystems.clear()
- for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)
- reconcileInventory = sfses
- } else {
- reconcileInventory = [...this.sharedFileSystems.values()]
- }
+ this.sharedFileSystems.clear()
+ for (const sfs of sfses) this.sharedFileSystems.set(sfs.name, sfs)
// Capture each SFS's mountability (PVC Bound) BEFORE reconciling so we can
// detect a flip. resolveContextMounts() injects the RO mount + podAffinity
@@ -3335,11 +2222,11 @@ export class McpServerWatcher implements McpServerProvider {
// on phase==='Ready': a transient wfc readiness dip while the PVC stays
// Bound must NOT re-roll consumers — see resolveContextMounts.)
const wasMountable = new Map<string, boolean>()
- for (const sfs of reconcileInventory) {
+ for (const sfs of sfses) {
wasMountable.set(sfs.name, this.sharedFileSystemReconciler.isMountable(sfs))
}
- await this.sharedFileSystemReconciler.fullReconcile(reconcileInventory)
+ await this.sharedFileSystemReconciler.fullReconcile(sfses)
// #592 gap fix: heal the consuming mcp-host Deployments on a mountability
// flip — inject the mount + affinity once the PVC binds, or drop it if the
@@ -3349,7 +2236,7 @@ export class McpServerWatcher implements McpServerProvider {
// an already-Bound SFS picked up on HCC restart still re-injects its mount
// within one resync interval instead of waiting for the (much longer) Host
// resync.
- for (const sfs of reconcileInventory) {
+ for (const sfs of sfses) {
const nowMountable = this.sharedFileSystemReconciler.isMountable(sfs)
if (nowMountable !== (wasMountable.get(sfs.name) ?? false)) {
await this.reconcileHostsReferencingSfs(sfs.name)
@@ -3360,14 +2247,86 @@ export class McpServerWatcher implements McpServerProvider {
}
}
+ private async runExternalEgressResync(): Promise<void> {
+ if (this.stopped) return
+ const servers = [...this.servers.values()]
+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
+ .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))
+
+ if (servers.length === 0) return
+
+ console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)
+ let index = 0
+ const workers = Array.from(
+ { length: Math.min(EXTERNAL_EGRESS_RESYNC_MAX_CONCURRENCY, servers.length) },
+ async () => {
+ while (!this.stopped) {
+ const server = servers[index++]
+ if (!server) return
+ const key = this.externalEgressRetryKey(server)
+ if (this.externalEgressInFlight.has(key)) {
+ console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)
+ continue
+ }
+ if (this.externalEgressRetryTimers.has(key)) {
+ console.log(`[K8s] Skipping external egress resync for ${key}; retry already scheduled`)
+ continue
+ }
+ const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)
+ if (jitterMs > 0) await delay(jitterMs)
+ try {
+ await this.runExternalEgressOnce('MODIFIED', server)
+ } catch (error) {
+ console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)
+ this.scheduleExternalEgressRetry('MODIFIED', server)
+ }
+ }
+ }
+ )
+ await Promise.all(workers)
+ }
+
+ private async reconcileInitialExternalEgressBeforeRuntime(
+ servers: McpServerCRD[]
+ ): Promise<Set<string>> {
+ const failures = new Set<string>()
+ const serversWithExternalEgress = servers
+ .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
+ .sort((a, b) => `${a.namespace}/${a.name}`.localeCompare(`${b.namespace}/${b.name}`))
+
+ if (serversWithExternalEgress.length === 0) return failures
+
+ console.log(
+ `[K8s] Reconciling external egress before startup runtime reconciliation for ` +
+ `${serversWithExternalEgress.length} McpServer(s)`
+ )
+
+ for (const server of serversWithExternalEgress) {
+ const key = this.externalEgressRetryKey(server)
+ try {
+ await this.runExternalEgressOnce('MODIFIED', server)
+ } catch (error) {
+ failures.add(key)
+ console.error(
+ `[K8s] Initial external egress reconciliation failed for ${key}; ` +
+ 'runtime reconciliation will stay blocked until retry succeeds:',
+ error
+ )
+ this.scheduleExternalEgressRetry('MODIFIED', server)
+ }
+ }
+
+ return failures
+ }
+
/**
* Start watching McpServer CRDs.
*/
- private getMcpServerWatchCallback(
- watchGeneration = this.mcpWatchGeneration
- ): (type: string, apiObj: McpServerWatchObject) => Promise<void> {
+ private getMcpServerWatchCallback(): (
+ type: string,
+ apiObj: McpServerWatchObject
+ ) => Promise<void> {
return async (type, apiObj) => {
- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
const server: McpServerCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.namespace,
@@ -3381,324 +2340,227 @@ export class McpServerWatcher implements McpServerProvider {
console.log(`[K8s] McpServer watch event: ${type} for ${server.name}`)
- const previous = this.servers.get(server.name)
-
// Update cache
- let desiredStateChanged = false
if (type === 'ADDED' || type === 'MODIFIED') {
- desiredStateChanged =
- previous === undefined || !sameMcpServerDesiredRevision(previous, server)
this.servers.set(server.name, server)
} else if (type === 'DELETED') {
- if (!previous?.uid || !server.uid || previous.uid === server.uid) {
- desiredStateChanged = this.servers.delete(server.name)
- }
- }
- if (desiredStateChanged) {
- this.mcpServerDesiredRevision += 1
- void this.runInitialNetworkPolicyConvergence()
+ this.servers.delete(server.name)
}
- // HCC writes McpServer status during reconciliation. Those writes emit
- // MODIFIED events but do not change the desired runtime or policy state.
- // Suppress that self-induced work while still publishing the fresh status
- // through discovery. Any UID/spec/annotation/label change remains live.
- if (type === 'MODIFIED' && previous && isMcpServerStatusOnlyUpdate(previous, server)) {
- this.changeCallback?.()
- return
+ // External egress is part of the workload's pre-start contract. Reconcile
+ // it before HCC creates or updates any managed runtime Deployment so a
+ // stdio MCP with egressBindings cannot start before ExternalEgressReady.
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ try {
+ await this.runExternalEgressOnce(type, server)
+ } catch (error) {
+ console.error(
+ `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
+ error
+ )
+ this.scheduleExternalEgressRetry(type, server)
+ this.changeCallback?.()
+ return
+ }
}
- await this.enqueueMcpServerReconciliation(server, () =>
- this.reconcileMcpServerWatchEvent(
- type as ExternalEgressWatchEventType,
- server,
- watchGeneration
- )
- )
- }
- }
-
- private async reconcileMcpServerWatchEvent(
- type: ExternalEgressWatchEventType,
- server: McpServerCRD,
- watchGeneration: number,
- retry?: ExternalEgressRetryHandle
- ): Promise<void> {
- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
- if (retry && !retry.isCurrent()) return
- if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
- this.scheduleExternalEgressRetry(type, server)
- return
- }
- const current = this.servers.get(server.name)
- if (type === 'DELETED') {
- // A same-name recreation may already be accepted while this callback
- // waits for its per-server effect. The newer identity owns the key.
- if (current) return
- } else {
- if (!current || current.namespace !== server.namespace) return
- server = current
- }
- const deleteAllowed = () =>
- this.mcpServerAbsentForDelete(server.name, server.namespace, watchGeneration)
- const contextInventoryGeneration = this.contextWatchGeneration
- const contextPolicyInventoryIsCurrent = (): boolean =>
- this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
- this.hasContextInventoryAuthority(contextInventoryGeneration)
-
- // External egress is part of the workload's pre-start contract. Reconcile
- // it before HCC creates or updates any managed runtime Deployment so a
- // stdio MCP with egressBindings cannot start before ExternalEgressReady.
- let retryCompletion: ExternalEgressRetryHandle | undefined
- if (type === 'ADDED' || type === 'MODIFIED') {
- try {
- retryCompletion = await this.runExternalEgressOnce(type, server, {
- retry,
- isCurrent: () =>
- this.mcpServerWatchEffectIsCurrent(server, watchGeneration) &&
- (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),
- })
- } catch (error) {
- console.error(
- `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
- error
- )
- this.scheduleExternalEgressRetry(type, server)
- this.changeCallback?.()
- return
- }
- } else {
- if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) return
+ // Trigger deployment reconciliation
try {
- retryCompletion = await this.runExternalEgressOnce(type, server, {
- deleteAllowed,
- retry,
- isCurrent: () =>
- !this.stopped &&
- watchGeneration === this.mcpWatchGeneration &&
- (!retry || this.hasMcpServerInventoryAuthority(watchGeneration)),
- })
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.reconciler.reconcile(server)
+ } else if (type === 'DELETED') {
+ await this.reconciler.reconcileDelete(server.name, server.namespace)
+ }
} catch (error) {
- console.error(
- `[K8s] External egress reconciliation failed for ${server.name}; runtime reconciliation blocked:`,
- error
- )
- this.scheduleExternalEgressRetry(type, server)
- this.changeCallback?.()
- return
- }
- }
-
- if (type === 'ADDED' || type === 'MODIFIED') {
- if (retry && !this.hasMcpServerInventoryAuthority(watchGeneration)) {
- this.scheduleExternalEgressRetry(type, server)
- return
- }
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- server = this.servers.get(server.name)!
- } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {
- return
- }
-
- // Trigger deployment reconciliation.
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- const currentServer = server
- await this.reconciler.reconcile(currentServer, {
- isCurrent: () => this.mcpServerWatchEffectIsCurrent(currentServer, watchGeneration),
- })
- } else if (type === 'DELETED') {
- await this.reconciler.reconcileDelete(server.name, server.namespace)
+ console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)
+ if (
+ (type === 'ADDED' || type === 'MODIFIED') &&
+ (server.spec.egressBindings?.length ?? 0) > 0
+ ) {
+ this.scheduleExternalEgressRetry(type, server)
+ }
}
- } catch (error) {
- console.error(`[K8s] Reconciliation failed for ${server.name}:`, error)
- this.scheduleExternalEgressRetry(type, server)
- this.changeCallback?.()
- return
- }
- if (type === 'ADDED' || type === 'MODIFIED') {
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- server = this.servers.get(server.name)!
- } else if (!(await this.authorizeMcpServerDeleteOrRetry(server, watchGeneration))) {
- return
- }
-
- // Trigger binding policy reconciliation (L3 ingress/egress).
- try {
- if (type === 'ADDED' || type === 'MODIFIED') {
- const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']
- if (bindingsJson) {
- const bindings: BindingDef[] = JSON.parse(bindingsJson)
- const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
- const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name
- const bindingServer = server
- await this.bindingReconciler.reconcileBindings(
- recipeName,
- bindings,
- mcpWorkloadName,
- bindingServer.name,
- {
- isCurrent: () => this.mcpServerWatchEffectIsCurrent(bindingServer, watchGeneration),
+ // Trigger binding policy reconciliation (L3 ingress/egress)
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ const bindingsJson = server.annotations?.['clerum.io/recipe-bindings']
+ if (bindingsJson) {
+ const bindings: BindingDef[] = JSON.parse(bindingsJson)
+ const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
+ const mcpWorkloadName = server.labels?.['clerum.io/workload'] ?? server.name
+ await this.bindingReconciler.reconcileBindings(
+ recipeName,
+ bindings,
+ mcpWorkloadName,
+ server.name
+ )
+ }
+ // Re-reconcile any cached Context that references this server.
+ // Fixes race condition where Context MODIFIED arrives before
+ // the McpServer ADDED event populates the cache.
+ for (const ctx of this.contexts.values()) {
+ if (ctx.spec.mcpServers?.includes(server.name)) {
+ console.log(
+ `[K8s] Re-reconciling context "${ctx.name}" after McpServer "${server.name}" cached`
+ )
+ await this.netPolReconciler.reconcileContext(ctx)
}
- )
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- server = this.servers.get(server.name)!
+ }
+ } else if (type === 'DELETED') {
+ const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
+ await this.bindingReconciler.cleanupBindings(recipeName)
+ await this.runExternalEgressOnce(type, server)
}
- // Re-reconcile cached Contexts after the server cache is populated.
- for (const selectedContext of this.contexts.values()) {
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- if (!selectedContext.spec.mcpServers?.includes(server.name)) continue
- const selectedContextId = selectedContext.spec.contextId
- console.log(
- `[K8s] Re-reconciling context "${selectedContext.name}" after McpServer "${server.name}" cached`
- )
- await this.enqueueContextReconciliation(selectedContextId, async () => {
- if (!this.mcpServerWatchEffectIsCurrent(server, watchGeneration)) return
- const currentContext = this.contexts.get(selectedContext.name)
- if (
- currentContext?.spec.contextId !== selectedContextId ||
- !currentContext.spec.mcpServers?.includes(server.name)
- ) {
- return
- }
- await this.netPolReconciler.reconcileContext(currentContext, {
- isCurrent: () =>
- contextPolicyInventoryIsCurrent() &&
- this.contexts.get(currentContext.name) === currentContext,
- })
- })
+ } catch (error) {
+ console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)
+ if (type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED') {
+ this.scheduleExternalEgressRetry(type, server)
}
- } else if (type === 'DELETED') {
- const recipeName = server.labels?.['clerum.io/recipe'] ?? server.name
- await this.bindingReconciler.cleanupBindings(recipeName, { deleteAllowed })
}
- } catch (error) {
- console.error(`[K8s] Binding/egress reconciliation failed for ${server.name}:`, error)
- this.scheduleExternalEgressRetry(type, server)
+
+ // Notify listeners
this.changeCallback?.()
- return
}
-
- retryCompletion?.complete()
- this.changeCallback?.()
}
- private async startMcpServerWatch(resourceVersion: string): Promise<number> {
- this.requireInventoryResourceVersion('McpServer', resourceVersion)
+ private async startMcpServerWatch(): Promise<void> {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_MCPSERVERS}`
console.log(`[K8s] Starting McpServer watch`)
- const watchGeneration = ++this.mcpWatchGeneration
- if (this.mcpWatchRequest) {
- this.mcpWatchRequest.abort()
- this.mcpWatchRequest = null
- }
- let watchEnded = false
- const watchCallback = this.getMcpServerWatchCallback(watchGeneration)
+ const watchCallback = this.getMcpServerWatchCallback()
const doneCallback = (err: Error | null) => {
- if (watchEnded) return
- watchEnded = true
- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return
- if (!this.retireMcpServerWatch(watchGeneration)) return
+ if (this.stopped) return
if (err) {
console.error('[K8s] McpServer watch error:', err)
}
- console.log('[K8s] McpServer watch ended; recovering authoritative inventory...')
- this.scheduleMcpServerCacheRecovery()
+ console.log('[K8s] McpServer watch ended, restarting...')
+ setTimeout(() => this.startMcpServerWatch(), err ? 5000 : 1000)
}
- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
- if (this.stopped || watchGeneration !== this.mcpWatchGeneration || watchEnded) {
- request.abort()
- return watchGeneration
- }
- this.mcpWatchRequest = request
- return watchGeneration
+ this.mcpWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
- private mcpServerWatchEffectIsCurrent(expected: McpServerCRD, watchGeneration: number): boolean {
- if (this.stopped || watchGeneration !== this.mcpWatchGeneration) return false
- const current = this.servers.get(expected.name)
- return current !== undefined && sameMcpServerDesiredRevision(expected, current)
+ private externalEgressRetryKey(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {
+ return `${server.namespace}/${server.name}`
}
- private runExternalEgressOnce(
- type: ExternalEgressWatchEventType,
+ private async runExternalEgressOnce(
+ type: string,
server: McpServerCRD,
- options: {
- deleteAllowed?: () => Promise<boolean>
- retry?: ExternalEgressRetryHandle
- isCurrent?: () => boolean
- } = {}
- ): Promise<ExternalEgressRetryHandle | undefined> {
- return this.externalEgressCoordinator.reconcile(type, server, options)
- }
+ options: { clearRetryOnSuccess?: boolean } = {}
+ ): Promise<void> {
+ const clearRetryOnSuccess = options.clearRetryOnSuccess ?? true
+ const key = this.externalEgressRetryKey(server)
+ const existing = this.externalEgressInFlight.get(key)
+ if (existing) {
+ console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)
+ await existing
+ }
- private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {
- this.externalEgressCoordinator.scheduleRetry(type, server)
+ const run = (async () => {
+ if (type === 'DELETED') {
+ await this.netPolReconciler.cleanupExternalEgress(server.name, server.namespace)
+ } else {
+ await this.netPolReconciler.reconcileExternalEgress(server)
+ }
+ })()
+ this.externalEgressInFlight.set(key, run)
+ try {
+ await run
+ if (clearRetryOnSuccess) {
+ this.clearExternalEgressRetry(server)
+ }
+ } finally {
+ if (this.externalEgressInFlight.get(key) === run) {
+ this.externalEgressInFlight.delete(key)
+ }
+ }
}
- private async performExternalEgressMutation(
- type: ExternalEgressWatchEventType,
- server: McpServerCRD,
- options: {
- deleteAllowed?: () => Promise<boolean>
- isCurrent: () => boolean
+ private clearExternalEgressRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {
+ const key = this.externalEgressRetryKey(server)
+ const timer = this.externalEgressRetryTimers.get(key)
+ if (timer) {
+ clearTimeout(timer)
+ this.externalEgressRetryTimers.delete(key)
}
- ): Promise<void> {
- if (type === 'DELETED') {
- const deleteAllowed = async (): Promise<boolean> => {
- if (!options.isCurrent()) return false
- if (options.deleteAllowed && !(await options.deleteAllowed())) return false
- return options.isCurrent()
- }
- await this.netPolReconciler.cleanupExternalEgress(
- server.name,
- server.namespace,
- undefined,
- deleteAllowed
+ this.externalEgressRetryAttempts.delete(key)
+ }
+
+ private scheduleExternalEgressRetry(type: string, server: McpServerCRD): void {
+ if (type !== 'ADDED' && type !== 'MODIFIED' && type !== 'DELETED') return
+ if (this.stopped) return
+
+ const key = this.externalEgressRetryKey(server)
+ if (this.externalEgressRetryTimers.has(key)) return
+
+ const attempt = (this.externalEgressRetryAttempts.get(key) ?? 0) + 1
+ const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
+ if (!delayMs) {
+ console.error(
+ `[K8s] External egress retry exhausted for McpServer "${server.name}" in namespace "${server.namespace}"`
)
return
}
- await this.netPolReconciler.reconcileExternalEgress(server, {
- isCurrent: options.isCurrent,
- })
+
+ this.externalEgressRetryAttempts.set(key, attempt)
+ console.warn(
+ `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length} ` +
+ `for McpServer "${server.name}" in ${delayMs}ms`
+ )
+
+ const timer = setTimeout(() => {
+ this.externalEgressRetryTimers.delete(key)
+ void this.retryExternalEgress(type, server)
+ }, delayMs)
+ this.externalEgressRetryTimers.set(key, timer)
+ }
+
+ private async retryExternalEgress(type: string, server: McpServerCRD): Promise<void> {
+ if (this.stopped) return
+ try {
+ if (type === 'DELETED') {
+ await this.runExternalEgressOnce(type, server)
+ } else {
+ const current = this.servers.get(server.name)
+ if (!current || current.namespace !== server.namespace) {
+ this.clearExternalEgressRetry(server)
+ return
+ }
+ await this.runExternalEgressOnce(type, current, { clearRetryOnSuccess: false })
+ try {
+ await this.reconciler.reconcile(current)
+ this.clearExternalEgressRetry(current)
+ } catch (error) {
+ console.error(
+ `[K8s] Runtime reconciliation after external egress retry failed for ${server.name}:`,
+ error
+ )
+ this.scheduleExternalEgressRetry(type, current)
+ }
+ }
+ } catch (error) {
+ console.error(`[K8s] External egress retry failed for ${server.name}:`, error)
+ this.scheduleExternalEgressRetry(type, server)
+ }
}
/**
* Start watching Context CRDs for NetworkPolicy reconciliation.
*/
- private async startContextWatch(resourceVersion: string): Promise<number> {
- this.requireInventoryResourceVersion('Context', resourceVersion)
+ private async startContextWatch(): Promise<void> {
const path = `/apis/${GROUP}/${VERSION}/namespaces/${config.namespace}/${PLURAL_CONTEXTS}`
console.log(`[K8s] Starting Context watch`)
- const watchGeneration = ++this.contextWatchGeneration
- if (this.ctxWatchRequest) {
- this.ctxWatchRequest.abort()
- this.ctxWatchRequest = null
- }
- let watchEnded = false
const watchCallback = async (
type: string,
- apiObj: {
- metadata: {
- name: string
- namespace?: string
- uid?: string
- generation?: number
- }
- spec: ContextSpec
- }
+ apiObj: { metadata: { name: string; namespace?: string }; spec: ContextSpec }
) => {
- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
const context: ContextCRD = {
name: apiObj.metadata.name,
namespace: apiObj.metadata.namespace || config.namespace,
- uid: apiObj.metadata.uid,
- generation: apiObj.metadata.generation,
spec: apiObj.spec,
}
@@ -3707,168 +2569,58 @@ export class McpServerWatcher implements McpServerProvider {
// Capture the previous spec before mutating the cache so we can
// re-reconcile any SFS that *was* referenced but is no longer.
const previous = this.contexts.get(context.name)
- // A scoped Context delta may replace only a prior, fully authoritative
- // safety certificate. Capturing it before this event changes desired
- // state proves there was no outstanding Context or McpServer revocation
- // work that the scoped reconciliation could accidentally certify.
- const previousSafetyCertificate = this.currentNetworkPolicySafetyCertificate()
// Cache the context for cross-resource re-reconciliation
- let desiredStateChanged = false
- let deltaSafetyCertificate: NetworkPolicySafetyCertificate | undefined
if (type === 'ADDED' || type === 'MODIFIED') {
- desiredStateChanged =
- previous === undefined || !sameContextDesiredRevision(previous, context)
this.contexts.set(context.name, context)
} else if (type === 'DELETED') {
- desiredStateChanged = this.contexts.delete(context.name)
- }
- if (desiredStateChanged) {
- this.contextDesiredRevision += 1
- // A same-identity Context MODIFIED event can complete its own stale
- // allow revocation below. Once that exact delta is current, it is safe
- // to restore readiness without waiting for an older pass's additive
- // fleet. Deletes and identity changes remain fail-closed until the
- // authoritative full pass certifies their broader cleanup.
- if (
- type === 'MODIFIED' &&
- previous?.uid === context.uid &&
- previous?.spec.contextId === context.spec.contextId &&
- previousSafetyCertificate !== null
- ) {
- deltaSafetyCertificate = {
- contextGeneration: previousSafetyCertificate.contextGeneration,
- serverGeneration: previousSafetyCertificate.serverGeneration,
- contextRevision: this.contextDesiredRevision,
- serverRevision: previousSafetyCertificate.serverRevision,
- }
- }
- void this.runInitialNetworkPolicyConvergence()
+ this.contexts.delete(context.name)
}
// Re-reconcile every SFS this Context referenced before or after the
// change so SharedFileSystem.status.mountedByContexts stays in sync
// without waiting for an SFS-level event.
const nextForSfs = type === 'DELETED' ? undefined : context
- const contextInventoryAuthoritative = () => this.hasContextInventoryAuthority(watchGeneration)
- const serverInventoryGeneration = this.mcpWatchGeneration
- void this.reconcileSharedFileSystemsReferencedByContext(
- previous,
- nextForSfs,
- contextInventoryAuthoritative
- )
-
- const previousContextId = previous?.spec.contextId
- const effectContextIds =
- previousContextId && previousContextId !== context.spec.contextId
- ? [previousContextId, context.spec.contextId]
- : [context.spec.contextId]
- await this.enqueueContextIdentityReconciliation(effectContextIds, async () => {
- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
+ void this.reconcileSharedFileSystemsReferencedByContext(previous, nextForSfs)
- try {
- if (previousContextId && previousContextId !== context.spec.contextId) {
- const oldIdDeleteAllowed = () =>
- this.contextIdAbsentForDelete(previousContextId, watchGeneration)
- if (await oldIdDeleteAllowed()) {
- await this.netPolReconciler.reconcileDeleteContext(
- previousContextId,
- oldIdDeleteAllowed
- )
- }
- }
-
- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
- const current = this.contexts.get(context.name)
- if (type === 'DELETED') {
- if (current) return
- } else if (current?.spec.contextId !== context.spec.contextId) {
- return
- }
- if (type === 'ADDED' || type === 'MODIFIED') {
- const selectedContext = current!
- // A scoped delta may only replace the safety certificate when its
- // own stale-allow revocation ran to completion — `reconcileContext`
- // aborts mid-pass whenever its authority fence breaks — and when
- // the last authoritative pass certified its namespace-wide
- // inventory, which a label-scoped delta cannot vouch for.
- // `deltaSafetyCertificate` is only ever built for MODIFIED, so this
- // is the one branch where it can be recorded.
- const scopedRevocationCompleted = await this.netPolReconciler.reconcileContext(
- selectedContext,
- {
- isCurrent: () =>
- contextInventoryAuthoritative() &&
- this.hasMcpServerInventoryAuthority(serverInventoryGeneration) &&
- this.contexts.get(selectedContext.name) === selectedContext,
- // Only opt in when there is a certificate to withhold. Without
- // one the result below is discarded, so a reported lost fence
- // would vanish: no throw to reach the retry, and the recovery
- // in the desired-state branch does not run for an event that
- // did not move the revision.
- honorsLostFence: deltaSafetyCertificate !== undefined,
- }
- )
- if (deltaSafetyCertificate && scopedRevocationCompleted) {
- if (this.netPolReconciler.hasCertifiedSafetyInventory()) {
- this.recordNetworkPolicySafetyCertificate(deltaSafetyCertificate)
- } else {
- console.warn(
- `[K8s] Not certifying the scoped delta for context "${context.spec.contextId}": the last authoritative safety pass ended without certifying its namespace-wide inventory`
- )
- }
- }
- } else if (type === 'DELETED') {
- const deleteAllowed = () =>
- this.contextAbsentForDelete(context.name, context.namespace, watchGeneration)
- if (!(await deleteAllowed())) return
- await this.netPolReconciler.reconcileDeleteContext(
- context.spec.contextId,
- deleteAllowed
- )
- }
- } catch (error) {
- console.error(
- `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,
- error
- )
- void this.runInitialNetworkPolicyConvergence()
- return
+ try {
+ if (type === 'ADDED' || type === 'MODIFIED') {
+ await this.netPolReconciler.reconcileContext(context)
+ } else if (type === 'DELETED') {
+ await this.netPolReconciler.reconcileDeleteContext(context.spec.contextId)
}
+ } catch (error) {
+ console.error(
+ `[K8s] NetworkPolicy reconciliation failed for context ${context.name}:`,
+ error
+ )
+ }
- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
- // Re-reconcile any Host that points at this Context so that changes to
- // spec.sharedFileSystems[] propagate into the mcp-host pod template.
- try {
- await this.reconcileHostsReferencingContext(context.name, contextInventoryAuthoritative)
- } catch (error) {
- console.error(
- `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,
- error
- )
+ // Re-reconcile any Host that points at this Context so that changes to
+ // spec.sharedFileSystems[] propagate into the mcp-host pod template.
+ try {
+ for (const host of this.hosts.values()) {
+ if (host.spec.contextRef !== context.name) continue
+ await this.hostReconciler.reconcile(host)
}
- })
+ } catch (error) {
+ console.error(
+ `[K8s] Host re-reconcile after Context "${context.name}" change failed:`,
+ error
+ )
+ }
}
const doneCallback = (err: Error | null) => {
- if (watchEnded) return
- watchEnded = true
- if (this.stopped || watchGeneration !== this.contextWatchGeneration) return
- if (!this.retireContextWatch(watchGeneration)) return
+ if (this.stopped) return
if (err) {
console.error('[K8s] Context watch error:', err)
}
- console.log('[K8s] Context watch ended; recovering authoritative inventory...')
- this.scheduleContextCacheRecovery()
+ console.log('[K8s] Context watch ended, restarting...')
+ setTimeout(() => this.startContextWatch(), err ? 5000 : 1000)
}
- const request = await this.watch.watch(path, { resourceVersion }, watchCallback, doneCallback)
- if (this.stopped || watchGeneration !== this.contextWatchGeneration || watchEnded) {
- request.abort()
- return watchGeneration
- }
- this.ctxWatchRequest = request
- return watchGeneration
+ this.ctxWatchRequest = await this.watch.watch(path, {}, watchCallback, doneCallback)
}
/**
@@ -3970,39 +2722,25 @@ export class McpServerWatcher implements McpServerProvider {
*/
async stop(): Promise<void> {
this.stopped = true
- this.externalEgressCoordinator.stop()
- this.retireMcpServerWatch()
- this.retireContextWatch()
this.ccCacheSynced = false
this.hostCacheSynced = false
this.hostWatchGeneration += 1
this.ccWatchGeneration += 1
- this.hostFleetScheduler.stop()
+ this.resolveHostFleetShutdown()
+ this.hostFleetReconcilePending?.resolve()
+ this.hostFleetReconcilePending = null
if (this.ccCacheRecoveryTimer) {
clearTimeout(this.ccCacheRecoveryTimer)
this.ccCacheRecoveryTimer = null
}
+ if (this.ccFleetRetryTimer) {
+ clearTimeout(this.ccFleetRetryTimer)
+ this.ccFleetRetryTimer = null
+ }
if (this.hostCacheRecoveryTimer) {
clearTimeout(this.hostCacheRecoveryTimer)
this.hostCacheRecoveryTimer = null
}
- if (this.sfsWatchRestartTimer) {
- clearTimeout(this.sfsWatchRestartTimer)
- this.sfsWatchRestartTimer = null
- }
- if (this.gfsWatchRestartTimer) {
- clearTimeout(this.gfsWatchRestartTimer)
- this.gfsWatchRestartTimer = null
- }
- this.hostCacheRecoveryIntent = null
- if (this.mcpServerCacheRecoveryTimer) {
- clearTimeout(this.mcpServerCacheRecoveryTimer)
- this.mcpServerCacheRecoveryTimer = null
- }
- if (this.contextCacheRecoveryTimer) {
- clearTimeout(this.contextCacheRecoveryTimer)
- this.contextCacheRecoveryTimer = null
- }
this.clearAllHostWatchRetries()
if (this.resyncTimer) {
clearInterval(this.resyncTimer)
@@ -4016,11 +2754,16 @@ export class McpServerWatcher implements McpServerProvider {
clearInterval(this.gfsResyncTimer)
this.gfsResyncTimer = null
}
- for (const timer of this.initialConvergenceRetryTimers.values()) {
+ if (this.externalEgressResyncTimer) {
+ clearTimeout(this.externalEgressResyncTimer)
+ this.externalEgressResyncTimer = null
+ }
+ for (const timer of this.externalEgressRetryTimers.values()) {
clearTimeout(timer)
}
- this.initialConvergenceRetryTimers.clear()
- this.initialConvergenceRetryAttempts.clear()
+ this.externalEgressRetryTimers.clear()
+ this.externalEgressRetryAttempts.clear()
+ this.externalEgressInFlight.clear()
if (this.mcpWatchRequest) {
console.log('[K8s] Stopping McpServer watch')
this.mcpWatchRequest.abort()
diff --git a/host-context-controller/src/main.ts b/host-context-controller/src/main.ts
index 565c537d3..0e1502aa5 100644
--- a/host-context-controller/src/main.ts
+++ b/host-context-controller/src/main.ts
@@ -18,7 +18,6 @@ import {
createMcpServerProvider,
getKubeConfig,
} from './k8sClient'
-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
import { ContextMapperServer } from './server'
import { StatelessLifecycleTracker } from './statelessLifecycleTracker'
import {
@@ -114,16 +113,6 @@ async function main(): Promise<void> {
const hasDesktopFn = hostReconciler
? (hostRef: string) => hostReconciler.hasDesktop(hostRef)
: undefined
- // Dev (no watcher) gets an explicit always-authoritative gate: no inventory
- // to certify there, and the server's fail-closed default would otherwise pin
- // /ready at 503 forever (R3-B1). See resolveProviderAuthoritativeFn.
- const providerAuthoritativeFn = resolveProviderAuthoritativeFn(watcher)
- // Desktop status gates on Host inventory authority alone (not full readiness).
- // Dev (no watcher) mirrors the provider gate above: there is no Host inventory
- // to certify, so authority is unconditional (permissive) — otherwise the
- // server's fail-closed default would pin /api/v1/desktop/* at 503 forever in
- // dev (R2-M1). See resolveHostAuthoritativeFn.
- const hostAuthoritativeFn = resolveHostAuthoritativeFn(watcher)
// Stateless heartbeat consumption — mcp-host pods authenticate their
// heartbeats toward control-api's /mcp-host facade (control-api is the
@@ -156,23 +145,16 @@ async function main(): Promise<void> {
`target=${config.controlApiBaseUrl})`
)
}
- server = new ContextMapperServer(
- provider,
- config.port,
- hostReconciler,
- hasDesktopFn,
- providerAuthoritativeFn,
- hostAuthoritativeFn
- )
+ server = new ContextMapperServer(provider, config.port, hostReconciler, hasDesktopFn)
await server.start()
// One-shot legacy sweep: delete the static `clerum-channel-reader`
// Deployment if it still exists in the channels namespace. MUST run
- // BEFORE provider.start(): provider startup establishes the Host inventory
- // watch and schedules its initial fleet convergence, which creates per-Host
- // channel-reader Deployments in the background. If the static is still alive
- // when that work begins, both pods compete on the same bot's getUpdates and
- // Telegram 409s one of them. Sweeping first guarantees zero overlap.
+ // BEFORE provider.start(): provider.start() invokes fullReconcile on
+ // the initial Host list, which creates per-Host channel-reader Deployments
+ // that immediately try to long-poll Telegram. If the static is still
+ // alive at that moment, both pods compete on the same bot's getUpdates
+ // and Telegram 409s one of them. Sweeping first guarantees zero overlap.
// Idempotent; 404 (already gone) is the steady state. See issue #273
// for the empirical reproduction.
if (!config.devMode && hostReconciler) {
@@ -217,11 +199,12 @@ async function main(): Promise<void> {
// reconcileChannelReaderRevision is self-contained (does not throw), so
// the .catch here is defensive — it catches any unexpected runtime error
// in the SecretInformer's own callback path.
- if (!config.devMode && watcher) {
+ if (!config.devMode && hostReconciler) {
const kc = getKubeConfig()
if (kc) {
+ const reconciler = hostReconciler
channelSecretInformer = new SecretInformer(kc, config.channelsNamespace, evt => {
- watcher.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {
+ reconciler.reconcileChannelReaderRevision(evt.name, evt.namespace).catch(err => {
console.error('[Main] reconcileChannelReaderRevision failed:', err)
})
})
diff --git a/host-context-controller/src/mcpServerSafety.test.ts b/host-context-controller/src/mcpServerSafety.test.ts
deleted file mode 100644
index 19f75caf3..000000000
--- a/host-context-controller/src/mcpServerSafety.test.ts
+++ /dev/null
@@ -1,267 +0,0 @@
-import { describe, expect, it, vi } from 'vitest'
-import {
- confirmAuthoritativeMcpServerAbsence,
- isMcpServerStatusOnlyUpdate,
- sameMcpServerDesiredRevision,
-} from './mcpServerSafety'
-import type { McpServerCRD } from './types'
-
-function makeServer(overrides: Partial<McpServerCRD> = {}): McpServerCRD {
- return {
- name: 'web-search',
- namespace: 'mcp-server',
- uid: 'web-search-uid',
- generation: 7,
- annotations: { 'clerum.io/recipe': 'research' },
- labels: { 'clerum.io/workload': 'web-search' },
- spec: {
- contextRef: 'default',
- image: 'clerum/web-search:test',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- ...overrides,
- }
-}
-
-describe('sameMcpServerDesiredRevision', () => {
- it('matches equal desired state while ignoring status and one-sided identity metadata', () => {
- const expected = makeServer({
- status: { conditions: [{ type: 'Ready', status: 'False' }] },
- })
- const current = makeServer({
- uid: undefined,
- generation: undefined,
- status: { conditions: [{ type: 'Ready', status: 'True' }] },
- })
-
- expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)
- expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)
- })
-
- it('rejects a different resource key', () => {
- const expected = makeServer()
-
- expect(sameMcpServerDesiredRevision(expected, makeServer({ name: 'doc-generator' }))).toBe(
- false
- )
- expect(sameMcpServerDesiredRevision(expected, makeServer({ namespace: 'other' }))).toBe(false)
- })
-
- it('rejects differing UID or generation only when both sides provide the field', () => {
- const expected = makeServer()
-
- expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: 'replacement-uid' }))).toBe(
- false
- )
- expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: 8 }))).toBe(false)
- expect(sameMcpServerDesiredRevision(expected, makeServer({ uid: undefined }))).toBe(true)
- expect(sameMcpServerDesiredRevision(expected, makeServer({ generation: undefined }))).toBe(true)
- })
-
- it('compares desired object fields by semantic content, not Kubernetes map key order', () => {
- const expected = makeServer({
- annotations: { first: '1', second: '2' },
- labels: { first: '1', second: '2' },
- })
-
- expect(
- sameMcpServerDesiredRevision(
- expected,
- makeServer({
- spec: {
- ...expected.spec,
- image: 'clerum/web-search:next',
- },
- labels: expected.labels,
- })
- )
- ).toBe(false)
- expect(
- sameMcpServerDesiredRevision(
- expected,
- makeServer({
- annotations: { second: '2', first: '1' },
- labels: expected.labels,
- })
- )
- ).toBe(true)
- expect(
- sameMcpServerDesiredRevision(
- expected,
- makeServer({
- annotations: { 'clerum.io/recipe': 'other' },
- labels: expected.labels,
- })
- )
- ).toBe(false)
- expect(
- sameMcpServerDesiredRevision(
- expected,
- makeServer({
- labels: { second: '2', first: '1' },
- annotations: expected.annotations,
- })
- )
- ).toBe(true)
- })
-
- it('ignores the controller-owned network-ready handshake annotation', () => {
- const expected = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })
- const current = makeServer({
- annotations: {
- 'clerum.io/pre-deploy': 'true',
- 'clerum.io/network-ready': 'true',
- },
- })
-
- expect(sameMcpServerDesiredRevision(expected, current)).toBe(true)
- expect(sameMcpServerDesiredRevision(current, expected)).toBe(true)
- })
-})
-
-describe('isMcpServerStatusOnlyUpdate', () => {
- it('accepts only an exact identity and generation with unchanged desired state', () => {
- const previous = makeServer({
- status: { conditions: [{ type: 'Ready', status: 'False' }] },
- })
- const current = makeServer({
- status: { conditions: [{ type: 'Ready', status: 'True' }] },
- })
-
- expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)
- })
-
- it('does not suppress events when UID or generation equality is only one-sided', () => {
- const previous = makeServer()
-
- expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ uid: undefined }))).toBe(false)
- expect(isMcpServerStatusOnlyUpdate(previous, makeServer({ generation: undefined }))).toBe(false)
- })
-
- it('does not suppress a desired-state change', () => {
- const previous = makeServer()
- const current = makeServer({
- labels: { 'clerum.io/workload': 'replacement' },
- })
-
- expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(false)
- })
-
- it('treats the controller-owned network-ready handshake as a status-only update', () => {
- const previous = makeServer({ annotations: { 'clerum.io/pre-deploy': 'true' } })
- const current = makeServer({
- annotations: {
- 'clerum.io/pre-deploy': 'true',
- 'clerum.io/network-ready': 'true',
- },
- })
-
- expect(isMcpServerStatusOnlyUpdate(previous, current)).toBe(true)
- })
-})
-
-describe('confirmAuthoritativeMcpServerAbsence', () => {
- it('stops before the API read when inventory is not authoritative', async () => {
- const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)
- const readCurrent = vi.fn()
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => false,
- resolveCurrent,
- readCurrent,
- })
- ).resolves.toBe(false)
- expect(resolveCurrent).not.toHaveBeenCalled()
- expect(readCurrent).not.toHaveBeenCalled()
- })
-
- it('stops before the API read when the cache already contains the current server', async () => {
- const readCurrent = vi.fn()
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => true,
- resolveCurrent: () => makeServer(),
- readCurrent,
- })
- ).resolves.toBe(false)
- expect(readCurrent).not.toHaveBeenCalled()
- })
-
- it('rejects absence when the API read returns a current object', async () => {
- const readCurrent = vi.fn().mockResolvedValue({
- metadata: { name: 'web-search', namespace: 'mcp-server' },
- })
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => true,
- resolveCurrent: () => undefined,
- readCurrent,
- })
- ).resolves.toBe(false)
- expect(readCurrent).toHaveBeenCalledOnce()
- })
-
- it('propagates a non-404 API read failure', async () => {
- const apiError = Object.assign(new Error('unavailable'), { code: 503 })
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => true,
- resolveCurrent: () => undefined,
- readCurrent: vi.fn().mockRejectedValue(apiError),
- })
- ).rejects.toBe(apiError)
- })
-
- it('rejects absence when inventory authority is lost after a 404', async () => {
- const inventoryAuthoritative = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false)
- const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative,
- resolveCurrent,
- readCurrent: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
- })
- ).resolves.toBe(false)
- expect(resolveCurrent).toHaveBeenCalledOnce()
- })
-
- it('rejects absence when the server is recreated in cache after a 404', async () => {
- const resolveCurrent = vi
- .fn<() => McpServerCRD | undefined>()
- .mockReturnValueOnce(undefined)
- .mockReturnValueOnce(makeServer())
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => true,
- resolveCurrent,
- readCurrent: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
- })
- ).resolves.toBe(false)
- expect(resolveCurrent).toHaveBeenCalledTimes(2)
- })
-
- it('confirms absence after a 404 when authority and cache absence remain stable', async () => {
- const inventoryAuthoritative = vi.fn(() => true)
- const resolveCurrent = vi.fn((): McpServerCRD | undefined => undefined)
-
- await expect(
- confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative,
- resolveCurrent,
- readCurrent: vi.fn().mockRejectedValue({ response: { statusCode: 404 } }),
- })
- ).resolves.toBe(true)
- expect(inventoryAuthoritative).toHaveBeenCalledTimes(2)
- expect(resolveCurrent).toHaveBeenCalledTimes(2)
- })
-})
diff --git a/host-context-controller/src/mcpServerSafety.ts b/host-context-controller/src/mcpServerSafety.ts
deleted file mode 100644
index 7f8d8438d..000000000
--- a/host-context-controller/src/mcpServerSafety.ts
+++ /dev/null
@@ -1,91 +0,0 @@
-import type { McpServerCRD } from './types'
-import { getErrorCode } from './utils'
-
-const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'
-
-/**
- * This acknowledgement is written by HCC after it has applied a server's
- * NetworkPolicies so WRC can admit its workload. It is not user intent and it
- * cannot change any NetworkPolicy decision. Treating that write as desired
- * state makes HCC invalidate its own authoritative revocation pass under a
- * large startup fleet, coupling /ready to additive runtime convergence again.
- */
-function desiredAnnotations(annotations: Record<string, string> | undefined): Record<string, string> {
- if (!annotations?.[NETWORK_READY_ANNOTATION]) return annotations ?? {}
- const { [NETWORK_READY_ANNOTATION]: _networkReady, ...remaining } = annotations
- return remaining
-}
-
-/**
- * Kubernetes represents objects such as labels and annotations as maps, whose
- * wire key order is not meaningful or stable across LIST and WATCH payloads.
- * Preserve array order (it is semantic) while sorting object keys so a
- * status-only event cannot masquerade as a desired-state revision.
- */
-function stableJson(value: unknown): string {
- if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`
- if (value && typeof value === 'object') {
- const record = value as Record<string, unknown>
- return `{${Object.keys(record)
- .sort()
- .map(key => `${JSON.stringify(key)}:${stableJson(record[key])}`)
- .join(',')}}`
- }
- return JSON.stringify(value) ?? 'undefined'
-}
-
-export function sameMcpServerDesiredRevision(
- expected: McpServerCRD,
- current: McpServerCRD
-): boolean {
- if (expected.name !== current.name || expected.namespace !== current.namespace) return false
- if (expected.uid && current.uid && expected.uid !== current.uid) return false
- if (
- expected.generation !== undefined &&
- current.generation !== undefined &&
- expected.generation !== current.generation
- ) {
- return false
- }
-
- return (
- stableJson(expected.spec) === stableJson(current.spec) &&
- stableJson(desiredAnnotations(expected.annotations)) ===
- stableJson(desiredAnnotations(current.annotations)) &&
- stableJson(expected.labels ?? {}) === stableJson(current.labels ?? {})
- )
-}
-
-export function isMcpServerStatusOnlyUpdate(
- previous: McpServerCRD,
- current: McpServerCRD
-): boolean {
- return (
- previous.uid === current.uid &&
- previous.generation === current.generation &&
- sameMcpServerDesiredRevision(previous, current)
- )
-}
-
-export interface AuthoritativeMcpServerAbsenceCheck {
- inventoryAuthoritative: () => boolean
- resolveCurrent: () => McpServerCRD | undefined
- readCurrent: () => Promise<unknown>
-}
-
-export async function confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative,
- resolveCurrent,
- readCurrent,
-}: AuthoritativeMcpServerAbsenceCheck): Promise<boolean> {
- if (!inventoryAuthoritative() || resolveCurrent()) return false
-
- try {
- await readCurrent()
- return false
- } catch (error: unknown) {
- if (getErrorCode(error) !== 404) throw error
- }
-
- return inventoryAuthoritative() && !resolveCurrent()
-}
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index 59465c3b3..75b8cc5f0 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -79,30 +79,6 @@ export const networkPoliciesTotal = gauge({
labelNames: ['layer'] as const,
})
-export const initialConvergenceRetriesTotal = counter({
- name: 'clerum_hcc_initial_convergence_retries_total',
- help: 'Initial background convergence retries scheduled after a failed fleet pass.',
- labelNames: ['lane'] as const,
-})
-
-export const initialConvergenceLastSuccessTimestampSeconds = gauge({
- name: 'clerum_hcc_initial_convergence_last_success_timestamp_seconds',
- help: 'Unix timestamp of the last successful initial background convergence pass.',
- labelNames: ['lane'] as const,
-})
-
-export const networkPolicySafetyPassDurationSeconds = histogram({
- name: 'clerum_hcc_networkpolicy_safety_pass_duration_seconds',
- help: 'Seconds until an authoritative NetworkPolicy safety pass has revoked stale allows.',
- labelNames: ['outcome'] as const,
-})
-
-export const networkPolicySafetyPassPoliciesTotal = counter({
- name: 'clerum_hcc_networkpolicy_safety_pass_policies_total',
- help: 'NetworkPolicies listed and revoked by authoritative HCC safety passes.',
- labelNames: ['operation'] as const,
-})
-
export const contextReconciliationsTotal = counter({
name: 'clerum_hcc_context_reconciliations_total',
help: 'Total Context CRD reconciliations',
@@ -225,13 +201,3 @@ export const hostDeleteCleanupTotal = counter({
help: 'Host deletion cleanup by outcome (queued/confirmed/completed/retried/superseded).',
labelNames: ['outcome'] as const,
})
-
-// External-egress DNS retry saturation (#205 audit R3-M4 / R2-L1). A binding
-// whose DNS never resolves is retried forever at the capped (maximum) backoff;
-// without this signal the still-denied binding converges nowhere yet stays
-// invisible. Tracks the COUNT of servers pinned at the cap — never a per-server
-// label — so cardinality stays flat regardless of fleet size.
-export const externalEgressRetriesAtCap = gauge({
- name: 'clerum_hcc_external_egress_retries_at_cap',
- help: 'McpServers whose external-egress DNS retry is pinned at the capped (maximum) backoff, i.e. repeatedly failing to converge.',
-})
diff --git a/host-context-controller/src/networkPolicyReconciler.test.ts b/host-context-controller/src/networkPolicyReconciler.test.ts
index 047ccd88e..8f3d3f3e1 100644
--- a/host-context-controller/src/networkPolicyReconciler.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.test.ts
@@ -4,37 +4,9 @@ import type { RecordWithTtl } from 'node:dns'
import * as dns from 'node:dns/promises'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
-import { confirmAuthoritativeMcpServerAbsence } from './mcpServerSafety'
-import {
- networkPolicySafetyPassDurationSeconds,
- networkPolicySafetyPassPoliciesTotal,
-} from './metrics'
-import {
- NetworkPolicyReconciler,
- PUBLIC_EGRESS_EXCEPT_CIDRS,
- sameContextDesiredRevision,
-} from './networkPolicyReconciler'
+import { NetworkPolicyReconciler, PUBLIC_EGRESS_EXCEPT_CIDRS } from './networkPolicyReconciler'
import { ContextCRD, McpServerCRD } from './types'
-function deferred<T = void>(): {
- promise: Promise<T>
- resolve: (value: T | PromiseLike<T>) => void
-} {
- let resolve!: (value: T | PromiseLike<T>) => void
- const promise = new Promise<T>(res => {
- resolve = res
- })
- return { promise, resolve }
-}
-
-vi.mock('./mcpServerSafety', async () => {
- const actual = await vi.importActual<typeof import('./mcpServerSafety')>('./mcpServerSafety')
- return {
- ...actual,
- confirmAuthoritativeMcpServerAbsence: vi.fn(actual.confirmAuthoritativeMcpServerAbsence),
- }
-})
-
// resolve4({ ttl: true }) returns RecordWithTtl[]; `rec` builds that shape and
// `resolve4Mock` sidesteps the string[]/RecordWithTtl[] overload inference.
const rec = (...ips: string[]): RecordWithTtl[] => ips.map(address => ({ address, ttl: 300 }))
@@ -50,7 +22,6 @@ vi.mock('./config', () => ({
port: 8081,
runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],
k8sApiCidrs: [],
- externalEgressDnsResolveTimeoutMs: 5_000,
// #299 sliding-window knobs read by reconcileExternalEgress.
externalEgressOverlapSec: 300,
externalEgressMaxEntries: 128,
@@ -68,23 +39,10 @@ vi.mock('node:dns/promises', () => ({
]),
}))
-vi.mock('./metrics', () => ({
- networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },
- networkPolicySafetyPassPoliciesTotal: { inc: vi.fn() },
-}))
-
function makeMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi.fn().mockImplementation(async ({ name }) => {
- if (
- name === 'allow-desktop-egress-rpc-proxy' ||
- name === 'allow-rpc-proxy-to-managed-mcp-servers'
- ) {
- throw Object.assign(new Error('not found'), { code: 404 })
- }
- return { metadata: { resourceVersion: '1' } }
- }),
+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
@@ -93,17 +51,7 @@ function makeMockNetworkingApi() {
function makeMockCustomApi() {
return {
- getNamespacedCustomObject: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
- listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
- getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({
- metadata: {
- generation: 7,
- resourceVersion: 'status-rv-default',
- },
- status: {},
- }),
+ getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({ status: {} }),
patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
}
}
@@ -122,48 +70,7 @@ function makeReconciler(
})
const cache = serverCache ?? new Map()
const reconciler = new NetworkPolicyReconciler(kc, cache)
- const listedPolicies = new Map<string, k8s.V1NetworkPolicy>()
- const networkingApi = {
- ...mockApi,
- listNamespacedNetworkPolicy: async (args: { namespace: string }) => {
- const response = await mockApi.listNamespacedNetworkPolicy(args)
- const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {
- const name = policy.metadata?.name ?? `unnamed-${index}`
- const materialized = {
- ...policy,
- metadata: {
- ...policy.metadata,
- name,
- namespace: policy.metadata?.namespace ?? args.namespace,
- uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,
- resourceVersion: policy.metadata?.resourceVersion ?? '1',
- },
- }
- listedPolicies.set(`${args.namespace}/${name}`, materialized)
- return materialized
- })
- return { ...response, items }
- },
- readNamespacedNetworkPolicy: async (args: { name: string; namespace: string }) => {
- const response = await mockApi.readNamespacedNetworkPolicy(args)
- const listed = listedPolicies.get(`${args.namespace}/${args.name}`)
- return {
- ...listed,
- ...response,
- metadata: {
- ...listed?.metadata,
- ...response.metadata,
- name: response.metadata?.name ?? listed?.metadata?.name ?? args.name,
- namespace: response.metadata?.namespace ?? listed?.metadata?.namespace ?? args.namespace,
- uid:
- response.metadata?.uid ?? listed?.metadata?.uid ?? `${args.namespace}:${args.name}:uid`,
- resourceVersion:
- response.metadata?.resourceVersion ?? listed?.metadata?.resourceVersion ?? '1',
- },
- }
- },
- }
- ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi
+ ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = mockApi
;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi
return reconciler
}
@@ -174,7 +81,6 @@ describe('NetworkPolicyReconciler', () => {
let reconciler: NetworkPolicyReconciler
beforeEach(() => {
- vi.useRealTimers()
mockApi = makeMockNetworkingApi()
mockCustomApi = makeMockCustomApi()
reconciler = makeReconciler(mockApi, undefined, mockCustomApi)
@@ -182,102 +88,6 @@ describe('NetworkPolicyReconciler', () => {
resolve4Mock.mockResolvedValue(rec('1.2.3.4', '5.6.7.8'))
})
- it('compares Context desired revisions canonically and fences identity changes', () => {
- const expected: ContextCRD = {
- name: 'default',
- namespace: 'mcp-server',
- uid: 'context-uid',
- generation: 7,
- spec: {
- contextId: 'default',
- mcpServers: ['alpha'],
- sharedFileSystems: [{ name: 'workspace', mountPath: '/workspace' }],
- },
- }
- const reordered: ContextCRD = {
- namespace: 'mcp-server',
- name: 'default',
- generation: 7,
- uid: 'context-uid',
- spec: {
- sharedFileSystems: [{ mountPath: '/workspace', name: 'workspace' }],
- mcpServers: ['alpha'],
- contextId: 'default',
- },
- }
-
- expect(sameContextDesiredRevision(expected, reordered)).toBe(true)
- expect(sameContextDesiredRevision(expected, { ...reordered, uid: 'replacement-uid' })).toBe(
- false
- )
- expect(sameContextDesiredRevision(expected, { ...reordered, generation: 8 })).toBe(false)
- })
-
- it('records the authoritative safety-pass inventory, completed revocations, and duration', async () => {
- mockApi.listNamespacedNetworkPolicy
- .mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-context',
- labels: { 'clerum.io/context': 'missing-context' },
- },
- },
- ],
- })
- .mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-host-egress',
- labels: { 'clerum.io/context': 'missing-context' },
- },
- },
- ],
- })
- .mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-rpc-egress',
- labels: { 'clerum.io/context': 'missing-context' },
- },
- },
- ],
- })
- .mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-external-egress',
- labels: { 'clerum.io/mcpserver': 'missing-server' },
- },
- },
- ],
- })
- const complete = vi.fn()
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- onAuthoritativeRevocationComplete: complete,
- })
-
- expect(complete).toHaveBeenCalledOnce()
- expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(
- { operation: 'listed' },
- 4
- )
- expect(networkPolicySafetyPassPoliciesTotal.inc).toHaveBeenCalledWith(
- { operation: 'revoked' },
- 4
- )
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
- { outcome: 'completed' },
- expect.any(Number)
- )
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
- })
-
describe('public egress blocklist parity', () => {
it('keeps the HCC and WRC IPv4 exclusion lists in sync', () => {
const wrcFactory = readFileSync(
@@ -340,50 +150,6 @@ describe('NetworkPolicyReconciler', () => {
// ─── L1: Infrastructure Policies ───────────────────────────────────
describe('L1 — ensureInfrastructurePolicies', () => {
- it.each([
- ['deny-all-mcp-server', 'default-deny'],
- ['allow-dns-egress-mcp-server', 'infrastructure'],
- ['allow-hcc-api-egress-mcp-server', 'infrastructure'],
- ['allow-k8s-api-egress-mcp-server', 'infrastructure'],
- ])(
- 'refuses to adopt a foreign policy that collides with %s',
- async (targetName, policyType) => {
- mockApi.createNamespacedNetworkPolicy.mockImplementation(
- async ({ body }: { body: k8s.V1NetworkPolicy }) => {
- if (body.metadata?.name === targetName) {
- throw Object.assign(new Error('already exists'), { code: 409 })
- }
- return {}
- }
- )
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
- if (
- name === 'allow-desktop-egress-rpc-proxy' ||
- name === 'allow-rpc-proxy-to-managed-mcp-servers'
- ) {
- throw Object.assign(new Error('not found'), { code: 404 })
- }
- return {
- metadata: {
- name,
- namespace,
- uid: 'foreign-baseline-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': policyType,
- },
- },
- }
- })
-
- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({ name: targetName })
- )
- }
- )
-
it('creates DNS egress policies in every runtime namespace', async () => {
await reconciler.ensureDefaultPolicies()
@@ -542,204 +308,34 @@ describe('NetworkPolicyReconciler', () => {
})
it('removes legacy static policies now replaced by generated scoped policies', async () => {
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({
- metadata: {
- name,
- namespace,
- uid: `${name}-uid`,
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- },
- },
- }))
-
await reconciler.ensureDefaultPolicies()
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'rpc-proxy',
name: 'allow-desktop-egress-rpc-proxy',
- body: {
- preconditions: {
- uid: 'allow-desktop-egress-rpc-proxy-uid',
- resourceVersion: '7',
- },
- },
})
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'mcp-server',
name: 'allow-rpc-proxy-to-managed-mcp-servers',
- body: {
- preconditions: {
- uid: 'allow-rpc-proxy-to-managed-mcp-servers-uid',
- resourceVersion: '7',
- },
- },
})
})
- it.each([
- ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],
- ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],
- ])('refuses to delete a foreign legacy policy %s/%s', async (targetNamespace, targetName) => {
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
- if (name !== targetName) {
- throw Object.assign(new Error('not found'), { code: 404 })
- }
- return {
- metadata: {
- name,
- namespace,
- uid: 'foreign-legacy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- },
- },
+ it('fails closed when legacy static policy cleanup fails', async () => {
+ const cleanupError = Object.assign(new Error('rbac denied'), { code: 403 })
+ mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name === 'allow-rpc-proxy-to-managed-mcp-servers') {
+ throw cleanupError
}
+ return {}
})
- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({ namespace: targetNamespace, name: targetName })
- )
+ await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(cleanupError)
})
-
- it.each([
- ['rpc-proxy', 'allow-desktop-egress-rpc-proxy'],
- ['mcp-server', 'allow-rpc-proxy-to-managed-mcp-servers'],
- ])(
- 'propagates an identity conflict deleting legacy policy %s/%s',
- async (targetNamespace, targetName) => {
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
- if (name !== targetName) {
- throw Object.assign(new Error('not found'), { code: 404 })
- }
- return {
- metadata: {
- name,
- namespace,
- uid: 'legacy-policy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- },
- },
- }
- })
- const conflict = Object.assign(new Error('identity changed'), { code: 409 })
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
- if (name === targetName && namespace === targetNamespace) throw conflict
- return {}
- })
-
- await expect(reconciler.ensureDefaultPolicies()).rejects.toBe(conflict)
- }
- )
})
// ─── L1: Allow API ─────────────────────────────────────────────────
describe('L1 — ensureAllowContextMapperApi', () => {
- it('updates the existing HCC-owned allow-api policy without requiring Context owner labels', async () => {
- mockApi.createNamespacedNetworkPolicy.mockImplementation(
- async ({ body }: { body: k8s.V1NetworkPolicy }) => {
- if (body.metadata?.name === 'allow-host-context-controller-api') {
- throw Object.assign(new Error('already exists'), { code: 409 })
- }
- return {}
- }
- )
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => ({
- metadata: {
- name,
- namespace,
- uid: 'existing-default-policy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'allow-api',
- },
- },
- }))
-
- await expect(reconciler.ensureDefaultPolicies()).resolves.toBeUndefined()
- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'allow-host-context-controller-api',
- namespace: 'mcp-server',
- })
- )
- })
-
- it('refuses to adopt a foreign policy that collides with the allow-api policy name', async () => {
- mockApi.createNamespacedNetworkPolicy.mockImplementation(
- async ({ body }: { body: k8s.V1NetworkPolicy }) => {
- if (body.metadata?.name === 'allow-host-context-controller-api') {
- throw Object.assign(new Error('already exists'), { code: 409 })
- }
- return {}
- }
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValue({
- metadata: {
- name: 'allow-host-context-controller-api',
- namespace: 'mcp-server',
- uid: 'foreign-default-policy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'allow-api',
- },
- },
- })
-
- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('revalidates allow-api ownership after a replace conflict', async () => {
- mockApi.createNamespacedNetworkPolicy.mockImplementation(
- async ({ body }: { body: k8s.V1NetworkPolicy }) => {
- if (body.metadata?.name === 'allow-host-context-controller-api') {
- throw Object.assign(new Error('already exists'), { code: 409 })
- }
- return {}
- }
- )
- mockApi.readNamespacedNetworkPolicy
- .mockResolvedValueOnce({
- metadata: {
- name: 'allow-host-context-controller-api',
- namespace: 'mcp-server',
- uid: 'owned-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'allow-api',
- },
- },
- })
- .mockResolvedValueOnce({
- metadata: {
- name: 'allow-host-context-controller-api',
- namespace: 'mcp-server',
- uid: 'foreign-uid',
- resourceVersion: '8',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'allow-api',
- },
- },
- })
- mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('conflict'), { code: 409 })
- )
-
- await expect(reconciler.ensureDefaultPolicies()).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- })
-
it('creates allow-api ingress policy using _from (K8s client convention)', async () => {
await reconciler.ensureDefaultPolicies()
@@ -759,82 +355,6 @@ describe('NetworkPolicyReconciler', () => {
// ─── L2: Context-Based Policies ────────────────────────────────────
describe('L2 — reconcileContext', () => {
- it('refuses to adopt a foreign policy that collides with a live Context policy name', async () => {
- const server: McpServerCRD = {
- name: 'mongo',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'mongo:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]))
- const context: ContextCRD = {
- name: 'dev',
- namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: [server.name] },
- }
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- name: 'ctx-dev-mongo',
- namespace: 'mcp-server',
- uid: 'foreign-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'dev',
- 'clerum.io/mcpserver': 'mongo',
- },
- },
- })
-
- await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('refuses to replace an HCC Context policy owned by a different Context/server pair', async () => {
- const server: McpServerCRD = {
- name: 'b-c',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'a',
- image: 'mongo:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]))
- const context: ContextCRD = {
- name: 'a',
- namespace: 'mcp-server',
- spec: { contextId: 'a', mcpServers: ['b-c'] },
- }
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- name: 'ctx-a-b-c',
- namespace: 'mcp-server',
- uid: 'other-hcc-owner-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'a-b',
- 'clerum.io/mcpserver': 'c',
- },
- },
- })
-
- await expect(rec.reconcileContext(context, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
it('creates context-allow ingress policy per server in mcp-server namespace', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
@@ -854,7 +374,7 @@ describe('NetworkPolicyReconciler', () => {
spec: { contextId: 'dev', mcpServers: ['mongo'] },
}
- await rec.reconcileContext(context, { isCurrent: () => true })
+ await rec.reconcileContext(context)
// First call: ingress in mcp-server namespace
const ingressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
@@ -896,352 +416,106 @@ describe('NetworkPolicyReconciler', () => {
)
})
- it('retains the ingress policy it just replaced when the authoritative server port changes', async () => {
- const oldServer: McpServerCRD = {
+ it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {
+ const cache = new Map<string, McpServerCRD>()
+ cache.set('mongo', {
name: 'mongo',
namespace: 'mcp-server',
spec: {
contextRef: 'dev',
image: 'mongo:latest',
- transport: { type: 'streamableHttp', port: 3000 },
+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
},
- }
- const cache = new Map([[oldServer.name, oldServer]])
+ })
+
const rec = makeReconciler(mockApi, cache)
const context: ContextCRD = {
name: 'dev',
namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: [oldServer.name] },
+ spec: { contextId: 'dev', mcpServers: ['mongo'] },
}
- await rec.reconcileContext(context, { isCurrent: () => true })
- const livePolicies = new Map(
- mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {
- const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body
- return [`${policy.metadata?.namespace}/${policy.metadata?.name}`, policy]
- })
- )
+ await rec.reconcileContext(context)
- vi.clearAllMocks()
- cache.set(oldServer.name, {
- ...oldServer,
- generation: 2,
- spec: {
- ...oldServer.spec,
- transport: { type: 'streamableHttp', port: 4000 },
- },
- })
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace }: { namespace?: string }) => ({
- items: [...livePolicies.values()].filter(
- policy => policy.metadata?.namespace === namespace
- ),
- })
- )
- mockApi.createNamespacedNetworkPolicy.mockImplementation(async ({ namespace, body }) => {
- livePolicies.set(`${namespace}/${body.metadata?.name}`, body)
- return {}
- })
- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, name, body }) => {
- livePolicies.set(`${namespace}/${name}`, body)
- return {}
+ // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)
+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
+ namespace: string
+ body: {
+ metadata: { name: string; namespace: string; labels: Record<string, string> }
+ spec: {
+ podSelector: { matchLabels: Record<string, string> }
+ policyTypes: string[]
+ egress: Array<{
+ to: Array<{
+ namespaceSelector: { matchLabels: Record<string, string> }
+ podSelector: { matchLabels: Record<string, string> }
+ }>
+ ports: Array<{ port: number }>
+ }>
+ }
}
- )
- // The delete mock MUST remove from livePolicies, or the port-4000
- // assertion below survives a spurious deletion and the test passes with
- // the bug reinjected (R3-H7).
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ namespace, name }) => {
- livePolicies.delete(`${namespace}/${name}`)
- return {}
+ }
+ expect(egressCall.namespace).toBe('mcp-host')
+ expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')
+ expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')
+ expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])
+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/context': 'dev',
})
-
- await rec.reconcileContext(context, { isCurrent: () => true })
-
- // Filter the real calls by name+namespace: the production delete always
- // carries `preconditions`, so `not.toHaveBeenCalledWith({name,namespace})`
- // never matched any call and was vacuously true (R3-H7).
- const deletedCtxDevMongo = mockApi.deleteNamespacedNetworkPolicy.mock.calls.some(
- call =>
- (call[0] as { name?: string; namespace?: string })?.name === 'ctx-dev-mongo' &&
- (call[0] as { name?: string; namespace?: string })?.namespace === 'mcp-server'
+ // Egress targets mcp-server namespace + specific server pod
+ const egressRule = egressCall.body.spec.egress[0]
+ expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(
+ 'mcp-server'
)
- expect(deletedCtxDevMongo).toBe(false)
- const ingress = livePolicies.get('mcp-server/ctx-dev-mongo')
- expect(ingress?.spec?.ingress?.[0]?.ports).toEqual([expect.objectContaining({ port: 4000 })])
+ expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')
+ expect(egressRule.ports[0].port).toBe(3000)
})
- it('stops remaining policy mutations after its combined authority lease is retired', async () => {
+ it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
namespace: 'mcp-server',
spec: {
- contextRef: 'dev',
+ contextRef: 'ctx-name',
image: 'mongo:latest',
- transport: { type: 'streamableHttp', port: 3000 },
+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
},
})
- const firstCreateStarted = deferred()
- const releaseFirstCreate = deferred()
- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- firstCreateStarted.resolve(undefined)
- await releaseFirstCreate.promise
- return {}
- })
+
const rec = makeReconciler(mockApi, cache)
- let contextGeneration = 4
- let serverGeneration = 8
- const capturedContextGeneration = contextGeneration
- const capturedServerGeneration = serverGeneration
const context: ContextCRD = {
- name: 'dev',
+ name: 'ctx-name',
namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: ['mongo'] },
+ spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },
}
- const reconcile = rec.reconcileContext(context, {
- isCurrent: () =>
- contextGeneration === capturedContextGeneration &&
- serverGeneration === capturedServerGeneration,
- })
- await firstCreateStarted.promise
-
- contextGeneration = 6
- serverGeneration = 10
- releaseFirstCreate.resolve(undefined)
- await reconcile
+ await rec.reconcileContext(context)
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
+ body: {
+ metadata: { labels: Record<string, string> }
+ spec: { podSelector: { matchLabels: Record<string, string> } }
+ }
+ }
+ expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')
+ expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/context': 'ctx-name',
+ })
})
- it('does not mutate a Context policy from a mixed McpServer cache revision', async () => {
+ it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {
const cache = new Map<string, McpServerCRD>()
cache.set('mongo', {
name: 'mongo',
namespace: 'mcp-server',
- uid: 'mongo-uid',
- generation: 1,
spec: {
contextRef: 'dev',
- image: 'mongo:old',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- })
- const firstCreateStarted = deferred()
- const releaseFirstCreate = deferred()
- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- firstCreateStarted.resolve(undefined)
- await releaseFirstCreate.promise
- return {}
- })
- const rec = makeReconciler(mockApi, cache)
- const context: ContextCRD = {
- name: 'dev',
- namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: ['mongo'] },
- }
-
- const reconcile = rec.reconcileContext(context, { isCurrent: () => true })
- await firstCreateStarted.promise
- cache.set('mongo', {
- name: 'mongo',
- namespace: 'mcp-server',
- uid: 'mongo-uid',
- generation: 2,
- spec: {
- contextRef: 'dev',
- image: 'mongo:new',
- transport: { type: 'streamableHttp', port: 4000 },
- },
- })
- releaseFirstCreate.resolve(undefined)
- await reconcile
-
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('continues a Context policy pass across a status-only McpServer replacement', async () => {
- const cache = new Map<string, McpServerCRD>()
- const selected: McpServerCRD = {
- name: 'mongo',
- namespace: 'mcp-server',
- uid: 'mongo-uid',
- generation: 1,
- spec: {
- contextRef: 'dev',
- image: 'mongo:stable',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- status: { conditions: [{ type: 'Ready', status: 'False' }] },
- }
- cache.set(selected.name, selected)
- const firstCreateStarted = deferred()
- const releaseFirstCreate = deferred()
- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- firstCreateStarted.resolve(undefined)
- await releaseFirstCreate.promise
- return {}
- })
- const rec = makeReconciler(mockApi, cache)
- const context: ContextCRD = {
- name: 'dev',
- namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: ['mongo'] },
- }
-
- const reconcile = rec.reconcileContext(context, { isCurrent: () => true })
- await firstCreateStarted.promise
- cache.set(selected.name, {
- ...selected,
- status: { conditions: [{ type: 'Ready', status: 'True' }] },
- })
- releaseFirstCreate.resolve(undefined)
- await reconcile
-
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)
- })
-
- it('does not adopt an McpServer that appeared after Context selection', async () => {
- const cache = new Map<string, McpServerCRD>()
- const inventoryListed = deferred()
- const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()
- mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- inventoryListed.resolve(undefined)
- return releaseInventory.promise
- })
- const rec = makeReconciler(mockApi, cache)
- const context: ContextCRD = {
- name: 'dev',
- namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: ['late-server'] },
- }
-
- const reconcile = rec.reconcileContext(context, { isCurrent: () => true })
- await inventoryListed.promise
- cache.set('late-server', {
- name: 'late-server',
- namespace: 'mcp-server',
- uid: 'late-server-uid',
- generation: 1,
- spec: {
- contextRef: 'dev',
- image: 'late-server:new',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- })
- releaseInventory.resolve({ items: [] })
- await reconcile
-
- expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('creates L2 egress counterpart in mcp-host namespace (bidirectional)', async () => {
- const cache = new Map<string, McpServerCRD>()
- cache.set('mongo', {
- name: 'mongo',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'mongo:latest',
- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
- },
- })
-
- const rec = makeReconciler(mockApi, cache)
- const context: ContextCRD = {
- name: 'dev',
- namespace: 'mcp-server',
- spec: { contextId: 'dev', mcpServers: ['mongo'] },
- }
-
- await rec.reconcileContext(context, { isCurrent: () => true })
-
- // Three calls: L2 ingress (mcp-server), L2 egress (mcp-host), L2 rpc-proxy egress (rpc-proxy)
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(3)
- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
- namespace: string
- body: {
- metadata: { name: string; namespace: string; labels: Record<string, string> }
- spec: {
- podSelector: { matchLabels: Record<string, string> }
- policyTypes: string[]
- egress: Array<{
- to: Array<{
- namespaceSelector: { matchLabels: Record<string, string> }
- podSelector: { matchLabels: Record<string, string> }
- }>
- ports: Array<{ port: number }>
- }>
- }
- }
- }
- expect(egressCall.namespace).toBe('mcp-host')
- expect(egressCall.body.metadata.name).toBe('ctx-dev-mongo-egress')
- expect(egressCall.body.metadata.labels['clerum.io/policy-type']).toBe('context-allow')
- expect(egressCall.body.spec.policyTypes).toEqual(['Egress'])
- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/context': 'dev',
- })
- // Egress targets mcp-server namespace + specific server pod
- const egressRule = egressCall.body.spec.egress[0]
- expect(egressRule.to[0].namespaceSelector.matchLabels['kubernetes.io/metadata.name']).toBe(
- 'mcp-server'
- )
- expect(egressRule.to[0].podSelector.matchLabels['clerum.io/mcpserver']).toBe('mongo')
- expect(egressRule.ports[0].port).toBe(3000)
- })
-
- it('scopes L2 mcp-host egress by Context CRD name, not namespace-wide selector', async () => {
- const cache = new Map<string, McpServerCRD>()
- cache.set('mongo', {
- name: 'mongo',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'ctx-name',
- image: 'mongo:latest',
- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
- },
- })
-
- const rec = makeReconciler(mockApi, cache)
- const context: ContextCRD = {
- name: 'ctx-name',
- namespace: 'mcp-server',
- spec: { contextId: 'ctx-id', mcpServers: ['mongo'] },
- }
-
- await rec.reconcileContext(context, { isCurrent: () => true })
-
- const egressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[1][0] as {
- body: {
- metadata: { labels: Record<string, string> }
- spec: { podSelector: { matchLabels: Record<string, string> } }
- }
- }
- expect(egressCall.body.metadata.labels['clerum.io/context']).toBe('ctx-id')
- expect(egressCall.body.spec.podSelector.matchLabels).toMatchObject({
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/context': 'ctx-name',
- })
- })
-
- it('creates L2 rpc-proxy egress to one MCP server instead of relying on static namespace egress', async () => {
- const cache = new Map<string, McpServerCRD>()
- cache.set('mongo', {
- name: 'mongo',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'mongo:latest',
- transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
+ image: 'mongo:latest',
+ transport: { type: 'streamableHttp', url: 'http://mongo:3000', port: 3000 },
},
})
@@ -1252,7 +526,7 @@ describe('NetworkPolicyReconciler', () => {
spec: { contextId: 'dev', mcpServers: ['mongo'] },
}
- await rec.reconcileContext(context, { isCurrent: () => true })
+ await rec.reconcileContext(context)
const rpcProxyEgressCall = mockApi.createNamespacedNetworkPolicy.mock.calls[2][0] as {
namespace: string
@@ -1292,118 +566,6 @@ describe('NetworkPolicyReconciler', () => {
// ─── L3: External Egress ───────────────────────────────────────────
describe('L3 — reconcileExternalEgress', () => {
- it('removes a deterministic safety policy created after its desired-state fence is lost', async () => {
- const server: McpServerCRD = {
- name: 'race-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'race:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
- },
- }
- const createStarted = deferred()
- const releaseCreate = deferred()
- let current = true
- mockApi.createNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- createStarted.resolve(undefined)
- await releaseCreate.promise
- return {
- metadata: {
- name: 'ext-egress-race-server-104-18-0-0-16-443',
- namespace: server.namespace,
- uid: 'created-race-policy',
- resourceVersion: '7',
- },
- }
- })
-
- const safety = (reconciler as any).reconcileExternalEgressSafety(
- server,
- [{ metadata: { name: 'ext-egress-race-server-stale' } }],
- () => current
- ) as Promise<boolean>
-
- await createStarted.promise
- current = false
- releaseCreate.resolve(undefined)
-
- expect(await safety).toBe(false)
- const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: created.metadata.name,
- namespace: server.namespace,
- body: { preconditions: { uid: 'created-race-policy', resourceVersion: '7' } },
- })
- )
- })
-
- it('refuses to adopt a foreign policy that collides with an external-egress name', async () => {
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
- },
- }
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- name: 'ext-egress-openai-mcp-104-18-0-0-16-443',
- namespace: 'mcp-server',
- uid: 'foreign-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': server.name,
- },
- },
- })
-
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('refuses to replace an HCC external-egress policy owned by a different server', async () => {
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
- },
- }
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- name: 'ext-egress-openai-mcp-104-18-0-0-16-443',
- namespace: 'mcp-server',
- uid: 'other-hcc-owner-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': 'different-server',
- },
- },
- })
-
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
it('creates egress policy for CIDR binding', async () => {
const server: McpServerCRD = {
name: 'openai-mcp',
@@ -1416,7 +578,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
@@ -1439,7 +601,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const body = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0].body
@@ -1477,69 +639,35 @@ describe('NetworkPolicyReconciler', () => {
)
})
- it('does not mutate policy or status after its McpServer authority lease is retired', async () => {
- const dnsStarted = deferred()
- const releaseDns = deferred<string[]>()
- vi.mocked(dns.resolve4).mockImplementationOnce(async () => {
- dnsStarted.resolve(undefined)
- return releaseDns.promise
- })
- let inventoryGeneration = 7
- const capturedGeneration = inventoryGeneration
+ it('fails closed when ExternalEgressReady status cannot be written', async () => {
+ const statusError = new Error('status update denied')
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
const server: McpServerCRD = {
- name: 'authority-race-mcp',
+ name: 'openai-mcp',
namespace: 'mcp-server',
- uid: 'authority-race-uid',
- generation: 3,
+ generation: 7,
spec: {
contextRef: 'dev',
- image: 'authority-race:test',
- transport: { type: 'streamableHttp', port: 3000 },
+ image: 'openai:latest',
+ transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
egressBindings: [{ dns: 'api.openai.com', port: 443 }],
},
}
- const reconcile = reconciler.reconcileExternalEgress(server, {
- isCurrent: () => inventoryGeneration === capturedGeneration,
- })
- await dnsStarted.promise
-
- // Retire and recover to a different generation. The recovered cache is
- // authoritative again, but never for this stale pass.
- inventoryGeneration = 9
- releaseDns.resolve(['1.2.3.4'])
- await reconcile
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
+ 'status update denied'
+ )
- expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
})
- it('identity-fences the status patch and preserves unrelated conditions', async () => {
- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
- metadata: {
- uid: 'openai-mcp-uid',
- generation: 7,
- resourceVersion: 'status-rv-17',
- },
- status: {
- conditions: [
- {
- type: 'RuntimeReady',
- status: 'True',
- reason: 'DeploymentAvailable',
- message: 'Runtime deployment is available',
- lastTransitionTime: '2026-07-29T00:00:00.000Z',
- observedGeneration: 7,
- },
- ],
- },
- })
+ it('fails closed when ExternalEgressReady status cannot be read', async () => {
+ const statusError = new Error('status read denied')
+ mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
const server: McpServerCRD = {
name: 'openai-mcp',
namespace: 'mcp-server',
- uid: 'openai-mcp-uid',
generation: 7,
spec: {
contextRef: 'dev',
@@ -1549,194 +677,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
-
- const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
- expect(patch.slice(0, 3)).toEqual([
- { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },
- { op: 'test', path: '/metadata/generation', value: 7 },
- { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },
- ])
- expect(patch).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- path: '/status/conditions',
- value: expect.arrayContaining([
- expect.objectContaining({
- type: 'RuntimeReady',
- reason: 'DeploymentAvailable',
- }),
- expect.objectContaining({
- type: 'ExternalEgressReady',
- status: 'True',
- observedGeneration: 7,
- }),
- ]),
- }),
- ])
- )
- })
-
- it('fences every identity field read from status when synthetic input omits them', async () => {
- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
- metadata: {
- uid: 'openai-mcp-uid',
- generation: 7,
- resourceVersion: 'status-rv-17',
- },
- status: {},
- })
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
-
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
-
- const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
- expect(patch.slice(0, 3)).toEqual([
- { op: 'test', path: '/metadata/uid', value: 'openai-mcp-uid' },
- { op: 'test', path: '/metadata/generation', value: 7 },
- { op: 'test', path: '/metadata/resourceVersion', value: 'status-rv-17' },
- ])
- })
-
- it.each([
- {
- name: 'UID',
- currentMetadata: {
- uid: 'replacement-uid',
- generation: 7,
- resourceVersion: 'status-rv-18',
- },
- },
- {
- name: 'generation',
- currentMetadata: {
- uid: 'openai-mcp-uid',
- generation: 8,
- resourceVersion: 'status-rv-18',
- },
- },
- ])(
- 'rejects a stale $name before patching external egress status',
- async ({ currentMetadata }) => {
- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
- metadata: currentMetadata,
- status: {},
- })
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- uid: 'openai-mcp-uid',
- generation: 7,
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
-
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/stale McpServer/i)
-
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- }
- )
-
- it('fails closed when a same-generation status writer wins the resourceVersion race', async () => {
- const conflict = Object.assign(new Error('JSON patch test failed'), { code: 409 })
- mockCustomApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
- metadata: {
- uid: 'openai-mcp-uid',
- generation: 7,
- resourceVersion: 'status-rv-17',
- },
- status: {
- conditions: [
- {
- type: 'RuntimeReady',
- status: 'True',
- reason: 'DeploymentAvailable',
- lastTransitionTime: '2026-07-29T00:00:00.000Z',
- },
- ],
- },
- })
- mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(conflict)
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- uid: 'openai-mcp-uid',
- generation: 7,
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
-
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toBe(conflict)
-
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
- expect.objectContaining({
- body: expect.arrayContaining([
- {
- op: 'test',
- path: '/metadata/resourceVersion',
- value: 'status-rv-17',
- },
- ]),
- })
- )
- })
-
- it('fails closed when ExternalEgressReady status cannot be written', async () => {
- const statusError = new Error('status update denied')
- mockCustomApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- generation: 7,
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
-
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
- 'status update denied'
- )
-
- expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
- })
-
- it('fails closed when ExternalEgressReady status cannot be read', async () => {
- const statusError = new Error('status read denied')
- mockCustomApi.getNamespacedCustomObjectStatus.mockRejectedValueOnce(statusError)
- const server: McpServerCRD = {
- name: 'openai-mcp',
- namespace: 'mcp-server',
- generation: 7,
- spec: {
- contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', url: 'http://openai:3000', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
-
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('status read denied')
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('status read denied')
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
@@ -1754,7 +695,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(dns.resolve4).not.toHaveBeenCalled()
expect(mockApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
@@ -1796,7 +737,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/public-web external egress bindings/
)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@ -1814,7 +755,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/public-web external egress bindings/
)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
@@ -1852,7 +793,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
@@ -1898,7 +839,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
@@ -1923,7 +864,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const patch = mockCustomApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body
const resolvedPatch = patch.find(
@@ -1948,7 +889,7 @@ describe('NetworkPolicyReconciler', () => {
}
// First reconcile bootstraps and writes the external-egress policy.
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -1975,7 +916,7 @@ describe('NetworkPolicyReconciler', () => {
// Second resync: same DNS answers → same IP set → must NOT rewrite the
// policy (a timestamp-only refresh is a no-op). This is the churn the audit
// caught: before the fix, RESOLVED_AT + renewed expiry forced a write here.
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -1991,7 +932,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ dns: 'api.example.com', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const written = (
mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
).body
@@ -2023,7 +964,7 @@ describe('NetworkPolicyReconciler', () => {
// Same DNS → set unchanged (changed=false) but window aging → renewalDue=true
// → the policy MUST be re-persisted (deleting `|| egressRenewalDue` breaks this,
// reintroducing the M1 overlap-grace loss).
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const wrote =
mockApi.createNamespacedNetworkPolicy.mock.calls.length +
mockApi.replaceNamespacedNetworkPolicy.mock.calls.length
@@ -2041,7 +982,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ dns: 'api.example.com', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const written = (
mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
).body
@@ -2094,7 +1035,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
// The reconcile must rewrite the policy WITHOUT the private IP.
const call =
mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??
@@ -2119,7 +1060,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ dns: 'api.example.com', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
// The min observed TTL (ms) must reflect the 15s answer — deleting the
// Math.min update leaves it Infinity and the resync degrades to the fixed
// interval (the #299 under-sampling failure mode).
@@ -2139,7 +1080,7 @@ describe('NetworkPolicyReconciler', () => {
}
// Round 1: DNS serves A only.
resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.3', ttl: 15 }])
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const written = (
mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
).body
@@ -2152,7 +1093,7 @@ describe('NetworkPolicyReconciler', () => {
// Round 2: DNS rotates to B. The written policy must be the UNION {A, B} —
// this only holds if previousAnnotations rehydration is actually wired in.
resolve4Mock.mockResolvedValueOnce([{ address: '140.82.112.4', ttl: 15 }])
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const call =
mockApi.replaceNamespacedNetworkPolicy.mock.calls[0]?.[0] ??
mockApi.createNamespacedNetworkPolicy.mock.calls[0]?.[0]
@@ -2174,7 +1115,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -2185,7 +1126,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -2205,7 +1146,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -2222,7 +1163,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -2238,7 +1179,7 @@ describe('NetworkPolicyReconciler', () => {
egressBindings: [{ cidr: '93.184.216.0/24', port: 443 }],
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
const created = mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
body: k8s.V1NetworkPolicy
}
@@ -2257,7 +1198,7 @@ describe('NetworkPolicyReconciler', () => {
mockApi.createNamespacedNetworkPolicy.mockClear()
mockApi.replaceNamespacedNetworkPolicy.mockClear()
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
// The reconciler must rewrite the policy back to the declared CIDR.
const wrote =
mockApi.createNamespacedNetworkPolicy.mock.calls.length +
@@ -2278,7 +1219,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/External egress reconciliation failed/
)
}
@@ -2299,7 +1240,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/resolved disallowed address/
)
@@ -2334,7 +1275,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/private, internal, metadata/
)
@@ -2355,7 +1296,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/resolved disallowed address/
)
@@ -2381,15 +1322,13 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-openai-mcp-old-example-com-443',
- namespace: 'mcp-server',
- })
- )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: 'ext-egress-openai-mcp-old-example-com-443',
+ namespace: 'mcp-server',
+ })
})
it('fails closed when an orphaned external egress policy cannot be deleted', async () => {
@@ -2412,7 +1351,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow('delete denied')
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow('delete denied')
})
it('deletes stale policies and fails when DNS resolution for that binding fails', async () => {
@@ -2435,21 +1374,17 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(/dns timeout/)
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(/dns timeout/)
expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-openai-mcp-api.openai.com-443',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-openai-mcp-old-example-com-443',
- namespace: 'mcp-server',
- })
- )
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: 'ext-egress-openai-mcp-api.openai.com-443',
+ namespace: 'mcp-server',
+ })
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: 'ext-egress-openai-mcp-old-example-com-443',
+ namespace: 'mcp-server',
+ })
expect(mockCustomApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
@@ -2480,36 +1415,13 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await expect(reconciler.reconcileExternalEgress(server, { isCurrent: () => true })).rejects.toThrow(
+ await expect(reconciler.reconcileExternalEgress(server)).rejects.toThrow(
/must declare dns or cidr/
)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
- it('bounds DNS resolution so a stuck resolver cannot block reconciliation indefinitely', async () => {
- vi.useFakeTimers()
- vi.mocked(dns.resolve4).mockImplementationOnce(() => new Promise(() => undefined))
- const server: McpServerCRD = {
- name: 'stuck-dns-mcp',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'stuck-dns:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
- },
- }
-
- const reconcile = reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
- const rejection = expect(reconcile).rejects.toThrow(/DNS resolution timed out after 5000ms/)
- await vi.advanceTimersByTimeAsync(5_000)
- await rejection
-
- expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
it('skips if no egressBindings', async () => {
const server: McpServerCRD = {
name: 'basic-mcp',
@@ -2521,7 +1433,7 @@ describe('NetworkPolicyReconciler', () => {
},
}
- await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ await reconciler.reconcileExternalEgress(server)
expect(mockApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -2542,14 +1454,6 @@ describe('NetworkPolicyReconciler', () => {
await expect((reconciler as any).listAllContextPolicies()).rejects.toThrow('api list denied')
})
- it('propagates all-context mcp-host egress policy list failures', async () => {
- mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))
-
- await expect((reconciler as any).listAllContextEgressPolicies()).rejects.toThrow(
- 'api list denied'
- )
- })
-
it('propagates all external-egress policy list failures', async () => {
mockApi.listNamespacedNetworkPolicy.mockRejectedValueOnce(new Error('api list denied'))
@@ -2559,2646 +1463,98 @@ describe('NetworkPolicyReconciler', () => {
})
})
- describe('delete-context TOCTOU re-check', () => {
- it('stops mid-inventory when the context is recreated between two deletes', async () => {
- // deleteAllowed is a live GET, so its answer can change while the loop
- // runs. Hoisting the check out of the loop would satisfy every existing
- // test and still delete the recreated context's own policies.
- mockApi.listNamespacedNetworkPolicy.mockResolvedValueOnce({
- items: [
- { metadata: { name: 'ctx-allow-first' } },
- { metadata: { name: 'ctx-allow-second' } },
- ],
- })
- const deleteAllowed = vi
- .fn<() => Promise<boolean>>()
- .mockResolvedValueOnce(true)
- .mockResolvedValue(false)
-
- await reconciler.reconcileDeleteContext('dev', deleteAllowed)
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'ctx-allow-first' })
- )
- expect(deleteAllowed).toHaveBeenCalledTimes(2)
- })
-
- it('re-checks before every mcp-host egress delete', async () => {
- mockApi.listNamespacedNetworkPolicy
- .mockResolvedValueOnce({ items: [] }) // context-allow
- .mockResolvedValueOnce({
- items: [
- { metadata: { name: 'host-egress-first' } },
- { metadata: { name: 'host-egress-second' } },
- ],
- })
- const deleteAllowed = vi
- .fn<() => Promise<boolean>>()
- .mockResolvedValueOnce(true)
- .mockResolvedValue(false)
-
- await reconciler.reconcileDeleteContext('dev', deleteAllowed)
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'host-egress-first' })
- )
- })
-
- it('re-checks before every rpc-proxy egress delete', async () => {
- mockApi.listNamespacedNetworkPolicy
- .mockResolvedValueOnce({ items: [] }) // context-allow
- .mockResolvedValueOnce({ items: [] }) // mcp-host egress
- .mockResolvedValueOnce({
- items: [
- { metadata: { name: 'rpc-egress-first' } },
- { metadata: { name: 'rpc-egress-second' } },
- ],
- })
- const deleteAllowed = vi
- .fn<() => Promise<boolean>>()
- .mockResolvedValueOnce(true)
- .mockResolvedValue(false)
-
- await reconciler.reconcileDeleteContext('dev', deleteAllowed)
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'rpc-egress-first' })
- )
- })
- })
-
- describe('L3 — DNS-derived retention by binding identity (B3)', () => {
- // Live DNS policy built through the reconciler's OWN builder so its identity
- // is exact-by-construction; each test drifts exactly one dimension.
- function makeDnsFixture() {
+ describe('startup fullReconcile', () => {
+ it('reconciles external egress policies for existing servers on startup', async () => {
const server: McpServerCRD = {
- name: 'openai-mcp',
+ name: 'airtable-mcp',
namespace: 'mcp-server',
spec: {
contextRef: 'dev',
- image: 'openai:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.openai.com', port: 443 }],
+ image: 'airtable:latest',
+ transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.airtable.com', port: 443 }],
},
}
- const binding = server.spec.egressBindings![0]
- const name = (reconciler as any).externalEgressPolicyName(server.name, binding) as string
- const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [
- '1.2.3.4/32',
- ]) as k8s.V1NetworkPolicy
- live.metadata!.uid = 'live-dns-uid'
- live.metadata!.resourceVersion = '11'
- return { server, binding, name, live }
- }
-
- async function runSafety(server: McpServerCRD, live: k8s.V1NetworkPolicy): Promise<boolean> {
- return (reconciler as any).reconcileExternalEgressSafety(
- server,
- [live],
- () => true
- ) as Promise<boolean>
- }
-
- function expectRevoked(name: string) {
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name,
- namespace: 'mcp-server',
- body: { preconditions: { uid: 'live-dns-uid', resourceVersion: '11' } },
- })
- )
- }
- it('(a) retains an unchanged DNS-derived allow — no revocation', async () => {
- const { server, live } = makeDnsFixture()
- await expect(runSafety(server, live)).resolves.toBe(true)
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
+ await reconciler.fullReconcile([], [server])
- it('(b) revokes on spec-level port drift under the same policy name', async () => {
- const { server, name, live } = makeDnsFixture()
- live.spec!.egress![0].ports = [{ port: 8443, protocol: 'TCP' }]
- await expect(runSafety(server, live)).resolves.toBe(true)
- expectRevoked(name)
+ const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
+ (call: unknown[]) => {
+ const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }
+ return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'
+ }
+ )
+ expect(externalPolicies).toHaveLength(1)
+ const body = (
+ externalPolicies[0][0] as {
+ body: {
+ metadata: { name: string }
+ spec: { egress: Array<{ ports: Array<{ port: number }> }> }
+ }
+ }
+ ).body
+ expect(body.metadata.name).toContain('ext-egress-airtable-mcp')
+ expect(body.spec.egress[0].ports[0].port).toBe(443)
})
- it('(c) revokes on protocol drift TCP→UDP', async () => {
- const { server, name, live } = makeDnsFixture()
- live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]
- await expect(runSafety(server, live)).resolves.toBe(true)
- expectRevoked(name)
- })
+ it('deletes orphaned external egress policies for servers no longer present', async () => {
+ mockApi.listNamespacedNetworkPolicy.mockImplementation(
+ async ({ labelSelector }: { labelSelector?: string }) => {
+ if (labelSelector?.includes('context-allow')) return { items: [] }
+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-old-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'old-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
+ )
- it('(d) revokes a foreign-owned policy even when the spec is identical', async () => {
- const { server, name, live } = makeDnsFixture()
- live.metadata!.labels!['clerum.io/managed-by'] = 'foreign-controller'
- await expect(runSafety(server, live)).resolves.toBe(true)
- expectRevoked(name)
- })
+ await reconciler.fullReconcile([], [], { serverInventoryComplete: true })
- it('(f) revokes a degenerate zero-cidr "allow" (deny in disguise)', async () => {
- const { server, name, live } = makeDnsFixture()
- live.spec!.egress = []
- await expect(runSafety(server, live)).resolves.toBe(true)
- expectRevoked(name)
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
+ name: 'ext-egress-old-server-api-example-com-443',
+ namespace: 'mcp-server',
+ })
})
- it('(e) queues recreation AT the revocation even when the pass then aborts and throws', async () => {
- const { server, binding, name } = makeDnsFixture()
- // Identity-drifted (protocol UDP) → this DNS allow is revoked (deleted).
- const live = (reconciler as any).buildExactHostEgressPolicy(server, name, binding, [
- '1.2.3.4/32',
- ]) as k8s.V1NetworkPolicy
- live.spec!.egress![0].ports = [{ port: 443, protocol: 'UDP' }]
- live.metadata!.uid = 'live-dns-uid'
- live.metadata!.resourceVersion = '11'
-
+ it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {
mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: !labelSelector && namespace === 'mcp-server' ? [live] : [],
- })
+ async ({ labelSelector }: { labelSelector?: string }) => {
+ if (labelSelector?.includes('context-allow')) return { items: [] }
+ if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
+ if (labelSelector?.includes('external-egress')) {
+ return {
+ items: [
+ {
+ metadata: {
+ name: 'ext-egress-old-server-api-example-com-443',
+ labels: { 'clerum.io/mcpserver': 'old-server' },
+ },
+ },
+ ],
+ }
+ }
+ return { items: [] }
+ }
)
- let resolved: McpServerCRD | undefined = server
- const spy = vi.fn()
- // Flip authority false AT the deletion, so the pass aborts right after the
- // revocation hook fires — reproducing the abort/throw window B3 must survive.
- mockApi.deleteNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- resolved = undefined
- return {}
+ await reconciler.fullReconcile([], [], { serverInventoryComplete: false })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
+ name: 'ext-egress-old-server-api-example-com-443',
+ namespace: 'mcp-server',
})
-
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
-
- await expect(
- rec.fullReconcile([], [server], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- resolveCurrentServer: () => resolved,
- onExternalEgressRevoked: spy,
- })
- ).rejects.toThrow(/inventory changed during authoritative revocation/)
-
- expect(spy).toHaveBeenCalledTimes(1)
- expect(spy).toHaveBeenCalledWith(
- expect.objectContaining({ name: server.name, namespace: server.namespace })
- )
- })
- })
-
- describe('startup fullReconcile', () => {
- it('discovers and revokes orphaned policies with one missing ownership marker in every safety lane', async () => {
- const partiallyLabelledByNamespace: Record<string, k8s.V1NetworkPolicy[]> = {
- 'mcp-server': [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/context': 'orphan-context',
- 'clerum.io/mcpserver': 'old-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- },
- {
- metadata: {
- name: 'ext-egress-orphan-server-old-example-com-443',
- labels: {
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': 'orphan-server',
- 'clerum.io/egress-class': 'exact-host',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- },
- ],
- 'mcp-host': [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server-egress',
- labels: {
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'orphan-context',
- 'clerum.io/mcpserver': 'old-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- },
- ],
- 'rpc-proxy': [
- {
- metadata: {
- name: 'rpc-egress-orphan-context-old-server',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/context': 'orphan-context',
- 'clerum.io/mcpserver': 'old-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- },
- ],
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: labelSelector ? [] : (partiallyLabelledByNamespace[namespace ?? ''] ?? []),
- })
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-orphan-context-old-server',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-orphan-context-old-server-egress',
- namespace: 'mcp-host',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'rpc-egress-orphan-context-old-server',
- namespace: 'rpc-proxy',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-orphan-server-old-example-com-443',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(4)
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it('repairs one-marker live widened policies in every lane before certification', async () => {
- const server: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'live-context',
- image: 'live:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '8.8.8.8/32', port: 443 }],
- },
- }
- const context: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [server.name] },
- }
- const cache = new Map([[server.name, server]])
- const rec = makeReconciler(mockApi, cache, mockCustomApi)
-
- await rec.reconcileContext(context, { isCurrent: () => true })
- await rec.reconcileExternalEgress(server, { isCurrent: () => true })
- const generated = mockApi.createNamespacedNetworkPolicy.mock.calls.map(
- call => (call[0] as { body: k8s.V1NetworkPolicy }).body
- )
- const widened = generated.map((policy, index) => ({
- ...policy,
- metadata: {
- ...policy.metadata,
- labels: Object.fromEntries(
- Object.entries(policy.metadata?.labels ?? {}).filter(
- ([label]) =>
- label !== (index % 2 === 0 ? 'clerum.io/managed-by' : 'clerum.io/policy-type')
- )
- ),
- },
- spec: {
- ...policy.spec,
- policyTypes: [...(policy.spec?.policyTypes ?? []), 'Ingress'],
- },
- }))
-
- vi.clearAllMocks()
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: labelSelector
- ? []
- : widened.filter(policy => policy.metadata?.namespace === namespace),
- })
- )
- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- const ordering: string[] = []
- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
- ordering.push(`replace:${name}`)
- return {}
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))
-
- await rec.fullReconcile([context], [server], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- resolveCurrentContext: () => context,
- resolveCurrentServer: () => server,
- onAuthoritativeRevocationComplete,
- })
-
- const certificationIndex = ordering.indexOf('certify')
- expect(certificationIndex).toBeGreaterThanOrEqual(4)
- for (const expectedName of [
- 'ctx-live-context-live-server',
- 'ctx-live-context-live-server-egress',
- 'rpc-egress-live-context-live-server',
- 'ext-egress-live-server-8-8-8-8-32-443',
- ]) {
- expect(ordering.indexOf(`replace:${expectedName}`)).toBeGreaterThanOrEqual(0)
- expect(ordering.indexOf(`replace:${expectedName}`)).toBeLessThan(certificationIndex)
- }
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it.each([
- [
- 'context ingress',
- 'mcp-server',
- {
- metadata: { name: 'ctx-reserved-context-server', labels: {} },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- },
- ],
- [
- 'host egress',
- 'mcp-host',
- {
- metadata: { name: 'ctx-reserved-context-server-egress', labels: {} },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- },
- ],
- [
- 'rpc egress',
- 'rpc-proxy',
- {
- metadata: { name: 'rpc-egress-reserved-context-server', labels: {} },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- },
- ],
- [
- 'external egress',
- 'mcp-server',
- {
- metadata: { name: 'ext-egress-reserved-server-example-443', labels: {} },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- },
- ],
- ])(
- 'fails closed without takeover for a both-markers-missing reserved %s name',
- async (_lane, namespace, candidate) => {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({
- namespace: listedNamespace,
- labelSelector,
- }: {
- namespace?: string
- labelSelector?: string
- }) => ({
- items: !labelSelector && listedNamespace === namespace ? [candidate] : [],
- })
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- }
- )
-
- it('fails closed for an explicit foreign ownership conflict on an HCC-shaped policy', async () => {
- const candidate: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'ext-egress-live-server-api-example-com-443',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': 'live-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],
- })
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('treats a non-reserved external policy missing its manager marker as ambiguous', async () => {
- const candidate: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'custom-egress-policy',
- labels: {
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': 'live-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: !labelSelector && namespace === 'mcp-server' ? [candidate] : [],
- })
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
-
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('lets the broad snapshot shadow a selected-owned object that became unrelated at the same UID', async () => {
- const selected: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'legacy-custom-policy',
- uid: 'stable-policy-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'orphan-context',
- 'clerum.io/mcpserver': 'old-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- }
- const broadRelabelled: k8s.V1NetworkPolicy = {
- ...selected,
- metadata: {
- ...selected.metadata,
- resourceVersion: '8',
- labels: { 'tenant.example/managed-by': 'tenant-controller' },
- },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace !== 'mcp-server') return { items: [] }
- if (labelSelector?.includes('policy-type=context-allow')) return { items: [selected] }
- if (!labelSelector) return { items: [broadRelabelled] }
- return { items: [] }
- }
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it('leaves unrelated namespace policies untouched and still certifies', async () => {
- const unrelated: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'tenant-owned-policy',
- labels: {
- 'app.kubernetes.io/managed-by': 'tenant-operator',
- 'tenant.example/purpose': 'custom-egress',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Egress'] },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: !labelSelector && namespace === 'mcp-server' ? [unrelated] : [],
- })
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('fails closed when the broad safety inventory LIST fails', async () => {
- mockApi.listNamespacedNetworkPolicy
- .mockResolvedValueOnce({ items: [] })
- .mockResolvedValueOnce({ items: [] })
- .mockResolvedValueOnce({ items: [] })
- .mockResolvedValueOnce({ items: [] })
- .mockRejectedValueOnce(new Error('broad inventory denied'))
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow('broad inventory denied')
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('withholds certification and mutation when Context authority is lost during the broad LIST', async () => {
- let contextAuthoritative = true
- const candidate: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'ctx-orphan-context-old-server',
- labels: {
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'orphan-context',
- 'clerum.io/mcpserver': 'old-server',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (!labelSelector && namespace === 'mcp-server') {
- contextAuthoritative = false
- return { items: [candidate] }
- }
- return { items: [] }
- }
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => contextAuthoritative,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('does not reconcile a queued Context when its current contextId changed', async () => {
- const queuedContext: ContextCRD = {
- name: 'mutable-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'captured-context', mcpServers: [] },
- }
- const currentContext: ContextCRD = {
- ...queuedContext,
- spec: { contextId: 'replacement-context', mcpServers: [] },
- }
- const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)
- const effectKeys: string[] = []
-
- await reconciler.fullReconcile([queuedContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => false,
- resolveCurrentContext: () => currentContext,
- runContextEffect: async (contextId, work) => {
- effectKeys.push(contextId)
- await work()
- },
- })
-
- expect(effectKeys).toEqual(['captured-context', 'captured-context'])
- expect(reconcileContext).not.toHaveBeenCalled()
- })
-
- it('retires a full-pass Context lease when the selected object is replaced', async () => {
- const selected: ContextCRD = {
- name: 'full-pass-context',
- namespace: 'mcp-server',
- spec: { contextId: 'full-pass-context', description: 'old', mcpServers: [] },
- }
- const replacement: ContextCRD = {
- ...selected,
- spec: { ...selected.spec, description: 'new' },
- }
- let current = selected
- let selectedLease: (() => boolean) | undefined
- const mutationStarted = deferred()
- const releaseMutation = deferred()
- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async (_context, options) => {
- selectedLease = options?.isCurrent
- mutationStarted.resolve(undefined)
- await releaseMutation.promise
- return true
- })
-
- const fullPass = reconciler.fullReconcile([selected], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- resolveCurrentContext: () => current,
- })
- await mutationStarted.promise
- expect(selectedLease?.()).toBe(true)
-
- current = replacement
- expect(selectedLease?.()).toBe(false)
- releaseMutation.resolve(undefined)
- await fullPass
- })
-
- it('deletes orphaned Context policies in all three namespaces through the canonical contextId effect key', async () => {
- const effectKeys: string[] = []
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server-egress',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'rpc-egress-orphan-context-old-server',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- runContextEffect: async (contextId, work) => {
- effectKeys.push(contextId)
- await work()
- },
- })
-
- expect(effectKeys).toEqual(['orphan-context', 'orphan-context', 'orphan-context'])
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-orphan-context-old-server',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-orphan-context-old-server-egress',
- namespace: 'mcp-host',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'rpc-egress-orphan-context-old-server',
- namespace: 'rpc-proxy',
- })
- )
- })
-
- it('routes desired and orphan Context policy lanes through canonical contextId effect keys', async () => {
- const desiredContext: ContextCRD = {
- name: 'desired-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'desired-context', mcpServers: [] },
- }
- const effectKeys: string[] = []
- vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'rpc-egress-orphan-context-old-server',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([desiredContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- runContextEffect: async (contextId, work) => {
- effectKeys.push(contextId)
- await work()
- },
- })
-
- expect(effectKeys).toEqual([
- 'orphan-context',
- 'orphan-context',
- 'desired-context',
- 'desired-context',
- ])
- })
-
- it('revokes every orphan allow lane before a slow additive Context effect', async () => {
- const desiredContext: ContextCRD = {
- name: 'desired-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'desired-context', mcpServers: [] },
- }
- const additiveStarted = deferred()
- const releaseAdditive = deferred()
- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {
- additiveStarted.resolve(undefined)
- await releaseAdditive.promise
- return true
- })
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-orphan-context-old-server-egress',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'rpc-egress-orphan-context-old-server',
- labels: { 'clerum.io/context': 'orphan-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-orphan-server-old.example',
- labels: { 'clerum.io/mcpserver': 'orphan-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- const fullPass = reconciler.fullReconcile([desiredContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- })
-
- await additiveStarted.promise
- try {
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-orphan-context-old-server',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-orphan-context-old-server-egress',
- namespace: 'mcp-host',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'rpc-egress-orphan-context-old-server',
- namespace: 'rpc-proxy',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-orphan-server-old.example',
- namespace: 'mcp-server',
- })
- )
- } finally {
- releaseAdditive.resolve(undefined)
- await fullPass
- }
- })
-
- it('revokes every stale allow lane for a live Context before signaling authoritative revocation', async () => {
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [] },
- }
- const additiveStarted = deferred()
- const releaseAdditive = deferred()
- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {
- additiveStarted.resolve(undefined)
- await releaseAdditive.promise
- return true
- })
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-live-context-removed-server',
- labels: { 'clerum.io/context': 'live-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-live-context-removed-server-egress',
- labels: { 'clerum.io/context': 'live-context' },
- },
- },
- ],
- }
- }
- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'rpc-egress-live-context-removed-server',
- labels: { 'clerum.io/context': 'live-context' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- const fullPass = reconciler.fullReconcile([liveContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- await additiveStarted.promise
- try {
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-live-context-removed-server',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ctx-live-context-removed-server-egress',
- namespace: 'mcp-host',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'rpc-egress-live-context-removed-server',
- namespace: 'rpc-proxy',
- })
- )
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- } finally {
- releaseAdditive.resolve(undefined)
- await fullPass
- }
- })
-
- it('revokes stale external egress for a live McpServer before readiness certification', async () => {
- const liveServer: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [],
- },
- }
- const ordering: string[] = []
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-live-server-old.example-443',
- labels: { 'clerum.io/mcpserver': 'live-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('delete')
- return {}
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- ordering.push('certify')
- })
-
- await reconciler.fullReconcile([], [liveServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-live-server-old.example-443',
- namespace: 'mcp-server',
- })
- )
- expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it('does not let one post-certification Context failure starve later additive owners', async () => {
- const contexts: ContextCRD[] = ['aaa-poisoned', 'bbb-healthy'].map(contextId => ({
- name: `${contextId}-resource`,
- namespace: 'mcp-server',
- spec: { contextId, mcpServers: [] },
- }))
- const ordering: string[] = []
- const additiveFailure = new Error('poisoned additive Context')
- vi.spyOn(reconciler, 'reconcileContext').mockImplementation(async context => {
- ordering.push(`context:${context.spec.contextId}`)
- if (context.spec.contextId === 'aaa-poisoned') throw additiveFailure
- return true
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- ordering.push('certify')
- })
- const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)
-
- const error = await reconciler
- .fullReconcile(contexts, [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- .then(
- () => undefined,
- failure => failure
- )
-
- expect(error).toBeInstanceOf(AggregateError)
- expect((error as AggregateError).errors).toEqual([additiveFailure])
- expect(ordering).toEqual(['certify', 'context:aaa-poisoned', 'context:bbb-healthy'])
- expect(errorLog).toHaveBeenCalledWith(
- '[NetPol] Additive Context reconciliation failed for "aaa-poisoned":',
- additiveFailure
- )
- })
-
- it('replaces same-name Context allow policies in every lane before certification', async () => {
- const oldServer: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- transport: { type: 'streamableHttp', port: 4000 },
- },
- }
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [oldServer.name] },
- }
- const serverCache = new Map([[oldServer.name, oldServer]])
- const rec = makeReconciler(mockApi, serverCache, mockCustomApi)
-
- await rec.reconcileContext(liveContext, { isCurrent: () => true })
- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(
- call => (call[0] as { body: k8s.V1NetworkPolicy }).body
- )
- expect(oldPolicies).toHaveLength(3)
-
- vi.clearAllMocks()
- serverCache.set(currentServer.name, currentServer)
- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) return { items: [] }
- return {
- items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),
- }
- }
- )
- vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)
- vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)
- const ordering: string[] = []
- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace }: { namespace?: string }) => {
- ordering.push(`replace:${namespace}`)
- return {}
- }
- )
- const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))
-
- await rec.fullReconcile([liveContext], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(ordering).toEqual([
- 'replace:mcp-server',
- 'replace:mcp-host',
- 'replace:rpc-proxy',
- 'certify',
- ])
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it.each([
- ['context-ingress', 'mcp-server'],
- ['context-host-egress', 'mcp-host'],
- ['context-rpc-egress', 'rpc-proxy'],
- ])(
- 'refuses a same-name %s safety replacement owned by another server',
- async (_lane, targetNamespace) => {
- const oldServer: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- transport: { type: 'streamableHttp', port: 4000 },
- },
- }
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [oldServer.name] },
- }
- const serverCache = new Map([[oldServer.name, oldServer]])
- const rec = makeReconciler(mockApi, serverCache, mockCustomApi)
-
- await rec.reconcileContext(liveContext, { isCurrent: () => true })
- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {
- const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body
- if (policy.metadata?.namespace !== targetNamespace) return policy
- return {
- ...policy,
- metadata: {
- ...policy.metadata,
- labels: {
- ...policy.metadata?.labels,
- 'clerum.io/mcpserver': 'different-server',
- },
- },
- }
- })
-
- vi.clearAllMocks()
- serverCache.set(currentServer.name, currentServer)
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) return { items: [] }
- return {
- items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),
- }
- }
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- rec.fullReconcile([liveContext], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/ownership/i)
-
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({ namespace: targetNamespace })
- )
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- }
- )
-
- it('fails the safety pass without certifying when a same-name Context replacement fails', async () => {
- const oldServer: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- transport: { type: 'streamableHttp', port: 4000 },
- },
- }
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [oldServer.name] },
- }
- const serverCache = new Map([[oldServer.name, oldServer]])
- const rec = makeReconciler(mockApi, serverCache, mockCustomApi)
-
- await rec.reconcileContext(liveContext, { isCurrent: () => true })
- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(
- call => (call[0] as { body: k8s.V1NetworkPolicy }).body
- )
-
- vi.clearAllMocks()
- serverCache.set(currentServer.name, currentServer)
- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) return { items: [] }
- return {
- items: oldPolicies.filter(policy => policy.metadata?.namespace === namespace),
- }
- }
- )
- const replacementFailure = new Error('replace unavailable')
- mockApi.replaceNamespacedNetworkPolicy.mockRejectedValueOnce(replacementFailure)
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- rec.fullReconcile([liveContext], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toBe(replacementFailure)
-
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
- { outcome: 'failed' },
- expect.any(Number)
- )
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
- })
-
- it('retains authoritative same-name policies when only API object key order differs', async () => {
- const server: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [server.name] },
- }
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
- const reverseObjectKeys = (value: unknown): unknown => {
- if (Array.isArray(value)) return value.map(reverseObjectKeys)
- if (value === null || typeof value !== 'object') return value
- return Object.fromEntries(
- Object.entries(value as Record<string, unknown>)
- .reverse()
- .map(([key, entry]) => [key, reverseObjectKeys(entry)])
- )
- }
-
- await rec.reconcileContext(liveContext, { isCurrent: () => true })
- const existingPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.map(call => {
- const policy = (call[0] as { body: k8s.V1NetworkPolicy }).body
- return { ...policy, spec: reverseObjectKeys(policy.spec) as k8s.V1NetworkPolicySpec }
- })
- vi.clearAllMocks()
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) return { items: [] }
- return {
- items: existingPolicies.filter(policy => policy.metadata?.namespace === namespace),
- }
- }
- )
- vi.spyOn(rec, 'reconcileContext').mockResolvedValue(true)
- vi.spyOn(rec, 'reconcileExternalEgress').mockResolvedValue(undefined)
-
- await rec.fullReconcile([liveContext], [server], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('revokes same-name DNS egress before certifying and leaves refresh to its coordinator', async () => {
- const oldServer: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
-
- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
- const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
- expect(oldPolicy).toBeDefined()
-
- vi.clearAllMocks()
- const ordering: string[] = []
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: [oldPolicy!] } : { items: [] }
- )
- mockApi.createNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('replace')
- return {}
- })
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('delete')
- return {}
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- ordering.push('certify')
- })
-
- await rec.fullReconcile([], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(ordering.slice(0, 2)).toEqual(['delete', 'certify'])
- expect(ordering).not.toContain('replace')
- expect(dns.resolve4).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it('revokes a divergent same-name DNS policy when its safe replacement cannot resolve', async () => {
- const oldServer: McpServerCRD = {
- name: 'live-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'live:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'UDP' }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
-
- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
- const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
- expect(oldPolicy).toBeDefined()
-
- vi.clearAllMocks()
- const ordering: string[] = []
- vi.mocked(dns.resolve4).mockImplementation(async () => {
- ordering.push('dns')
- throw new Error('dns unavailable')
- })
- let currentPolicies = [oldPolicy!]
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
- )
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('delete')
- currentPolicies = []
- return {}
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- ordering.push('certify')
- })
-
- await expect(
- rec.fullReconcile([], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).resolves.toBeUndefined()
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: oldPolicy!.metadata!.name!,
- namespace: oldServer.namespace,
- })
- )
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- expect(dns.resolve4).not.toHaveBeenCalled()
- expect(ordering).toEqual(['delete', 'certify'])
- })
-
- it('revokes a renamed DNS binding before certification and performs no DNS lookup on the safety path', async () => {
- const oldServer: McpServerCRD = {
- name: 'renamed-dns-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'dns:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'old.example.com', port: 443 }],
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- egressBindings: [{ dns: 'new.example.com', port: 443 }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
- const oldPolicy = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
-
- vi.clearAllMocks()
- const ordering: string[] = []
- let currentPolicies = [oldPolicy]
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
- )
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
- ordering.push('delete')
- currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)
- return {}
- })
- vi.mocked(dns.resolve4).mockImplementation(async () => {
- ordering.push('dns')
- return ['1.2.3.4']
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- expect(dns.resolve4).not.toHaveBeenCalled()
- ordering.push('certify')
- })
-
- await rec.fullReconcile([], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(ordering).toEqual(['delete', 'certify'])
- expect(dns.resolve4).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: oldPolicy.metadata!.name!,
- namespace: oldServer.namespace,
- body: {
- preconditions: {
- uid: `${oldServer.namespace}:${oldPolicy.metadata!.name!}:uid`,
- resourceVersion: '1',
- },
- },
- })
- })
-
- it('replaces deterministic drift and revokes renamed DNS before certification without resolving DNS', async () => {
- const oldServer: McpServerCRD = {
- name: 'mixed-drift-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'mixed:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [
- { cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' },
- { dns: 'old.example.com', port: 443, protocol: 'TCP' },
- ],
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- egressBindings: [
- { cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' },
- { dns: 'new.example.com', port: 443, protocol: 'TCP' },
- ],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
- const oldPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .filter(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
-
- vi.clearAllMocks()
- const ordering: string[] = []
- let currentPolicies = oldPolicies
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
- )
- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('replace')
- return {}
- })
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
- ordering.push('delete')
- currentPolicies = currentPolicies.filter(policy => policy.metadata?.name !== name)
- return {}
- })
- vi.mocked(dns.resolve4).mockImplementation(async () => {
- ordering.push('dns')
- return ['1.2.3.4']
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- expect(dns.resolve4).not.toHaveBeenCalled()
- ordering.push('certify')
- })
-
- await rec.fullReconcile([], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(ordering).toEqual(['replace', 'delete', 'certify'])
- expect(dns.resolve4).not.toHaveBeenCalled()
- })
-
- it('aborts certification when an identity-drifted safety delete reports a recreate conflict', async () => {
- const server: McpServerCRD = {
- name: 'recreated-dns-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'dns:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.example.com', port: 443 }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
- await rec.reconcileExternalEgress(server, { isCurrent: () => true })
- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
- // B3: identity-drift the live policy (protocol TCP→UDP) so the safety pass
- // still revokes it. A same-intent (identity-stable) DNS allow is now
- // RETAINED (see the retention pins in the B3 describe block); the revoke /
- // abort machinery this test guards fires only for a drifted policy.
- existing.spec!.egress = existing.spec!.egress!.map(rule => ({
- ...rule,
- ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),
- }))
-
- vi.clearAllMocks()
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }
- )
- const conflict = Object.assign(new Error('delete precondition conflict'), { code: 409 })
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(conflict)
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- rec.fullReconcile([], [server], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toBe(conflict)
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: existing.metadata!.name!,
- namespace: server.namespace,
- body: {
- preconditions: {
- uid: `${server.namespace}:${existing.metadata!.name!}:uid`,
- resourceVersion: '1',
- },
- },
- })
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it.each([
- [
- 'foreign relabel',
- {
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': 'fresh-read-server',
- },
- },
- ],
- ['UID drift', { uid: 'replacement-uid' }],
- ['resourceVersion drift', { resourceVersion: '2' }],
- ])(
- 'aborts a deterministic safety replace after fresh-read %s',
- async (_case, metadataDrift) => {
- const oldServer: McpServerCRD = {
- name: 'fresh-read-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'fresh:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
-
- vi.clearAllMocks()
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValue({
- metadata: {
- ...metadataDrift,
- },
- })
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- rec.fullReconcile([], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/ownership/)
-
- expect(mockApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: existing.metadata!.name!,
- namespace: oldServer.namespace,
- })
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- }
- )
-
- it('uses the fresh identity to complete a deterministic safety replacement before certification', async () => {
- const oldServer: McpServerCRD = {
- name: 'fresh-success-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'fresh:test',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'TCP' }],
- },
- }
- const currentServer: McpServerCRD = {
- ...oldServer,
- spec: {
- ...oldServer.spec,
- egressBindings: [{ cidr: '8.8.8.8/32', port: 443, protocol: 'UDP' }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[oldServer.name, oldServer]]), mockCustomApi)
- await rec.reconcileExternalEgress(oldServer, { isCurrent: () => true })
- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')!
-
- vi.clearAllMocks()
- const ordering: string[] = []
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: [existing] } : { items: [] }
- )
- mockApi.readNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('read')
- return { metadata: { resourceVersion: '1' } }
- })
- mockApi.replaceNamespacedNetworkPolicy.mockImplementation(async () => {
- ordering.push('replace')
- return {}
- })
- const onAuthoritativeRevocationComplete = vi.fn(() => ordering.push('certify'))
-
- await rec.fullReconcile([], [currentServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(ordering.slice(0, 3)).toEqual(['read', 'replace', 'certify'])
- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: existing.metadata!.name!,
- namespace: oldServer.namespace,
- body: expect.objectContaining({
- metadata: expect.objectContaining({
- name: existing.metadata!.name!,
- namespace: oldServer.namespace,
- resourceVersion: '1',
- }),
- spec: expect.objectContaining({
- egress: expect.arrayContaining([
- expect.objectContaining({
- ports: [expect.objectContaining({ protocol: 'UDP' })],
- }),
- ]),
- }),
- }),
- })
- })
-
- it('revokes an identity-drifted DNS policy before readiness and defers its refresh', async () => {
- const server: McpServerCRD = {
- name: 'stable-dns-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'stable:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
- await rec.reconcileExternalEgress(server, { isCurrent: () => true })
- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
- expect(existing).toBeDefined()
- // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a
- // same-intent DNS allow is now retained. The revoke-before-readiness +
- // defer-refresh machinery this test guards still fires on the drifted one.
- existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({
- ...rule,
- ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),
- }))
-
- vi.clearAllMocks()
- let currentPolicies = [existing!]
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: currentPolicies } : { items: [] }
- )
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
- currentPolicies = []
- return {}
- })
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await rec.fullReconcile([], [server], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: existing!.metadata!.name!,
- namespace: server.namespace,
- })
- )
- expect(dns.resolve4).not.toHaveBeenCalled()
- })
-
- it('does not certify safety when an identity-drifted DNS policy cannot be revoked', async () => {
- const server: McpServerCRD = {
- name: 'undeletable-dns-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'stable:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'api.example.com', port: 443, protocol: 'TCP' }],
- },
- }
- const rec = makeReconciler(mockApi, new Map([[server.name, server]]), mockCustomApi)
- await rec.reconcileExternalEgress(server, { isCurrent: () => true })
- const existing = mockApi.createNamespacedNetworkPolicy.mock.calls
- .map(call => (call[0] as { body: k8s.V1NetworkPolicy }).body)
- .find(policy => policy.metadata?.labels?.['clerum.io/policy-type'] === 'external-egress')
- expect(existing).toBeDefined()
- // B3: identity-drift (protocol TCP→UDP) so this policy is revoked; a
- // same-intent DNS allow is now retained. The no-certify-when-revoke-fails
- // machinery this test guards still fires on the drifted one.
- existing!.spec!.egress = existing!.spec!.egress!.map(rule => ({
- ...rule,
- ports: rule.ports?.map(port => ({ ...port, protocol: 'UDP' })),
- }))
-
- vi.clearAllMocks()
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) =>
- labelSelector?.includes('external-egress') ? { items: [existing!] } : { items: [] }
- )
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(new Error('delete denied'))
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- rec.fullReconcile([], [server], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow('delete denied')
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: existing!.metadata!.name!,
- namespace: server.namespace,
- })
- )
- })
-
- it('does not certify revocation when an earlier Context changes while a later owner is paused', async () => {
- const contexts: ContextCRD[] = ['first', 'later'].map(contextId => ({
- name: `${contextId}-resource`,
- namespace: 'mcp-server',
- spec: { contextId, mcpServers: [] },
- }))
- const currentContexts = new Map(contexts.map(context => [context.name, context]))
- let contextDesiredRevision = 10
- let laterPaused = false
- const laterStarted = deferred()
- const releaseLater = deferred()
- const onAuthoritativeRevocationComplete = vi.fn()
-
- const fullPass = reconciler.fullReconcile(contexts, [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- resolveCurrentContext: name => currentContexts.get(name),
- runContextEffect: async (contextId, work) => {
- if (contextId === 'later' && !laterPaused) {
- laterPaused = true
- laterStarted.resolve(undefined)
- await releaseLater.promise
- }
- await work()
- },
- contextDesiredRevision: () => contextDesiredRevision,
- onAuthoritativeRevocationComplete,
- })
-
- await laterStarted.promise
- currentContexts.set('first-resource', {
- ...contexts[0],
- spec: { ...contexts[0].spec, mcpServers: ['replacement'] },
- })
- contextDesiredRevision += 1
- releaseLater.resolve(undefined)
-
- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('does not certify revocation when a new owner is added after the global inventory LISTs', async () => {
- const anchorContext: ContextCRD = {
- name: 'anchor-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'anchor', mcpServers: [] },
- }
- const currentContexts = new Map([[anchorContext.name, anchorContext]])
- let contextDesiredRevision = 21
- const safetyEffectStarted = deferred()
- const releaseSafetyEffect = deferred()
- let safetyEffectPaused = false
- const onAuthoritativeRevocationComplete = vi.fn()
-
- const fullPass = reconciler.fullReconcile([anchorContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- resolveCurrentContext: name => currentContexts.get(name),
- runContextEffect: async (_contextId, work) => {
- if (!safetyEffectPaused) {
- safetyEffectPaused = true
- safetyEffectStarted.resolve(undefined)
- await releaseSafetyEffect.promise
- }
- await work()
- },
- contextDesiredRevision: () => contextDesiredRevision,
- onAuthoritativeRevocationComplete,
- })
-
- await safetyEffectStarted.promise
- currentContexts.set('late-resource', {
- name: 'late-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'late', mcpServers: [] },
- })
- contextDesiredRevision += 1
- releaseSafetyEffect.resolve(undefined)
-
- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('does not certify revocation when a new McpServer owner is added after inventory LISTs', async () => {
- const anchorServer: McpServerCRD = {
- name: 'anchor-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'anchor:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const currentServers = new Map([[anchorServer.name, anchorServer]])
- let serverDesiredRevision = 31
- const safetyEffectStarted = deferred()
- const releaseSafetyEffect = deferred()
- let safetyEffectPaused = false
- const onAuthoritativeRevocationComplete = vi.fn()
-
- const fullPass = reconciler.fullReconcile([], [anchorServer], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- resolveCurrentServer: name => currentServers.get(name),
- runServerEffect: async (_serverName, work) => {
- if (!safetyEffectPaused) {
- safetyEffectPaused = true
- safetyEffectStarted.resolve(undefined)
- await releaseSafetyEffect.promise
- }
- await work()
- },
- serverDesiredRevision: () => serverDesiredRevision,
- onAuthoritativeRevocationComplete,
- })
-
- await safetyEffectStarted.promise
- currentServers.set('late-server', {
- name: 'late-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'late:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- })
- serverDesiredRevision += 1
- releaseSafetyEffect.resolve(undefined)
-
- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('does not certify revocation when desired inventory changes while default policies are ensured', async () => {
- const context: ContextCRD = {
- name: 'default-window-context',
- namespace: 'mcp-server',
- spec: { contextId: 'default-window', mcpServers: [] },
- }
- const server: McpServerCRD = {
- name: 'default-window-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default-window',
- image: 'default-window:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- let contextDesiredRevision = 41
- let serverDesiredRevision = 51
- const defaultsStarted = deferred()
- const releaseDefaults = deferred()
- const onAuthoritativeRevocationComplete = vi.fn()
- vi.spyOn(reconciler, 'ensureDefaultPolicies').mockImplementation(async () => {
- defaultsStarted.resolve(undefined)
- await releaseDefaults.promise
- })
-
- const fullPass = reconciler.fullReconcile([context], [server], {
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- contextDesiredRevision: () => contextDesiredRevision,
- serverDesiredRevision: () => serverDesiredRevision,
- onAuthoritativeRevocationComplete,
- })
-
- await defaultsStarted.promise
- contextDesiredRevision += 1
- serverDesiredRevision += 1
- releaseDefaults.resolve(undefined)
-
- await expect(fullPass).rejects.toThrow(/desired NetworkPolicy inventory changed/i)
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('does not signal authoritative revocation when a live-owner delete fails', async () => {
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [] },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-live-context-removed-server',
- labels: { 'clerum.io/context': 'live-context' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(
- new Error('policy delete unavailable')
- )
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([liveContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow('policy delete unavailable')
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('does not delete or signal revocation after live Context authority is lost', async () => {
- const liveContext: ContextCRD = {
- name: 'live-context-resource',
- namespace: 'mcp-server',
- spec: { contextId: 'live-context', mcpServers: [] },
- }
- let authoritative = true
- const inventoryListed = deferred()
- const releaseInventory = deferred<{ items: k8s.V1NetworkPolicy[] }>()
- mockApi.listNamespacedNetworkPolicy.mockImplementationOnce(async () => {
- inventoryListed.resolve(undefined)
- return releaseInventory.promise
- })
- const onAuthoritativeRevocationComplete = vi.fn()
-
- const fullPass = reconciler.fullReconcile([liveContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => authoritative,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- await inventoryListed.promise
- authoritative = false
- releaseInventory.resolve({
- items: [
- {
- metadata: {
- name: 'ctx-live-context-removed-server',
- labels: { 'clerum.io/context': 'live-context' },
- },
- },
- ],
- })
-
- await fullPass
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('uses selected lane LISTs plus one broad LIST per bounded namespace before additive convergence', async () => {
- const contexts: ContextCRD[] = ['alpha', 'beta', 'gamma'].map(contextId => ({
- name: `${contextId}-resource`,
- namespace: 'mcp-server',
- spec: { contextId, mcpServers: [] },
- }))
- const servers: McpServerCRD[] = ['one', 'two'].map(name => ({
- name,
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: `${name}:latest`,
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }))
- const additiveStarted = deferred()
- const releaseAdditive = deferred()
- vi.spyOn(reconciler, 'reconcileContext').mockImplementationOnce(async () => {
- additiveStarted.resolve(undefined)
- await releaseAdditive.promise
- return true
- })
- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
-
- const fullPass = reconciler.fullReconcile(contexts, servers, {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- })
-
- await additiveStarted.promise
- try {
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'mcp-server',
- labelSelector: expect.stringContaining('policy-type=context-allow'),
- })
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'mcp-host',
- labelSelector: expect.stringContaining('policy-type=context-allow'),
- })
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'rpc-proxy',
- labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),
- })
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'mcp-server',
- labelSelector: expect.stringContaining('policy-type=external-egress'),
- })
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'mcp-server',
- })
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'mcp-host',
- })
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
- namespace: 'rpc-proxy',
- })
- } finally {
- releaseAdditive.resolve(undefined)
- await fullPass
- }
- })
-
- it('leaves startup external egress creation to its dedicated coordinator', async () => {
- const server: McpServerCRD = {
- name: 'airtable-mcp',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'dev',
- image: 'airtable:latest',
- transport: { type: 'streamableHttp', url: 'http://airtable:3000', port: 3000 },
- egressBindings: [{ dns: 'api.airtable.com', port: 443 }],
- },
- }
-
- await reconciler.fullReconcile([], [server])
-
- const externalPolicies = mockApi.createNamespacedNetworkPolicy.mock.calls.filter(
- (call: unknown[]) => {
- const arg = call[0] as { body: { metadata: { labels: Record<string, string> } } }
- return arg.body.metadata.labels['clerum.io/policy-type'] === 'external-egress'
- }
- )
- expect(externalPolicies).toHaveLength(0)
- expect(dns.resolve4).not.toHaveBeenCalled()
- })
-
- it('deletes orphaned external egress policies for servers no longer present', async () => {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('context-allow')) return { items: [] }
- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-old-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'old-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], { serverInventoryComplete: true })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'ext-egress-old-server-api-example-com-443',
- namespace: 'mcp-server',
- })
- )
- })
-
- it('fails closed by deleting HCC-managed policies with missing owner labels', async () => {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- const managedLabels = { 'clerum.io/managed-by': 'host-context-controller' }
- if (labelSelector?.includes('policy-type=context-allow')) {
- return {
- items: [
- {
- metadata: {
- name:
- namespace === 'mcp-host'
- ? 'malformed-host-egress'
- : 'malformed-context-ingress',
- labels: managedLabels,
- },
- },
- ],
- }
- }
- if (labelSelector?.includes('rpc-proxy-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'malformed-rpc-proxy-egress',
- labels: managedLabels,
- },
- },
- ],
- }
- }
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'malformed-external-egress',
- labels: managedLabels,
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], {
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- serverInventoryComplete: true,
- })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'malformed-context-ingress',
- namespace: 'mcp-server',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'malformed-host-egress',
- namespace: 'mcp-host',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'malformed-rpc-proxy-egress',
- namespace: 'rpc-proxy',
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'malformed-external-egress',
- namespace: 'mcp-server',
- })
- )
- })
-
- it('skips external egress orphan cleanup when startup server discovery is incomplete', async () => {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('context-allow')) return { items: [] }
- if (labelSelector?.includes('rpc-proxy-egress')) return { items: [] }
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-old-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'old-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], { serverInventoryComplete: false })
-
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
- name: 'ext-egress-old-server-api-example-com-443',
- namespace: 'mcp-server',
- })
- })
-
- it('finishes desired Context reconciliation but stops Context and rpc-proxy orphan cleanup when authority is lost after inventory listing', async () => {
- const desiredContext: ContextCRD = {
- name: 'current',
- namespace: 'mcp-server',
- spec: { contextId: 'current', mcpServers: [] },
- }
- const reconcileContext = vi.spyOn(reconciler, 'reconcileContext').mockResolvedValue(true)
- const contextInventoryAuthoritative = vi
- .fn()
- .mockReturnValueOnce(true)
- .mockReturnValueOnce(false)
- .mockReturnValue(true)
-
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('policy-type=context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-deleted-context-old-server',
- labels: { 'clerum.io/context': 'deleted-context' },
- },
- },
- ],
- }
- }
- if (labelSelector?.includes('external-egress')) return { items: [] }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([desiredContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative,
- })
-
- expect(reconcileContext).toHaveBeenCalledWith(desiredContext, {
- isCurrent: expect.any(Function),
- })
- expect(contextInventoryAuthoritative).toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
- name: 'ctx-deleted-context-old-server',
- namespace: 'mcp-server',
- })
- expect(mockApi.listNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
- expect.objectContaining({
- namespace: 'rpc-proxy',
- labelSelector: expect.stringContaining('policy-type=rpc-proxy-egress'),
- })
- )
- })
-
- it('stops external-egress orphan cleanup when authority is lost after inventory listing', async () => {
- const desiredServer: McpServerCRD = {
- name: 'current-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'current',
- image: 'current:latest',
- transport: { type: 'streamableHttp', url: 'http://current:3000', port: 3000 },
- },
- }
- const serverInventoryAuthoritative = vi
- .fn()
- .mockReturnValueOnce(true)
- .mockReturnValueOnce(false)
- .mockReturnValue(true)
-
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-deleted-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'deleted-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [desiredServer], {
- ensureDefaults: false,
- serverInventoryComplete: true,
- serverInventoryAuthoritative,
- })
-
- expect(serverInventoryAuthoritative).toHaveBeenCalled()
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
- name: 'ext-egress-deleted-server-api-example-com-443',
- namespace: 'mcp-server',
- })
- })
-
- it('does not delete any Context policy lane when a same-contextId CRD exists before its watch event arrives', async () => {
- mockCustomApi.listNamespacedCustomObject.mockResolvedValue({
- items: [{ metadata: { name: 'recreated-context' }, spec: { contextId: 'recreated' } }],
- })
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- if (namespace === 'mcp-server' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-recreated-server',
- labels: { 'clerum.io/context': 'recreated' },
- },
- },
- ],
- }
- }
- if (namespace === 'mcp-host' && labelSelector?.includes('context-allow')) {
- return {
- items: [
- {
- metadata: {
- name: 'ctx-recreated-server-egress',
- labels: { 'clerum.io/context': 'recreated' },
- },
- },
- ],
- }
- }
- if (namespace === 'rpc-proxy' && labelSelector?.includes('rpc-proxy-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'rpc-egress-recreated-server',
- labels: { 'clerum.io/context': 'recreated' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- })
-
- expect(mockCustomApi.listNamespacedCustomObject).toHaveBeenCalled()
- expect(mockApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({
- namespace: 'mcp-host',
- labelSelector: expect.stringContaining('policy-type=context-allow'),
- })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('does not delete external-egress policy when a same-name McpServer exists before ADDED arrives', async () => {
- mockCustomApi.getNamespacedCustomObject.mockResolvedValue({
- metadata: { name: 'recreated-server', uid: 'new-uid' },
- })
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-recreated-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'recreated-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- serverInventoryComplete: true,
- serverInventoryAuthoritative: () => true,
- })
-
- expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledWith(
- expect.objectContaining({ plural: 'mcpservers', name: 'recreated-server' })
- )
- expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('delegates startup McpServer orphan safety to the canonical absence helper', async () => {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ labelSelector }: { labelSelector?: string }) => {
- if (labelSelector?.includes('external-egress')) {
- return {
- items: [
- {
- metadata: {
- name: 'ext-egress-orphan-server-api-example-com-443',
- labels: { 'clerum.io/mcpserver': 'orphan-server' },
- },
- },
- ],
- }
- }
- return { items: [] }
- }
- )
-
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- serverInventoryComplete: true,
- serverInventoryAuthoritative: () => true,
- })
-
- expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledTimes(1)
- expect(confirmAuthoritativeMcpServerAbsence).toHaveBeenCalledWith({
- inventoryAuthoritative: expect.any(Function),
- resolveCurrent: expect.any(Function),
- readCurrent: expect.any(Function),
- })
- })
- })
+ })
+ })
describe('L3 — cleanupExternalEgress', () => {
it('deletes all external egress policies for a server', async () => {
@@ -5208,16 +1564,7 @@ describe('NetworkPolicyReconciler', () => {
await reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
- name: 'ext-egress-openai-mcp-api-443',
- namespace: 'mcp-server',
- body: {
- preconditions: {
- uid: 'mcp-server:ext-egress-openai-mcp-api-443:uid',
- resourceVersion: '1',
- },
- },
- })
+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
})
it('fails closed when stale external egress policy deletion fails', async () => {
@@ -5243,467 +1590,5 @@ describe('NetworkPolicyReconciler', () => {
reconciler.cleanupExternalEgress('openai-mcp', 'mcp-server')
).resolves.toBeUndefined()
})
-
- it('does not report a revocation when the policy was already absent', async () => {
- const error = Object.assign(new Error('not found'), { code: 404 })
- const onDeleted = vi.fn()
- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
- items: [{ metadata: { name: 'ext-egress-openai-mcp-old-api-443' } }],
- })
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(error)
-
- await reconciler.cleanupExternalEgress(
- 'openai-mcp',
- 'mcp-server',
- undefined,
- undefined,
- onDeleted
- )
-
- expect(onDeleted).not.toHaveBeenCalled()
- })
- })
-
- describe('scoped delta certification vs. a stuck safety inventory', () => {
- const ambiguousReservedPolicy: k8s.V1NetworkPolicy = {
- metadata: { name: 'ctx-reserved-context-server', labels: {} },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- }
- const deltaContext: ContextCRD = {
- name: 'default',
- namespace: 'mcp-server',
- spec: { contextId: 'default', mcpServers: [] },
- }
-
- function stickSafetyInventory(): void {
- // Only the namespace-wide safety inventory sees the foreign policy. The
- // label-scoped LISTs the delta lane uses stay clean, which is exactly why
- // the delta cannot vouch for the namespace.
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items: !labelSelector && namespace === 'mcp-server' ? [ambiguousReservedPolicy] : [],
- })
- )
- }
-
- it('refuses to certify a scoped delta while the safety pass is stuck on its inventory', async () => {
- stickSafetyInventory()
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
-
- // The delta's own revocation still completes — its label-scoped LISTs are
- // clean. What it may not do is vouch for the namespace-wide inventory,
- // which the readiness lane checks separately.
- expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- })
-
- it('certifies a scoped delta again once a later safety pass completes', async () => {
- stickSafetyInventory()
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: vi.fn(),
- })
- ).rejects.toThrow(/Ambiguous NetworkPolicy ownership/)
-
- mockApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- const onAuthoritativeRevocationComplete = vi.fn()
- await reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
-
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
- })
-
- it('does not certify the safety inventory until an authoritative pass has run', async () => {
- // Fail-closed default: before any authoritative safety pass certifies, the
- // namespace-wide inventory is NOT certified, even though the scoped delta's
- // own label-scoped revocation completes (returns true). A delta cannot
- // vouch for the namespace-wide inventory it never enumerated.
- expect(await reconciler.reconcileContext(deltaContext, { isCurrent: () => true })).toBe(true)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- })
- })
-
- describe('lost delete fence during stale-allow revocation', () => {
- const staleContext: ContextCRD = {
- name: 'default',
- namespace: 'mcp-server',
- spec: { contextId: 'default', mcpServers: [] },
- }
- const stalePolicy: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'ctx-default-alpha',
- namespace: 'mcp-server',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'default',
- 'clerum.io/mcpserver': 'alpha',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- }
- const lostFence = Object.assign(new Error('the UID in the precondition does not match'), {
- code: 409,
- })
-
- function listOnlyTheStalePolicy(): void {
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace }: { namespace?: string }) => ({
- items: namespace === 'mcp-server' ? [stalePolicy] : [],
- })
- )
- }
-
- it('refuses to certify a scoped delta instead of throwing the lost fence', async () => {
- listOnlyTheStalePolicy()
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)
-
- // 409 is what Kubernetes returns when the uid/resourceVersion
- // preconditions no longer match: the object live under that name is not
- // the one this pass classified. Nothing was revoked, so the delta must
- // decline to certify — and it must not blow up the caller, which would
- // only force a spurious full re-convergence.
- // The scoped delta lane reads this boolean and withholds the certificate,
- // so it is the one caller allowed to take the lost fence as an outcome
- // rather than a throw.
- expect(await reconciler.reconcileContext(staleContext, { honorsLostFence: true, isCurrent: () => true })).toBe(false)
- expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: 'ctx-default-alpha', namespace: 'mcp-server' })
- )
- })
-
- it('withdraws certification when an additive revocation loses the fence after certifying', async () => {
- // The additive phase re-runs reconcileContext, which carries a second
- // revocation lane over a fresh LIST. Losing the fence there happens after
- // the pass already certified, and the outcome recorder is first-wins — so
- // the duration sample stays 'completed', which is right, but the safety
- // fence must still degrade. It is an assertion about live allows, and it
- // may only ever move toward unsafe.
- const additiveContext: ContextCRD = {
- name: 'additive',
- namespace: 'mcp-server',
- spec: { contextId: 'additive', mcpServers: [] },
- }
- const additiveStalePolicy: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'ctx-additive-alpha',
- namespace: 'mcp-server',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'additive',
- 'clerum.io/mcpserver': 'alpha',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- }
- // Self-synchronising: the stale policy only becomes visible once the pass
- // has already certified, so the safety phase is clean by construction and
- // only the additive re-LIST finds work.
- let certified = false
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace }: { namespace?: string }) => ({
- items: certified && namespace === 'mcp-server' ? [additiveStalePolicy] : [],
- })
- )
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValue(lostFence)
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- certified = true
- })
-
- await expect(
- reconciler.fullReconcile([additiveContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow()
-
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- })
-
- it('marks the inventory uncertified the moment the authoritative pass loses the fence', async () => {
- // The doom must be visible mid-pass, not only after the unwind. A lost
- // fence is the one doom cause that does not bump a watch generation, so
- // it is the only one where the whole certification machinery stays green
- // while the pass is already condemned. A scoped delta interleaving in
- // that window would certify readiness over a stale allow.
- //
- // Two contexts, each with its own stale policy: the first loses the fence
- // and the second's delete gives an observation point while the pass is
- // condemned but still unwinding.
- const secondContext: ContextCRD = {
- name: 'second',
- namespace: 'mcp-server',
- spec: { contextId: 'second', mcpServers: [] },
- }
- const secondStalePolicy: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'ctx-second-beta',
- namespace: 'mcp-server',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'context-allow',
- 'clerum.io/context': 'second',
- 'clerum.io/mcpserver': 'beta',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
- }
- mockApi.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace }: { namespace?: string }) => ({
- items: namespace === 'mcp-server' ? [stalePolicy, secondStalePolicy] : [],
- })
- )
- let certifiedDuringUnwind: boolean | undefined
- mockApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => {
- if (
- certifiedDuringUnwind === undefined &&
- mockApi.deleteNamespacedNetworkPolicy.mock.calls.length > 1
- ) {
- certifiedDuringUnwind = reconciler.hasCertifiedSafetyInventory()
- return {}
- }
- throw lostFence
- })
-
- await expect(
- reconciler.fullReconcile([staleContext, secondContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: vi.fn(),
- })
- ).rejects.toThrow()
-
- expect(certifiedDuringUnwind).toBe(false)
- })
-
- it('fails the additive phase when it loses the delete fence', async () => {
- listOnlyTheStalePolicy()
- // Safety phase revokes cleanly and certifies; the additive phase then
- // races another writer and loses the fence. The additive caller discards
- // reconcileContext's boolean, so swallowing the 409 there would record
- // the pass as a success, disarm the convergence retry, and leave a stale
- // allow live with no NetworkPolicy resync to recover it.
- mockApi.deleteNamespacedNetworkPolicy
- .mockResolvedValueOnce({})
- .mockRejectedValueOnce(lostFence)
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([staleContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow()
-
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- })
-
- it('refuses to certify the safety pass instead of throwing the lost fence', async () => {
- listOnlyTheStalePolicy()
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(lostFence)
- const onAuthoritativeRevocationComplete = vi.fn()
-
- await expect(
- reconciler.fullReconcile([staleContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/Desired NetworkPolicy inventory changed/)
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- })
-
- it('still fails loud when the delete fails for any other reason', async () => {
- listOnlyTheStalePolicy()
- const forbidden = Object.assign(new Error('forbidden'), { code: 403 })
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)
-
- await expect(reconciler.reconcileContext(staleContext, { isCurrent: () => true })).rejects.toBe(forbidden)
- })
-
- it('still certifies when the stale policy was already gone', async () => {
- listOnlyTheStalePolicy()
- mockApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('not found'), { code: 404 })
- )
-
- expect(await reconciler.reconcileContext(staleContext, { isCurrent: () => true })).toBe(true)
- })
- })
-
- describe('same-name conflict creating a safety external-egress policy', () => {
- const server: McpServerCRD = {
- name: 'renamed-server',
- namespace: 'mcp-server',
- spec: {
- contextRef: 'default',
- image: 'renamed:latest',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ cidr: '104.18.0.0/16', port: 443 }],
- },
- }
- const desiredName = 'ext-egress-renamed-server-104-18-0-0-16-443'
- const stalePredecessor: k8s.V1NetworkPolicy = {
- metadata: {
- name: 'ext-egress-renamed-server-stale',
- namespace: 'mcp-server',
- uid: 'stale-uid',
- resourceVersion: '3',
- },
- }
-
- function reconcileExternalEgressSafety(): Promise<boolean> {
- return (
- reconciler as unknown as {
- reconcileExternalEgressSafety: (
- server: McpServerCRD,
- existingPolicies: k8s.V1NetworkPolicy[],
- isCurrent: () => boolean
- ) => Promise<boolean>
- }
- ).reconcileExternalEgressSafety(server, [stalePredecessor], () => true)
- }
-
- it('converges an HCC-owned same-name policy instead of aborting the pass', async () => {
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- name: desiredName,
- namespace: server.namespace,
- uid: 'own-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': server.name,
- },
- },
- })
-
- // This is the only creation route that bypasses applyNetworkPolicy. A
- // same-name object HCC already owns is an idempotent re-run, not a
- // reason to fail the pass and hold readiness down.
- expect(await reconcileExternalEgressSafety()).toBe(true)
- expect(mockApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledWith(
- expect.objectContaining({ name: desiredName, namespace: server.namespace })
- )
- })
-
- it('still fails closed when the same-name policy is foreign', async () => {
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(
- Object.assign(new Error('already exists'), { code: 409 })
- )
- mockApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
- metadata: {
- name: desiredName,
- namespace: server.namespace,
- uid: 'foreign-uid',
- resourceVersion: '7',
- labels: {
- 'clerum.io/managed-by': 'foreign-controller',
- 'clerum.io/policy-type': 'external-egress',
- 'clerum.io/mcpserver': server.name,
- },
- },
- })
-
- await expect(reconcileExternalEgressSafety()).rejects.toThrow(/ownership/i)
- expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
- })
-
- it('still fails loud when the create fails for any other reason', async () => {
- const forbidden = Object.assign(new Error('forbidden'), { code: 403 })
- mockApi.createNamespacedNetworkPolicy.mockRejectedValueOnce(forbidden)
-
- await expect(reconcileExternalEgressSafety()).rejects.toBe(forbidden)
- })
- })
-
- describe('safety-pass duration on the uncertified return path', () => {
- it('samples a failed outcome when the pass returns without certifying', async () => {
- const onAuthoritativeRevocationComplete = vi.fn()
-
- // Losing inventory authority ends the pass without an exception and
- // without certifying. Operators alert on "the pass never completed", so
- // this path has to produce a `failed` sample — absence is not a signal.
- await expect(
- reconciler.fullReconcile([], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => false,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).resolves.toBeUndefined()
-
- expect(onAuthoritativeRevocationComplete).not.toHaveBeenCalled()
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
- { outcome: 'failed' },
- expect.any(Number)
- )
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
- })
-
- it('keeps the completed sample when a certified pass fails an additive effect afterwards', async () => {
- const context: ContextCRD = {
- name: 'default',
- namespace: 'mcp-server',
- spec: { contextId: 'default', mcpServers: [] },
- }
- const additiveFailure = new Error('additive list unavailable')
- let certified = false
- const onAuthoritativeRevocationComplete = vi.fn(() => {
- certified = true
- })
- mockApi.listNamespacedNetworkPolicy.mockImplementation(async () => {
- if (certified) throw additiveFailure
- return { items: [] }
- })
-
- await expect(
- reconciler.fullReconcile([context], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete,
- })
- ).rejects.toThrow(/additive Context NetworkPolicy reconciliations failed/)
-
- expect(onAuthoritativeRevocationComplete).toHaveBeenCalledOnce()
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledWith(
- { outcome: 'completed' },
- expect.any(Number)
- )
- expect(networkPolicySafetyPassDurationSeconds.observe).toHaveBeenCalledTimes(1)
- })
})
})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index 70eab2a1a..8778887cd 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -27,14 +27,6 @@ import {
POLICY_TYPE_LABEL,
} from './constants'
import { accumulateHostExactHostEgress } from './externalEgressAccumulator'
-import {
- confirmAuthoritativeMcpServerAbsence,
- sameMcpServerDesiredRevision,
-} from './mcpServerSafety'
-import {
- networkPolicySafetyPassDurationSeconds,
- networkPolicySafetyPassPoliciesTotal,
-} from './metrics'
import {
ContextCRD,
EgressBinding,
@@ -43,181 +35,21 @@ import {
McpServerCrdStatus,
McpServerResolvedEgressIP,
} from './types'
-import { applyNetworkPolicy, getErrorCode, replaceWithConflictRetry } from './utils'
+import { applyNetworkPolicy, getErrorCode } from './utils'
type JsonPatchOperation = {
- op: 'add' | 'replace' | 'remove' | 'test'
+ op: 'add' | 'replace' | 'remove'
path: string
value?: unknown
}
-type NetworkPolicyMutationOptions = {
- // Required (not optional): reconcileContext and reconcileExternalEgress are the
- // two functions that CREATE/RETAIN context-allow (L2) and external-egress ALLOW
- // (L3) policies, so the authority fence must be supplied at compile time. A
- // fail-open `?? (() => true)` default here would silently readmit the exact
- // fail-open class hardened out of bindingPolicyReconciler (required isCurrent).
- isCurrent: () => boolean
- onRevoked?: () => void
- /**
- * Set only by a caller that reads the returned boolean and withholds
- * readiness on false. It downgrades a lost delete fence from a throw to a
- * reported outcome; a caller that discards the boolean must not set it, or a
- * stale allow survives while the pass is recorded as a success.
- */
- honorsLostFence?: boolean
-}
-
-export function sameContextDesiredRevision(expected: ContextCRD, current: ContextCRD): boolean {
- return (
- expected.name === current.name &&
- expected.namespace === current.namespace &&
- expected.uid === current.uid &&
- expected.generation === current.generation &&
- JSON.stringify(canonicalizeNetworkPolicyValue(expected.spec)) ===
- JSON.stringify(canonicalizeNetworkPolicyValue(current.spec))
- )
-}
-
-function canonicalizeNetworkPolicyValue(value: unknown): unknown {
- if (Array.isArray(value)) return value.map(canonicalizeNetworkPolicyValue)
- if (value === null || typeof value !== 'object') return value
- return Object.fromEntries(
- Object.entries(value as Record<string, unknown>)
- .filter(([, entry]) => entry !== undefined)
- .sort(([left], [right]) => left.localeCompare(right))
- .map(([key, entry]) => [key, canonicalizeNetworkPolicyValue(entry)])
- )
-}
-
-function sameNetworkPolicySpec(
- existing: k8s.V1NetworkPolicy,
- desired: k8s.V1NetworkPolicy
-): boolean {
- return (
- JSON.stringify(canonicalizeNetworkPolicyValue(existing.spec)) ===
- JSON.stringify(canonicalizeNetworkPolicyValue(desired.spec))
- )
-}
-
-function hasExpectedPolicyOwnership(policy: k8s.V1NetworkPolicy, policyType: string): boolean {
- const labels = policy.metadata?.labels ?? {}
- return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[POLICY_TYPE_LABEL] === policyType
-}
-
const GROUP = 'clerum.io'
const VERSION = 'v1alpha1'
const PLURAL_MCPSERVERS = 'mcpservers'
-const PLURAL_CONTEXTS = 'contexts'
const CONTEXT_LABEL = 'clerum.io/context'
const EGRESS_CLASS_LABEL = 'clerum.io/egress-class'
const RPC_PROXY_EGRESS_POLICY_TYPE = 'rpc-proxy-egress'
const RPC_PROXY_APP_LABEL = 'rpc-proxy'
-
-type SafetyInventoryLane =
- | 'context-ingress'
- | 'context-host-egress'
- | 'context-rpc-egress'
- | 'external-egress'
-
-type SafetyInventoryClassification = 'owned' | 'repairable' | 'ambiguous' | 'unrelated'
-
-function policyHasOwnerShape(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {
- const labels = policy.metadata?.labels ?? {}
- const hasServerOwner = Boolean(labels[MCPSERVER_LABEL])
- if (lane === 'external-egress') return hasServerOwner
- return Boolean(labels[CONTEXT_LABEL]) && hasServerOwner
-}
-
-function policyHasReservedName(policy: k8s.V1NetworkPolicy, lane: SafetyInventoryLane): boolean {
- const name = policy.metadata?.name ?? ''
- switch (lane) {
- case 'context-ingress':
- return name.startsWith('ctx-')
- case 'context-host-egress':
- return name.startsWith('ctx-') && name.endsWith('-egress')
- case 'context-rpc-egress':
- return name.startsWith('rpc-egress-')
- case 'external-egress':
- return name.startsWith('ext-egress-')
- }
-}
-
-function policyNameBelongsToDifferentLane(
- policy: k8s.V1NetworkPolicy,
- lane: SafetyInventoryLane
-): boolean {
- const name = policy.metadata?.name ?? ''
- if (lane === 'context-ingress') return name.startsWith('ext-egress-')
- if (lane === 'external-egress') return name.startsWith('ctx-')
- return false
-}
-
-function expectedPolicyType(lane: SafetyInventoryLane): string {
- return lane === 'context-rpc-egress'
- ? RPC_PROXY_EGRESS_POLICY_TYPE
- : lane === 'external-egress'
- ? EXTERNAL_EGRESS_POLICY_TYPE
- : 'context-allow'
-}
-
-/**
- * Classify broad namespace inventory without taking ownership of unrelated
- * policies. A single missing HCC marker is recoverable only when owner labels
- * corroborate the lane. Reserved HCC names and conflicting ownership markers
- * are never adopted implicitly.
- */
-function classifySafetyInventoryPolicy(
- policy: k8s.V1NetworkPolicy,
- lane: SafetyInventoryLane
-): SafetyInventoryClassification {
- const labels = policy.metadata?.labels ?? {}
- const managedBy = labels[MANAGED_BY_LABEL]
- const policyType = labels[POLICY_TYPE_LABEL]
- const expectedType = expectedPolicyType(lane)
- const managedMatches = managedBy === MANAGED_BY_VALUE
- const typeMatches = policyType === expectedType
- const managedMissing = managedBy === undefined
- const typeMissing = policyType === undefined
- const reservedName = policyHasReservedName(policy, lane)
- const ownerShape = policyHasOwnerShape(policy, lane)
-
- if (managedMatches && typeMatches) return 'owned'
-
- const oneMarkerMissing = (managedMatches && typeMissing) || (managedMissing && typeMatches)
- if (managedMissing && typeMatches && ownerShape) {
- return reservedName ? 'repairable' : 'ambiguous'
- }
- if (managedMatches && typeMissing && ownerShape) {
- if (reservedName) return 'repairable'
- if (policyNameBelongsToDifferentLane(policy, lane)) return 'unrelated'
- return 'ambiguous'
- }
-
- const knownOtherHccType =
- policyType !== undefined &&
- ['context-allow', RPC_PROXY_EGRESS_POLICY_TYPE, EXTERNAL_EGRESS_POLICY_TYPE].includes(
- policyType
- ) &&
- !typeMatches
- const hasRawConflictingMarker =
- (managedBy !== undefined && !managedMatches) || (policyType !== undefined && !typeMatches)
- const hasConflictingMarker =
- (managedBy !== undefined && !managedMatches) ||
- (policyType !== undefined && !typeMatches && !knownOtherHccType)
- if (
- reservedName &&
- ((managedMissing && typeMissing) || hasRawConflictingMarker || oneMarkerMissing)
- ) {
- return 'ambiguous'
- }
- if (ownerShape && hasConflictingMarker) {
- return 'ambiguous'
- }
-
- return 'unrelated'
-}
-
export const PUBLIC_EGRESS_EXCEPT_CIDRS = [
'0.0.0.0/8',
'10.0.0.0/8',
@@ -366,45 +198,6 @@ function k8sApiEgressPodSelector(namespace: string): k8s.V1LabelSelector {
return { matchLabels: { 'clerum.io/k8s-api-egress': 'true' } }
}
-export interface NetworkPolicyFullReconcileOptions {
- serverInventoryComplete?: boolean
- ensureDefaults?: boolean
- /**
- * Dynamic authority checks are intentionally callbacks: watch authority can
- * be lost after the startup snapshot but before an orphan delete begins.
- * Omitted callbacks preserve the historical authoritative-by-default API.
- */
- contextInventoryAuthoritative?: () => boolean
- serverInventoryAuthoritative?: () => boolean
- runContextEffect?: (contextId: string, work: () => Promise<void>) => Promise<void>
- runServerEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
- resolveCurrentContext?: (name: string) => ContextCRD | undefined
- resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined
- resolveCurrentServer?: (name: string) => McpServerCRD | undefined
- /**
- * Monotonic desired-state revisions fence ordinary watch events that arrive
- * after an owner has completed its safety turn. Watch generations remain the
- * separate authority fence for LIST -> WATCH recovery.
- */
- contextDesiredRevision?: () => number
- serverDesiredRevision?: () => number
- /**
- * Called only after every authoritative orphan-allow revocation lane has
- * completed. Additive Context/McpServer convergence intentionally happens
- * afterwards and must not delay this safety boundary.
- */
- onAuthoritativeRevocationComplete?: () => void
- /**
- * Fired at the moment each external-egress allow owned by an McpServer is
- * revoked (or replaced) during the authoritative safety pass — before any
- * isCurrent abort or inventory-changed throw can unwind the pass. The owner
- * uses it to queue an additive recreation for the affected server; queuing in
- * a pass epilogue instead would reproduce exactly the abort gap this hook
- * closes.
- */
- onExternalEgressRevoked?: (server: McpServerCRD) => void
-}
-
export class NetworkPolicyReconciler {
private networkingApi: k8s.NetworkingV1Api
private customApi: k8s.CustomObjectsApi
@@ -412,22 +205,6 @@ export class NetworkPolicyReconciler {
/** Reference to the MCP server cache so we can look up ports. */
private serverCache: Map<string, McpServerCRD>
- /**
- * Set whenever an authoritative safety pass ends without certifying, cleared
- * as soon as one certifies again. Only that pass enumerates the bounded
- * namespaces and classifies every object it finds; the scoped delta lane sees
- * nothing but its own label selectors. A delta therefore cannot vouch for a
- * namespace whose classified inventory is still stuck on an unresolved
- * policy, and must not certify readiness on its behalf.
- *
- * Initialised fail-closed (true): hasCertifiedSafetyInventory() reports false
- * until the first authoritative pass actually certifies, so certification
- * cannot be implied before any pass has run. This does not depend on the
- * readiness gate's generation-equality check for safety — it removes that
- * coupling rather than relying on it.
- */
- private safetyPassLeftUncertified = true
-
// H2 (issue #299): smallest DNS TTL (ms) observed across external-egress
// resolutions. The resync loop advances to <= this/2 so a rotating low-TTL host
// is sampled every rotation. Ratchets down and never recovers until restart —
@@ -526,7 +303,7 @@ export class NetworkPolicyReconciler {
},
}
- await this.applyFixedPolicy(name, ns, policy, 'default-deny')
+ await applyNetworkPolicy(this.networkingApi, name, ns, policy, '[NetPol]')
}
}
@@ -600,7 +377,7 @@ export class NetworkPolicyReconciler {
egress,
},
}
- await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)
+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')
}
/** Allow egress to host-context-controller API gateway in control-plane namespace. */
@@ -637,7 +414,7 @@ export class NetworkPolicyReconciler {
],
},
}
- await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)
+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')
}
/**
@@ -682,7 +459,7 @@ export class NetworkPolicyReconciler {
],
},
}
- await this.applyFixedPolicy(name, namespace, policy, INFRA_POLICY_TYPE)
+ await applyNetworkPolicy(this.networkingApi, name, namespace, policy, '[NetPol]')
}
/**
@@ -725,26 +502,46 @@ export class NetworkPolicyReconciler {
},
}
- await this.applyFixedPolicy(name, config.namespace, policy, 'allow-api')
+ await this.applyPolicy(name, policy)
}
// ─── Context-Based Policies ──────────────────────────────────────────
- private buildContextAllowPolicies(
- context: ContextCRD,
- server: McpServerCRD
- ): {
- ingress: k8s.V1NetworkPolicy
- hostEgress: k8s.V1NetworkPolicy
- rpcProxyEgress: k8s.V1NetworkPolicy
- } {
+ /**
+ * Reconcile NetworkPolicies for a Context CRD.
+ * Creates one policy per allowed MCP server.
+ */
+ async reconcileContext(context: ContextCRD): Promise<void> {
const contextId = context.spec.contextId
- const serverName = server.name
- const name = this.policyName(contextId, serverName)
- const port = server.spec.transport.port || 3000
+ const allowedServers = context.spec.mcpServers || []
+
+ console.log(
+ `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`
+ )
- return {
- ingress: {
+ // Build desired policy names
+ const desiredNames = new Set(
+ allowedServers.map(serverName => this.policyName(contextId, serverName))
+ )
+
+ // Get existing policies for this context
+ const existingPolicies = await this.listPoliciesForContext(contextId)
+
+ // Create or update policies for each allowed server
+ for (const serverName of allowedServers) {
+ const server = this.serverCache.get(serverName)
+ if (!server) {
+ console.warn(
+ `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`
+ )
+ continue
+ }
+
+ const name = this.policyName(contextId, serverName)
+ const port = server.spec.transport.port || 3000
+
+ // L2 ingress: allow scoped mcp-host/rpc-proxy access to this MCP server.
+ const ingressPolicy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
@@ -793,6 +590,7 @@ export class NetworkPolicyReconciler {
},
},
{
+ // MCP Proxy ingress: allow proxy to reach MCP servers
namespaceSelector: {
matchLabels: {
'kubernetes.io/metadata.name': config.namespace,
@@ -809,12 +607,18 @@ export class NetworkPolicyReconciler {
},
],
},
- },
- hostEgress: {
+ }
+
+ await this.applyPolicy(name, ingressPolicy)
+
+ // L2 egress counterpart: allow traffic FROM mcp-host pods TO this MCP server
+ // Without this, L0 egress deny-all blocks agents from reaching MCP servers.
+ const egressName = `${name}-egress`
+ const egressPolicy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
- name: `${name}-egress`,
+ name: egressName,
namespace: config.hostNamespace,
labels: {
[MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
@@ -851,322 +655,78 @@ export class NetworkPolicyReconciler {
},
],
},
- },
- rpcProxyEgress: this.buildRpcProxyEgressPolicy(contextId, serverName, port),
- }
- }
-
- /**
- * Reconcile NetworkPolicies for a Context CRD.
- * Creates one policy per allowed MCP server.
- *
- * Returns whether this Context's stale-allow revocation ran to completion.
- * Callers that certify readiness from a scoped delta MUST honour it: the pass
- * aborts as soon as its authority fence breaks, and an aborted pass leaves
- * stale allows live.
- */
- async reconcileContext(
- context: ContextCRD,
- options: NetworkPolicyMutationOptions
- ): Promise<boolean> {
- const callerIsCurrent = options.isCurrent
- if (!callerIsCurrent()) return false
- const contextId = context.spec.contextId
- const allowedServers = context.spec.mcpServers || []
- const selectedServers = new Map(
- allowedServers.map(serverName => [serverName, this.serverCache.get(serverName)])
- )
- const isCurrent = (): boolean =>
- callerIsCurrent() &&
- allowedServers.every(serverName => {
- const selected = selectedServers.get(serverName)
- const current = this.serverCache.get(serverName)
- if (!selected || !current) return selected === current
- return sameMcpServerDesiredRevision(selected, current)
- })
- if (!isCurrent()) return false
-
- console.log(
- `[NetPol] Reconciling context "${contextId}" — allowed servers: [${allowedServers.join(', ')}]`
- )
-
- // Create or update policies for each allowed server.
- for (const serverName of allowedServers) {
- if (!isCurrent()) return false
- const server = this.serverCache.get(serverName)
- if (!server) {
- console.warn(
- `[NetPol] McpServer "${serverName}" not found in cache — skipping policy for context "${contextId}"`
- )
- continue
}
- const desired = this.buildContextAllowPolicies(context, server)
- const ingressName = desired.ingress.metadata!.name!
- await this.applyContextIngressPolicy(ingressName, desired.ingress, isCurrent)
- if (!isCurrent()) return false
-
- const hostEgressName = desired.hostEgress.metadata!.name!
- await this.applyOwnedPolicy(
- hostEgressName,
+ await applyNetworkPolicy(
+ this.networkingApi,
+ egressName,
config.hostNamespace,
- desired.hostEgress,
- 'context-host-egress',
- isCurrent
- )
- if (!isCurrent()) return false
-
- const rpcProxyEgressName = desired.rpcProxyEgress.metadata!.name!
- await this.applyOwnedPolicy(
- rpcProxyEgressName,
- config.rpcProxyNamespace,
- desired.rpcProxyEgress,
- 'context-rpc-egress',
- isCurrent
+ egressPolicy,
+ '[NetPol]'
)
- }
-
- // Re-LIST every lane after writes so a same-name policy that was just
- // replaced is compared in its new form and cannot delete itself.
- return await this.revokeStaleContextAllows(
- context,
- isCurrent,
- undefined,
- undefined,
- options.honorsLostFence === true
- )
- }
- /**
- * Whether the last authoritative safety pass certified its namespace-wide
- * inventory. Only that pass enumerates the bounded namespaces and classifies
- * every object it finds; a scoped delta sees nothing but its own label
- * selectors, so it cannot vouch for a namespace whose classified inventory is
- * still stuck. The readiness lane consults this before letting a delta
- * replace the safety certificate.
- */
- hasCertifiedSafetyInventory(): boolean {
- return !this.safetyPassLeftUncertified
- }
-
- private async revokeStaleContextAllows(
- context: ContextCRD,
- isCurrent: () => boolean,
- listedPolicies?: {
- context?: k8s.V1NetworkPolicy[]
- hostEgress?: k8s.V1NetworkPolicy[]
- rpcProxyEgress?: k8s.V1NetworkPolicy[]
- },
- onRevoked?: () => void,
- callerHonorsLostFence = false
- ): Promise<boolean> {
- if (!isCurrent()) return false
- const safetySnapshotProvided = listedPolicies !== undefined
- // A delete whose uid/resourceVersion preconditions were rejected revoked
- // nothing. Reporting it instead of throwing is only safe for a caller that
- // reads the returned boolean: the authoritative snapshot lane aborts its
- // pass on it, and the scoped delta lane declines to certify. Every other
- // caller discards the boolean, so for them a swallowed 409 would leave a
- // stale allow live while the pass is recorded as a success — they keep the
- // loud default and let the failure reach the retry policy.
- const lostFenceOutcome =
- safetySnapshotProvided || callerHonorsLostFence ? ('report' as const) : ('throw' as const)
- let lostDeleteFence = false
- /**
- * Revoke one orphaned policy in any lane. Returns false only when the
- * caller must abort the pass outright, which is what the authoritative
- * safety-snapshot lane does with a lost delete fence; the scoped lane
- * records it and keeps going so the rest of its work still lands.
- */
- const revokeOrphanedPolicy = async (
- namespace: string,
- existing: k8s.V1NetworkPolicy
- ): Promise<boolean> => {
- const deleted = await this.deleteSafetyPolicySnapshot(
- namespace,
- existing,
- isCurrent,
- undefined,
- lostFenceOutcome
- )
- if (deleted) return true
- if (safetySnapshotProvided) {
- // The authoritative pass is condemned from here: it will unwind and
- // throw. Losing the fence is the only doom cause that does not bump a
- // watch generation, so without this the certification machinery stays
- // green for the whole unwind and a concurrent scoped delta can certify
- // readiness over the allow this pass failed to revoke. Record it the
- // moment it is known rather than when the pass finally reports.
- this.safetyPassLeftUncertified = true
- return false
- }
- lostDeleteFence = true
- return true
- }
- const contextId = context.spec.contextId
- const allowedServers = context.spec.mcpServers || []
- const desiredIngress = new Map<string, k8s.V1NetworkPolicy>()
- const desiredHostEgress = new Map<string, k8s.V1NetworkPolicy>()
- const desiredRpcProxyEgress = new Map<string, k8s.V1NetworkPolicy>()
- for (const serverName of allowedServers) {
- const server = this.serverCache.get(serverName)
- if (!server) continue
- const desired = this.buildContextAllowPolicies(context, server)
- desiredIngress.set(desired.ingress.metadata!.name!, desired.ingress)
- desiredHostEgress.set(desired.hostEgress.metadata!.name!, desired.hostEgress)
- desiredRpcProxyEgress.set(desired.rpcProxyEgress.metadata!.name!, desired.rpcProxyEgress)
+ // L2 rpc-proxy egress: allow rpc-proxy pods to reach this MCP server
+ await this.ensureRpcProxyEgress(contextId, serverName, port)
}
- const existingPolicies =
- listedPolicies?.context ?? (await this.listPoliciesForContext(contextId))
- if (!isCurrent()) return false
+ // Delete policies for servers no longer in the context
for (const existing of existingPolicies) {
- if (!isCurrent()) return false
const existingName = existing.metadata?.name || ''
- const desired = desiredIngress.get(existingName)
- if (!desired) {
+ if (!desiredNames.has(existingName)) {
console.log(`[NetPol] Deleting orphaned policy "${existingName}"`)
- if (!(await revokeOrphanedPolicy(config.namespace, existing))) return false
- onRevoked?.()
- } else if (
- !sameNetworkPolicySpec(existing, desired) ||
- !hasExpectedPolicyOwnership(existing, 'context-allow')
- ) {
- console.log(`[NetPol] Replacing stale same-name policy "${existingName}"`)
- if (safetySnapshotProvided) {
- if (
- !(await this.replaceSafetyPolicySnapshot(
- config.namespace,
- existing,
- desired,
- 'context-ingress',
- isCurrent
- ))
- ) {
- return false
- }
- } else {
- await this.replaceSafetyPolicySnapshot(
- config.namespace,
- existing,
- desired,
- 'context-ingress',
- isCurrent
- )
- }
- onRevoked?.()
+ await this.deletePolicy(existingName)
}
}
- const existingEgressPolicies =
- listedPolicies?.hostEgress ?? (await this.listEgressPoliciesForContext(contextId))
- if (!isCurrent()) return false
+ // Delete orphaned L2 egress counterparts in mcp-host namespace
+ const existingEgressPolicies = await this.listEgressPoliciesForContext(contextId)
+ const desiredEgressNames = new Set(
+ allowedServers.map(serverName => `${this.policyName(contextId, serverName)}-egress`)
+ )
for (const existing of existingEgressPolicies) {
- if (!isCurrent()) return false
const existingName = existing.metadata?.name || ''
- const desired = desiredHostEgress.get(existingName)
- if (!desired) {
+ if (!desiredEgressNames.has(existingName)) {
console.log(`[NetPol] Deleting orphaned L2 egress policy "${existingName}"`)
- if (!(await revokeOrphanedPolicy(config.hostNamespace, existing))) return false
- onRevoked?.()
- } else if (
- !sameNetworkPolicySpec(existing, desired) ||
- !hasExpectedPolicyOwnership(existing, 'context-allow')
- ) {
- console.log(`[NetPol] Replacing stale same-name L2 egress policy "${existingName}"`)
- if (safetySnapshotProvided) {
- if (
- !(await this.replaceSafetyPolicySnapshot(
- config.hostNamespace,
- existing,
- desired,
- 'context-host-egress',
- isCurrent
- ))
- ) {
- return false
- }
- } else {
- await this.replaceSafetyPolicySnapshot(
- config.hostNamespace,
- existing,
- desired,
- 'context-host-egress',
- isCurrent
- )
- }
- onRevoked?.()
+ await this.deleteEgressPolicy(existingName)
}
}
- const existingRpcProxyPolicies =
- listedPolicies?.rpcProxyEgress ?? (await this.listRpcProxyEgressPoliciesForContext(contextId))
- if (!isCurrent()) return false
+ // Delete orphaned rpc-proxy egress counterparts. Keeping these after a
+ // server is removed from a Context would preserve an old in-cluster route.
+ const existingRpcProxyPolicies = await this.listRpcProxyEgressPoliciesForContext(contextId)
+ const desiredRpcProxyNames = new Set(
+ allowedServers.map(serverName => `rpc-egress-${contextId}-${serverName}`)
+ )
for (const existing of existingRpcProxyPolicies) {
- if (!isCurrent()) return false
const existingName = existing.metadata?.name || ''
- const desired = desiredRpcProxyEgress.get(existingName)
- if (!desired) {
+ if (!desiredRpcProxyNames.has(existingName)) {
console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${existingName}"`)
- if (!(await revokeOrphanedPolicy(config.rpcProxyNamespace, existing))) return false
- onRevoked?.()
- } else if (
- !sameNetworkPolicySpec(existing, desired) ||
- !hasExpectedPolicyOwnership(existing, RPC_PROXY_EGRESS_POLICY_TYPE)
- ) {
- console.log(`[NetPol] Replacing stale same-name rpc-proxy egress policy "${existingName}"`)
- if (safetySnapshotProvided) {
- if (
- !(await this.replaceSafetyPolicySnapshot(
- config.rpcProxyNamespace,
- existing,
- desired,
- 'context-rpc-egress',
- isCurrent
- ))
- ) {
- return false
- }
- } else {
- await this.replaceSafetyPolicySnapshot(
- config.rpcProxyNamespace,
- existing,
- desired,
- 'context-rpc-egress',
- isCurrent
- )
- }
- onRevoked?.()
+ await this.deletePolicyInNamespace(config.rpcProxyNamespace, existingName)
}
}
-
- return isCurrent() && !lostDeleteFence
}
/**
* Delete all NetworkPolicies for a deleted Context.
*/
- async reconcileDeleteContext(
- contextId: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ async reconcileDeleteContext(contextId: string): Promise<void> {
console.log(`[NetPol] Context "${contextId}" deleted — removing all policies`)
const policies = await this.listPoliciesForContext(contextId)
for (const policy of policies) {
- if (deleteAllowed && !(await deleteAllowed())) return
- await this.deleteSafetyPolicySnapshot(config.namespace, policy)
+ const name = policy.metadata?.name || ''
+ await this.deletePolicy(name)
}
// Also delete L2 egress counterparts in mcp-host namespace
const egressPolicies = await this.listEgressPoliciesForContext(contextId)
for (const policy of egressPolicies) {
- if (deleteAllowed && !(await deleteAllowed())) return
- await this.deleteSafetyPolicySnapshot(config.hostNamespace, policy)
+ const name = policy.metadata?.name || ''
+ await this.deleteEgressPolicy(name)
}
// Also delete L2 rpc-proxy egress policies
- await this.deleteRpcProxyPoliciesForContext(contextId, deleteAllowed)
+ await this.deleteRpcProxyPoliciesForContext(contextId)
}
/**
@@ -1175,742 +735,80 @@ export class NetworkPolicyReconciler {
async fullReconcile(
contexts: ContextCRD[],
servers: McpServerCRD[] = [],
- options: NetworkPolicyFullReconcileOptions = {}
- ): Promise<void> {
- const onAuthoritativeRevocationComplete = options.onAuthoritativeRevocationComplete
- if (!onAuthoritativeRevocationComplete) {
- await this.fullReconcileInternal(contexts, servers, options)
- return
- }
-
- const safetyPassStartedAt = performance.now()
- let safetyPassOutcomeRecorded = false
- /**
- * Settles both observable results of this pass: the duration sample and the
- * cross-lane fence the scoped delta reads. First outcome wins for both — a
- * pass that certified and only then failed an additive effect still revoked
- * every stale allow its inventory implied.
- */
- const recordSafetyPassOutcome = (outcome: 'completed' | 'failed'): void => {
- // The fence is an assertion about live allows, so it only ever moves
- // toward unsafe. The additive phase re-runs reconcileContext, which
- // carries a second revocation lane over a fresh LIST; losing the fence
- // there happens after this pass certified, and first-outcome-wins would
- // otherwise leave the gate green over an allow that was not revoked.
- if (outcome === 'failed') this.safetyPassLeftUncertified = true
- // The duration sample describes the pass's primary outcome, so it stays
- // first-wins: a certified pass that later fails an additive effect did
- // complete its authoritative revocation.
- if (safetyPassOutcomeRecorded) return
- safetyPassOutcomeRecorded = true
- this.safetyPassLeftUncertified = outcome === 'failed'
- networkPolicySafetyPassDurationSeconds.observe(
- { outcome },
- (performance.now() - safetyPassStartedAt) / 1_000
- )
- }
-
- try {
- await this.fullReconcileInternal(contexts, servers, {
- ...options,
- onAuthoritativeRevocationComplete: () => {
- onAuthoritativeRevocationComplete()
- recordSafetyPassOutcome('completed')
- },
- })
- // The pass returned without certifying: no exception, no certificate.
- // Operators alert on this exact condition, so it must produce a sample
- // rather than the absence of one. Stated as a condition rather than
- // leaning on first-outcome-wins to swallow it, because the fence below is
- // deliberately not first-wins.
- if (!safetyPassOutcomeRecorded) recordSafetyPassOutcome('failed')
- } catch (error) {
- recordSafetyPassOutcome('failed')
- throw error
- }
- }
-
- private async fullReconcileInternal(
- contexts: ContextCRD[],
- servers: McpServerCRD[] = [],
- options: NetworkPolicyFullReconcileOptions = {}
+ options: { serverInventoryComplete?: boolean; ensureDefaults?: boolean } = {}
): Promise<void> {
console.log(`[NetPol] Running full reconciliation for ${contexts.length} Context(s)`)
- // This is intentionally scoped to callers that use the authoritative
- // revocation callback. Ordinary additive reconciles must not be presented
- // as readiness safety passes.
- const measureSafetyPass = options.onAuthoritativeRevocationComplete !== undefined
- let safetyPassListedPolicies = 0
- let safetyPassRevokedPolicies = 0
- const recordSafetyPassRevocation = (): void => {
- if (measureSafetyPass) safetyPassRevokedPolicies += 1
- }
-
- // The revision fence starts at invocation, before any asynchronous
- // default-policy setup can interleave a watch update with this snapshot.
- const selectedContextDesiredRevision = options.contextDesiredRevision?.()
- const selectedServerDesiredRevision = options.serverDesiredRevision?.()
- const runContextEffect =
- options.runContextEffect ?? ((_contextId: string, work: () => Promise<void>) => work())
- const runServerEffect =
- options.runServerEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
- const contextInventoryIsCurrent = (): boolean =>
- (options.contextInventoryAuthoritative?.() ?? true) &&
- (options.serverInventoryAuthoritative?.() ?? true)
-
if (options.ensureDefaults !== false) {
await this.ensureDefaultPolicies()
}
- // Authority is monotonic within this pass. Once a watch loses authority,
- // this inventory snapshot cannot safely become authoritative again even if
- // the callback flips back before the next delete.
- let contextAuthorityLost = false
- let contextAuthorityWarningLogged = false
- const contextCleanupAuthoritative = (): boolean => {
- if (
- !contextAuthorityLost &&
- options.contextInventoryAuthoritative &&
- !options.contextInventoryAuthoritative()
- ) {
- contextAuthorityLost = true
- }
- if (contextAuthorityLost && !contextAuthorityWarningLogged) {
- contextAuthorityWarningLogged = true
- console.warn(
- '[NetPol] Skipping remaining Context policy orphan cleanup because Context inventory authority was lost'
- )
- }
- return !contextAuthorityLost
+ // Reconcile each context
+ for (const context of contexts) {
+ await this.reconcileContext(context)
}
- let serverAuthorityLost = false
- let serverAuthorityWarningLogged = false
- const serverCleanupAuthoritative = (): boolean => {
- if (
- !serverAuthorityLost &&
- options.serverInventoryAuthoritative &&
- !options.serverInventoryAuthoritative()
- ) {
- serverAuthorityLost = true
- }
- if (serverAuthorityLost && !serverAuthorityWarningLogged) {
- serverAuthorityWarningLogged = true
- console.warn(
- '[NetPol] Skipping remaining external egress orphan cleanup because McpServer inventory authority was lost'
- )
- }
- return !serverAuthorityLost
+ // Reconcile external egress for all existing servers on startup so a
+ // controller restart converges pre-existing McpServer CRDs too.
+ for (const server of servers) {
+ await this.reconcileExternalEgress(server)
}
+ // Delete policies for contexts that no longer exist
const desiredContextIds = new Set(contexts.map(c => c.spec.contextId))
- const cleanupOrphanedContextPolicies = async (
- policies: k8s.V1NetworkPolicy[],
- deletePolicy: (policy: k8s.V1NetworkPolicy) => Promise<boolean>,
- describePolicy: (name: string, contextId: string) => string
- ): Promise<void> => {
- if (!contextCleanupAuthoritative()) return
- for (const policy of policies) {
- const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]
- if (!contextId) {
- if (!contextCleanupAuthoritative()) return
- const name = policy.metadata?.name
- if (!name) {
- throw new Error('HCC-managed Context NetworkPolicy is missing its name and owner label')
- }
- console.warn(
- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${CONTEXT_LABEL}`
- )
- if (await deletePolicy(policy)) recordSafetyPassRevocation()
- continue
- }
- if (!desiredContextIds.has(contextId)) {
- const name = policy.metadata?.name || ''
- await runContextEffect(contextId, async () => {
- if (
- !(await this.contextOrphanDeleteAllowed(
- contextId,
- contextCleanupAuthoritative,
- options.resolveCurrentContextById
- ))
- ) {
- return
- }
- console.log(describePolicy(name, contextId))
- if (await deletePolicy(policy)) recordSafetyPassRevocation()
- })
- }
+ const allContextPolicies = await this.listAllContextPolicies()
+
+ for (const policy of allContextPolicies) {
+ const contextId = policy.metadata?.labels?.[CONTEXT_LABEL]
+ if (contextId && !desiredContextIds.has(contextId)) {
+ const name = policy.metadata?.name || ''
+ console.log(
+ `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`
+ )
+ await this.deletePolicy(name)
}
}
- let allContextPolicies = contextCleanupAuthoritative()
- ? await this.listAllContextPolicies()
- : []
- let allContextEgressPolicies = contextCleanupAuthoritative()
- ? await this.listAllContextEgressPolicies()
- : []
- let allRpcProxyEgressPolicies = contextCleanupAuthoritative()
- ? await this.listAllRpcProxyEgressPolicies()
- : []
- let allExternalPolicies: k8s.V1NetworkPolicy[] = []
- if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {
- allExternalPolicies = await this.listAllExternalEgressPolicies()
- }
-
- // Label selectors alone cannot prove safety: removing either ownership
- // marker would hide an active allow from the selected LIST. Audit the
- // bounded namespaces, then merge only owned or conservatively repairable
- // candidates into their lane. Ambiguous reserved names fail the pass.
- const needsMcpServerBroadInventory =
- contextCleanupAuthoritative() ||
- (options.serverInventoryComplete !== false && serverCleanupAuthoritative())
- const broadMcpServerPolicies = needsMcpServerBroadInventory
- ? await this.listNamespacePoliciesForSafety(config.namespace)
- : []
- const broadHostPolicies = contextCleanupAuthoritative()
- ? await this.listNamespacePoliciesForSafety(config.hostNamespace)
- : []
- const broadRpcProxyPolicies = contextCleanupAuthoritative()
- ? await this.listNamespacePoliciesForSafety(config.rpcProxyNamespace)
- : []
-
- if (contextCleanupAuthoritative()) {
- allContextPolicies = this.mergeSafetyLaneInventory(
- allContextPolicies,
- broadMcpServerPolicies,
- 'context-ingress'
- )
- allContextEgressPolicies = this.mergeSafetyLaneInventory(
- allContextEgressPolicies,
- broadHostPolicies,
- 'context-host-egress'
- )
- allRpcProxyEgressPolicies = this.mergeSafetyLaneInventory(
- allRpcProxyEgressPolicies,
- broadRpcProxyPolicies,
- 'context-rpc-egress'
- )
- }
- if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {
- allExternalPolicies = this.mergeSafetyLaneInventory(
- allExternalPolicies,
- broadMcpServerPolicies,
- 'external-egress'
- )
- }
- safetyPassListedPolicies +=
- allContextPolicies.length +
- allContextEgressPolicies.length +
- allRpcProxyEgressPolicies.length +
- allExternalPolicies.length
-
- // Delete orphaned Context policies across every L2 lane. Each lane uses
- // the canonical contextId effect key and rechecks authority plus live CRD
- // presence immediately before its delete.
- await cleanupOrphanedContextPolicies(
- allContextPolicies,
- policy =>
- this.deleteSafetyPolicySnapshot(config.namespace, policy, contextCleanupAuthoritative),
- (name, contextId) =>
- `[NetPol] Deleting orphaned policy "${name}" (context "${contextId}" no longer exists)`
- )
- await cleanupOrphanedContextPolicies(
- allContextEgressPolicies,
- policy =>
- this.deleteSafetyPolicySnapshot(config.hostNamespace, policy, contextCleanupAuthoritative),
- name => `[NetPol] Deleting orphaned L2 egress policy "${name}"`
- )
- await cleanupOrphanedContextPolicies(
- allRpcProxyEgressPolicies,
- policy =>
- this.deleteSafetyPolicySnapshot(
- config.rpcProxyNamespace,
- policy,
- contextCleanupAuthoritative
- ),
- name => `[NetPol] Deleting orphaned rpc-proxy egress policy "${name}"`
- )
-
- let liveDesiredRevisionChanged = false
- for (const context of contexts) {
- const contextId = context.spec.contextId
- await runContextEffect(contextId, async () => {
- const current = options.resolveCurrentContext
- ? options.resolveCurrentContext(context.name)
- : context
- if (!current || current.spec.contextId !== contextId) {
- liveDesiredRevisionChanged = true
- return
- }
- const contextEffectIsCurrent = (): boolean => {
- if (!contextCleanupAuthoritative() || !serverCleanupAuthoritative()) return false
- if (!options.resolveCurrentContext) return true
- const latest = options.resolveCurrentContext(current.name)
- return latest !== undefined && sameContextDesiredRevision(current, latest)
- }
- const forContext = (policy: k8s.V1NetworkPolicy): boolean =>
- policy.metadata?.labels?.[CONTEXT_LABEL] === contextId
- const completed = await this.revokeStaleContextAllows(
- current,
- contextEffectIsCurrent,
- {
- context: allContextPolicies.filter(forContext),
- hostEgress: allContextEgressPolicies.filter(forContext),
- rpcProxyEgress: allRpcProxyEgressPolicies.filter(forContext),
- },
- recordSafetyPassRevocation
- )
- if (!completed && contextCleanupAuthoritative() && serverCleanupAuthoritative()) {
- liveDesiredRevisionChanged = true
- }
- })
+ // Also clean up orphaned rpc-proxy egress policies
+ const allRpcProxyPolicies = await this.listAllRpcProxyEgressPolicies()
+ for (const policy of allRpcProxyPolicies) {
+ const ctxId = policy.metadata?.labels?.[CONTEXT_LABEL]
+ if (ctxId && !desiredContextIds.has(ctxId)) {
+ const pName = policy.metadata?.name || ''
+ console.log(`[NetPol] Deleting orphaned rpc-proxy egress policy "${pName}"`)
+ await this.deletePolicyInNamespace(config.rpcProxyNamespace, pName)
+ }
}
// Clean up external egress policies for servers that no longer exist.
- if (options.serverInventoryComplete !== false && serverCleanupAuthoritative()) {
+ if (options.serverInventoryComplete !== false) {
const desiredServerNames = new Set(servers.map(s => s.name))
+ const allExternalPolicies = await this.listAllExternalEgressPolicies()
for (const policy of allExternalPolicies) {
const serverName = policy.metadata?.labels?.[MCPSERVER_LABEL]
- if (!serverName) {
- if (!serverCleanupAuthoritative()) break
- const name = policy.metadata?.name
- if (!name) {
- throw new Error(
- 'HCC-managed external egress NetworkPolicy is missing its name and owner label'
- )
- }
- console.warn(
- `[NetPol] Deleting malformed HCC-managed policy "${name}" without ${MCPSERVER_LABEL}`
- )
- if (
- await this.deleteSafetyPolicySnapshot(
- config.namespace,
- policy,
- serverCleanupAuthoritative
- )
- ) {
- recordSafetyPassRevocation()
- }
- continue
- }
- if (!desiredServerNames.has(serverName)) {
+ if (serverName && !desiredServerNames.has(serverName)) {
const name = policy.metadata?.name || ''
- await runServerEffect(serverName, async () => {
- if (
- !(await confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: serverCleanupAuthoritative,
- resolveCurrent: () => options.resolveCurrentServer?.(serverName),
- readCurrent: () =>
- this.customApi.getNamespacedCustomObject({
- group: GROUP,
- version: VERSION,
- namespace: config.namespace,
- plural: PLURAL_MCPSERVERS,
- name: serverName,
- }),
- }))
- ) {
- return
- }
- console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)
- if (
- await this.deleteSafetyPolicySnapshot(
- config.namespace,
- policy,
- serverCleanupAuthoritative
- )
- ) {
- recordSafetyPassRevocation()
- }
- })
+ console.log(`[NetPol] Deleting orphaned external egress policy "${name}"`)
+ await this.deletePolicyInNamespace(config.namespace, name)
}
}
- } else if (options.serverInventoryComplete === false) {
+ } else {
console.warn(
'[NetPol] Skipping external egress orphan cleanup because server inventory is incomplete'
)
}
- if (options.serverInventoryComplete !== false) {
- for (const server of servers) {
- await runServerEffect(server.name, async () => {
- const current = options.resolveCurrentServer
- ? options.resolveCurrentServer(server.name)
- : server
- if (!current) {
- liveDesiredRevisionChanged = true
- return
- }
- const serverEffectIsCurrent = (): boolean => {
- if (!serverCleanupAuthoritative()) return false
- if (!options.resolveCurrentServer) return true
- const latest = options.resolveCurrentServer(current.name)
- return latest !== undefined && sameMcpServerDesiredRevision(current, latest)
- }
- const completed = await this.reconcileExternalEgressSafety(
- current,
- allExternalPolicies.filter(
- policy => policy.metadata?.labels?.[MCPSERVER_LABEL] === server.name
- ),
- serverEffectIsCurrent,
- () => {
- recordSafetyPassRevocation()
- options.onExternalEgressRevoked?.(current)
- }
- )
- if (!completed && serverCleanupAuthoritative()) {
- liveDesiredRevisionChanged = true
- }
- })
- }
- }
-
- const inventoryDesiredRevisionChanged =
- (selectedContextDesiredRevision !== undefined &&
- options.contextDesiredRevision?.() !== selectedContextDesiredRevision) ||
- (selectedServerDesiredRevision !== undefined &&
- options.serverDesiredRevision?.() !== selectedServerDesiredRevision)
- if (
- (liveDesiredRevisionChanged || inventoryDesiredRevisionChanged) &&
- contextCleanupAuthoritative() &&
- serverCleanupAuthoritative()
- ) {
- throw new Error('Desired NetworkPolicy inventory changed during authoritative revocation')
- }
-
- if (
- options.serverInventoryComplete !== false &&
- contextCleanupAuthoritative() &&
- serverCleanupAuthoritative()
- ) {
- if (measureSafetyPass) {
- networkPolicySafetyPassPoliciesTotal.inc({ operation: 'listed' }, safetyPassListedPolicies)
- networkPolicySafetyPassPoliciesTotal.inc(
- { operation: 'revoked' },
- safetyPassRevokedPolicies
- )
- }
- options.onAuthoritativeRevocationComplete?.()
- }
-
- // Additive Context convergence happens only after every stale allow policy
- // implied by the authoritative inventory has been revoked. A slow or
- // failing positive effect therefore cannot extend a stale-permission
- // window after readiness becomes available. External egress creation and
- // DNS refresh intentionally do not run here:
- // ExternalEgressConvergenceCoordinator is their sole startup, retry, and
- // periodic-resync owner.
- const contextAdditiveFailures: unknown[] = []
- for (const context of contexts) {
- const contextId = context.spec.contextId
- try {
- await runContextEffect(contextId, async () => {
- const current = options.resolveCurrentContext
- ? options.resolveCurrentContext(context.name)
- : context
- if (!current || current.spec.contextId !== contextId) return
- const contextEffectIsCurrent = (): boolean => {
- if (!contextInventoryIsCurrent()) return false
- if (!options.resolveCurrentContext) return true
- const latest = options.resolveCurrentContext(current.name)
- return latest !== undefined && sameContextDesiredRevision(current, latest)
- }
- await this.reconcileContext(current, { isCurrent: contextEffectIsCurrent })
- })
- } catch (error) {
- contextAdditiveFailures.push(error)
- console.error(`[NetPol] Additive Context reconciliation failed for "${contextId}":`, error)
- }
- }
-
- if (contextAdditiveFailures.length > 0) {
- throw new AggregateError(
- contextAdditiveFailures,
- 'One or more additive Context NetworkPolicy reconciliations failed'
- )
- }
-
console.log('[NetPol] Full reconciliation complete')
}
- private async contextOrphanDeleteAllowed(
- contextId: string,
- inventoryAuthoritative: () => boolean,
- resolveCurrentContextById?: (contextId: string) => ContextCRD | undefined
- ): Promise<boolean> {
- if (!inventoryAuthoritative() || resolveCurrentContextById?.(contextId)) return false
-
- const response = (await this.customApi.listNamespacedCustomObject({
- group: GROUP,
- version: VERSION,
- namespace: config.namespace,
- plural: PLURAL_CONTEXTS,
- })) as { items?: Array<{ spec?: { contextId?: string } }> }
- if ((response.items ?? []).some(context => context.spec?.contextId === contextId)) return false
-
- return inventoryAuthoritative() && !resolveCurrentContextById?.(contextId)
- }
-
// ─── L3 External Egress ────────────────────────────────────────────
- private buildPublicWebEgressPolicy(server: McpServerCRD, name: string): k8s.V1NetworkPolicy {
- return {
- apiVersion: 'networking.k8s.io/v1',
- kind: 'NetworkPolicy',
- metadata: {
- name,
- namespace: server.namespace,
- labels: {
- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
- [MCPSERVER_LABEL]: server.name,
- [EGRESS_CLASS_LABEL]: 'public-web',
- },
- },
- spec: {
- podSelector: {
- matchLabels: { [MCPSERVER_LABEL]: server.name },
- },
- policyTypes: ['Egress'],
- egress: [
- {
- to: [
- {
- ipBlock: {
- cidr: '0.0.0.0/0',
- except: PUBLIC_EGRESS_EXCEPT_CIDRS,
- },
- },
- ],
- ports: [
- { port: 443, protocol: 'TCP' },
- { port: 80, protocol: 'TCP' },
- ],
- },
- ],
- },
- }
- }
-
- private buildExactHostEgressPolicy(
- server: McpServerCRD,
- name: string,
- binding: EgressBinding,
- cidrs: string[]
- ): k8s.V1NetworkPolicy {
- return {
- apiVersion: 'networking.k8s.io/v1',
- kind: 'NetworkPolicy',
- metadata: {
- name,
- namespace: server.namespace,
- labels: {
- [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
- [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
- [MCPSERVER_LABEL]: server.name,
- [EGRESS_CLASS_LABEL]: 'exact-host',
- },
- },
- spec: {
- podSelector: {
- matchLabels: { [MCPSERVER_LABEL]: server.name },
- },
- policyTypes: ['Egress'],
- egress: cidrs.map(cidr => ({
- to: [{ ipBlock: { cidr } }],
- ports: [{ port: binding.port!, protocol: binding.protocol ?? 'TCP' }],
- })),
- },
- }
- }
-
- private async resolveExternalEgressDns(
- hostname: string
- ): Promise<{ address: string; ttl: number }[]> {
- // Resolve with TTL (issue #299 H2 ratchet needs the smallest observed TTL)
- // while keeping the #205 bounded-timeout race so a silent resolver cannot
- // block safety convergence indefinitely.
- let timeout: ReturnType<typeof setTimeout> | undefined
- try {
- return await Promise.race([
- dns.resolve4(hostname, { ttl: true }),
- new Promise<never>((_, reject) => {
- timeout = setTimeout(
- () =>
- reject(
- new Error(
- `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`
- )
- ),
- config.externalEgressDnsResolveTimeoutMs
- )
- }),
- ])
- } finally {
- if (timeout !== undefined) clearTimeout(timeout)
- }
- }
-
- private async reconcileExternalEgressSafety(
- server: McpServerCRD,
- existingPolicies: k8s.V1NetworkPolicy[],
- isCurrent: () => boolean,
- onRevoked?: () => void
- ): Promise<boolean> {
- if (!isCurrent()) return false
- const desired = new Map<string, { binding: EgressBinding; policy?: k8s.V1NetworkPolicy }>()
- for (const binding of server.spec.egressBindings ?? []) {
- const name = this.externalEgressPolicyName(server.name, binding)
- if (!name) continue
- const egressClass = binding.egressClass ?? 'exact-host'
- if (
- egressClass === 'public-web' &&
- !binding.dns &&
- !binding.cidr &&
- binding.port === undefined &&
- binding.protocol === undefined
- ) {
- desired.set(name, { binding, policy: this.buildPublicWebEgressPolicy(server, name) })
- } else if (
- egressClass === 'exact-host' &&
- binding.cidr &&
- isAllowedExternalEgressCidr(binding.cidr) &&
- binding.port !== undefined &&
- Number.isInteger(binding.port) &&
- binding.port >= 1 &&
- binding.port <= 65535
- ) {
- desired.set(name, {
- binding,
- policy: this.buildExactHostEgressPolicy(server, name, binding, [binding.cidr]),
- })
- } else if (
- egressClass === 'exact-host' &&
- binding.dns &&
- isPublicDnsHostname(binding.dns) &&
- binding.port !== undefined &&
- Number.isInteger(binding.port) &&
- binding.port >= 1 &&
- binding.port <= 65535
- ) {
- desired.set(name, { binding })
- }
- }
-
- const stale: k8s.V1NetworkPolicy[] = []
- const unprovableDnsPolicies: k8s.V1NetworkPolicy[] = []
- const deterministicReplacements: Array<{
- snapshot: k8s.V1NetworkPolicy
- desired: k8s.V1NetworkPolicy
- }> = []
- for (const existing of existingPolicies) {
- const name = existing.metadata?.name
- const selected = name ? desired.get(name) : undefined
- if (!selected) {
- stale.push(existing)
- continue
- }
- desired.delete(name!)
- if (!selected.policy) {
- // DNS binding: the /32s cannot be proven without a fresh lookup, but
- // the binding IDENTITY (protocol, spec-level ports, selector,
- // ownership) can. Retain the live allow when identity is unchanged;
- // revoke only on an identity change (a port move in the CRD changes the
- // policy NAME and is handled by the stale lane above). The comparison is
- // "modulo cidr": the desired policy is rebuilt WITH the live cidrs, so
- // the cidrs compare against themselves and everything else must match
- // exactly. A policy with no ipBlock peers is a deny disguised as an
- // allow and is never retained.
- const existingCidrs = (existing.spec?.egress ?? [])
- .flatMap(rule => rule.to ?? [])
- .map(peer => peer.ipBlock?.cidr)
- .filter((cidr): cidr is string => typeof cidr === 'string')
- const identityUnchanged =
- existingCidrs.length > 0 &&
- hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE) &&
- sameNetworkPolicySpec(
- existing,
- this.buildExactHostEgressPolicy(server, name!, selected.binding, existingCidrs)
- )
- if (!identityUnchanged) unprovableDnsPolicies.push(existing)
- continue
- }
- const retainable =
- sameNetworkPolicySpec(existing, selected.policy) &&
- hasExpectedPolicyOwnership(existing, EXTERNAL_EGRESS_POLICY_TYPE)
- if (!retainable) {
- deterministicReplacements.push({ snapshot: existing, desired: selected.policy })
- }
- }
-
- // Preserve traffic only with deterministic CIDR/public-web policies. A
- // renamed deterministic binding is created before its stale predecessor is
- // removed; a same-name conflict is converged only under the ownership
- // guard, which still refuses to adopt a foreign live object.
- if (stale.length > 0) {
- for (const selected of desired.values()) {
- if (!selected.policy) continue
- if (
- !(await this.createSafetyPolicy(
- server.namespace,
- selected.policy,
- 'external-egress',
- isCurrent
- ))
- ) {
- return false
- }
- }
- }
- for (const replacement of deterministicReplacements) {
- if (
- !(await this.replaceSafetyPolicySnapshot(
- server.namespace,
- replacement.snapshot,
- replacement.desired,
- 'external-egress',
- isCurrent
- ))
- ) {
- return false
- }
- onRevoked?.()
- }
-
- // DNS-derived /32s are unprovable without a fresh lookup. Revoke them
- // immediately; replacement belongs to the additive lane and must not put
- // external DNS latency back on the global readiness path.
- await this.cleanupExternalEgress(
- server.name,
- server.namespace,
- unprovableDnsPolicies,
- async () => isCurrent(),
- onRevoked
- )
- if (!isCurrent()) return false
-
- await this.cleanupExternalEgress(
- server.name,
- server.namespace,
- stale,
- async () => isCurrent(),
- onRevoked
- )
- return isCurrent()
- }
-
/**
* Reconcile external egress policies for an McpServer with egressBindings.
* For DNS bindings, resolves hostnames to IPs and creates ipBlock rules.
* For CIDR bindings, creates ipBlock rules directly.
*/
- async reconcileExternalEgress(
- server: McpServerCRD,
- options: NetworkPolicyMutationOptions
- ): Promise<void> {
- const isCurrent = options.isCurrent
- if (!isCurrent()) return
+ async reconcileExternalEgress(server: McpServerCRD): Promise<void> {
let existingPolicies: k8s.V1NetworkPolicy[]
try {
existingPolicies = await this.listExternalEgressPoliciesForServer(
@@ -1918,48 +816,36 @@ export class NetworkPolicyReconciler {
server.namespace
)
} catch (error) {
- if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
[],
'False',
'InventoryListFailed',
- `Failed to list existing external egress policies: ${this.errorMessage(error)}`,
- isCurrent
+ `Failed to list existing external egress policies: ${this.errorMessage(error)}`
)
throw error
}
- if (!isCurrent()) return
const bindings = server.spec.egressBindings
if (!bindings || bindings.length === 0) {
try {
- await this.cleanupExternalEgress(
- server.name,
- server.namespace,
- existingPolicies,
- async () => isCurrent()
- )
+ await this.cleanupExternalEgress(server.name, server.namespace, existingPolicies)
} catch (error) {
- if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
[],
'False',
'CleanupFailed',
- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,
- isCurrent
+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`
)
throw error
}
- if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
[],
'True',
'NoEgressBindings',
- 'No external egress bindings declared',
- isCurrent
+ 'No external egress bindings declared'
)
return
}
@@ -1971,11 +857,9 @@ export class NetworkPolicyReconciler {
const desiredPolicyNames = new Set<string>()
const resolvedEgressIPs: McpServerResolvedEgressIP[] = []
const failures: string[] = []
- const resolvedAtMs = Date.now()
- const resolvedAt = new Date(resolvedAtMs).toISOString()
+ const resolvedAt = new Date().toISOString()
for (const binding of bindings) {
- if (!isCurrent()) return
const egressClass = binding.egressClass ?? 'exact-host'
if (egressClass !== 'exact-host' && egressClass !== 'public-web') {
failures.push(`egressClass "${String(binding.egressClass)}" is not supported`)
@@ -2021,13 +905,47 @@ export class NetworkPolicyReconciler {
desiredPolicyNames.add(name)
- // public-web is a static policy. Preserve #205's ownership-fenced write
- // (applyOwnedPolicy asserts ownership + honors the isCurrent generation
- // fence) while keeping dev's F2 no-churn gate so the TTL-accelerated
- // resync does not rewrite it every tick.
- const policy = this.buildPublicWebEgressPolicy(server, name)
+ const policy: k8s.V1NetworkPolicy = {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name,
+ namespace: server.namespace,
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
+ [MCPSERVER_LABEL]: server.name,
+ [EGRESS_CLASS_LABEL]: egressClass,
+ },
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { [MCPSERVER_LABEL]: server.name },
+ },
+ policyTypes: ['Egress'],
+ egress: [
+ {
+ to: [
+ {
+ ipBlock: {
+ cidr: '0.0.0.0/0',
+ except: PUBLIC_EGRESS_EXCEPT_CIDRS,
+ },
+ },
+ ],
+ ports: [
+ { port: 443, protocol: 'TCP' },
+ { port: 80, protocol: 'TCP' },
+ ],
+ },
+ ],
+ },
+ }
+
+ // public-web is a static policy — write only when new/changed/drifted so
+ // the TTL-accelerated resync does not rewrite it every tick (audit F2).
if (this.externalEgressWriteNeeded(existingPolicy, policy)) {
- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
+ await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')
}
continue
}
@@ -2075,8 +993,7 @@ export class NetworkPolicyReconciler {
maxEntries: config.externalEgressMaxEntries,
}
try {
- const records = await this.resolveExternalEgressDns(binding.dns)
- if (!isCurrent()) return
+ const records = await dns.resolve4(binding.dns, { ttl: true })
const uniqueIps = [...new Set(records.map(r => r.address))].sort()
if (uniqueIps.length === 0) {
failures.push(`hostname "${binding.dns}" resolved to no IPv4 addresses`)
@@ -2177,25 +1094,43 @@ export class NetworkPolicyReconciler {
desiredPolicyNames.add(name)
- // Build the exact-host policy from #205's helper, then fold in the #299
- // sliding-window state annotations so the next reconcile rehydrates the
- // accumulated set instead of collapsing to a single snapshot.
- const policy = this.buildExactHostEgressPolicy(server, name, binding, cidrs)
- if (stateAnnotations) {
- policy.metadata = { ...(policy.metadata ?? {}), annotations: stateAnnotations }
+ const policy: k8s.V1NetworkPolicy = {
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name,
+ namespace: server.namespace,
+ labels: {
+ [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
+ [POLICY_TYPE_LABEL]: EXTERNAL_EGRESS_POLICY_TYPE,
+ [MCPSERVER_LABEL]: server.name,
+ [EGRESS_CLASS_LABEL]: egressClass,
+ },
+ // Persist the sliding-window state so the next reconcile rehydrates the
+ // accumulated set instead of collapsing to a single snapshot (#299).
+ ...(stateAnnotations ? { annotations: stateAnnotations } : {}),
+ },
+ spec: {
+ podSelector: {
+ matchLabels: { [MCPSERVER_LABEL]: server.name },
+ },
+ policyTypes: ['Egress'],
+ egress: cidrs.map(cidr => ({
+ to: [{ ipBlock: { cidr } }],
+ ports: [{ port: binding.port, protocol }],
+ })),
+ },
}
// Write only when the enforced rules changed / the policy is new / drifted
// (audit F2 no-churn + L1 self-heal), OR when the DNS window is aging and
// must be re-persisted (audit M1). A pure timestamp refresh is a no-op.
- // Route the write through applyOwnedPolicy so #205's ownership assertion
- // and isCurrent generation fence still guard the mutation.
if (
this.externalEgressWriteNeeded(existingPolicy, policy) ||
egressRenewalDue ||
egressStateChanged
) {
- await this.applyOwnedPolicy(name, server.namespace, policy, 'external-egress', isCurrent)
+ await applyNetworkPolicy(this.networkingApi, name, server.namespace, policy, '[NetPol]')
} else {
console.log(`[NetPol] External egress "${name}" unchanged — no-op (issue #299 no-churn)`)
}
@@ -2208,23 +1143,18 @@ export class NetworkPolicyReconciler {
existingPolicies.filter(policy => {
const name = policy.metadata?.name
return Boolean(name) && !desiredPolicyNames.has(name!)
- }),
- async () => isCurrent(),
- options.onRevoked
+ })
)
} catch (error) {
- if (!isCurrent()) return
await this.writeExternalEgressStatus(
server,
resolvedEgressIPs,
'False',
'CleanupFailed',
- `Failed to delete stale external egress policies: ${this.errorMessage(error)}`,
- isCurrent
+ `Failed to delete stale external egress policies: ${this.errorMessage(error)}`
)
throw error
}
- if (!isCurrent()) return
if (failures.length > 0) {
const message = failures.join('; ')
@@ -2233,8 +1163,7 @@ export class NetworkPolicyReconciler {
resolvedEgressIPs,
'False',
'ExternalEgressRejected',
- message,
- isCurrent
+ message
)
throw new Error(`External egress reconciliation failed for "${server.name}": ${message}`)
}
@@ -2246,8 +1175,7 @@ export class NetworkPolicyReconciler {
'Reconciled',
bindings.some(binding => binding.egressClass === 'public-web')
? 'External egress policies reconciled; public-web allows public TCP 80/443 with private and special ranges excluded'
- : 'External egress policies reconciled',
- isCurrent
+ : 'External egress policies reconciled'
)
}
@@ -2257,9 +1185,7 @@ export class NetworkPolicyReconciler {
async cleanupExternalEgress(
serverName: string,
namespace: string,
- policies?: k8s.V1NetworkPolicy[],
- deleteAllowed?: () => Promise<boolean>,
- onDeleted?: () => void
+ policies?: k8s.V1NetworkPolicy[]
): Promise<void> {
const policiesToDelete =
policies ?? (await this.listExternalEgressPoliciesForServer(serverName, namespace))
@@ -2268,19 +1194,13 @@ export class NetworkPolicyReconciler {
const name = policy.metadata?.name
if (!name) continue
try {
- if (deleteAllowed && !(await deleteAllowed())) return
- await this.deleteSafetyPolicySnapshot(namespace, policy, undefined, () => {
- console.log(`[NetPol] Deleted external egress policy "${name}"`)
- onDeleted?.()
- })
+ await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
+ console.log(`[NetPol] Deleted external egress policy "${name}"`)
} catch (error: unknown) {
if (getErrorCode(error) !== 404) {
console.error(`[NetPol] Failed to delete external egress policy "${name}":`, error)
throw error
}
- // A concurrent actor may already have removed the policy. The safety
- // invariant still holds, but this pass did not revoke it and must not
- // inflate the authoritative safety-pass `revoked` counter.
}
}
}
@@ -2362,19 +1282,10 @@ export class NetworkPolicyReconciler {
resolvedEgressIPs: McpServerResolvedEgressIP[],
status: 'True' | 'False' | 'Unknown',
reason: string,
- message: string,
- isCurrent: () => boolean = () => true
+ message: string
): Promise<void> {
- if (!isCurrent()) return
let currentStatus: McpServerCrdStatus = {}
let hasStatusObject = false
- let currentMetadata:
- | {
- uid?: string
- generation?: number
- resourceVersion?: string
- }
- | undefined
try {
const current = (await this.customApi.getNamespacedCustomObjectStatus({
@@ -2383,15 +1294,7 @@ export class NetworkPolicyReconciler {
namespace: server.namespace,
plural: PLURAL_MCPSERVERS,
name: server.name,
- })) as {
- metadata?: {
- uid?: string
- generation?: number
- resourceVersion?: string
- }
- status?: McpServerCrdStatus
- }
- currentMetadata = current.metadata
+ })) as { status?: McpServerCrdStatus }
hasStatusObject = typeof current.status === 'object' && current.status !== null
currentStatus = current.status ?? {}
} catch (error) {
@@ -2407,20 +1310,6 @@ export class NetworkPolicyReconciler {
)
throw error
}
- if (!isCurrent()) return
-
- if (
- (server.uid !== undefined && currentMetadata?.uid !== server.uid) ||
- (server.generation !== undefined && currentMetadata?.generation !== server.generation)
- ) {
- throw new Error(
- `Refusing to write external egress status for stale McpServer "${server.name}": ` +
- `expected uid=${String(server.uid)} generation=${String(server.generation)}, ` +
- `read uid=${String(currentMetadata?.uid)} generation=${String(
- currentMetadata?.generation
- )}`
- )
- }
const nextConditions = this.mergeStatusCondition(currentStatus.conditions ?? [], {
type: 'ExternalEgressReady',
@@ -2445,36 +1334,14 @@ export class NetworkPolicyReconciler {
return
}
- const identityFence: JsonPatchOperation[] = []
- const expectedUid = server.uid ?? currentMetadata?.uid
- if (expectedUid !== undefined) {
- identityFence.push({ op: 'test', path: '/metadata/uid', value: expectedUid })
- }
- const expectedGeneration = server.generation ?? currentMetadata?.generation
- if (expectedGeneration !== undefined) {
- identityFence.push({
- op: 'test',
- path: '/metadata/generation',
- value: expectedGeneration,
- })
- }
- if (currentMetadata?.resourceVersion !== undefined) {
- identityFence.push({
- op: 'test',
- path: '/metadata/resourceVersion',
- value: currentMetadata.resourceVersion,
- })
- }
- const statusMutation: JsonPatchOperation[] = hasStatusObject
+ const statusPatch: JsonPatchOperation[] = hasStatusObject
? [
{ op: 'add', path: '/status/resolvedEgressIPs', value: stableResolvedEgressIPs },
{ op: 'add', path: '/status/conditions', value: nextConditions },
]
: [{ op: 'add', path: '/status', value: nextStatus }]
- const statusPatch = [...identityFence, ...statusMutation]
try {
- if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObjectStatus({
group: GROUP,
version: VERSION,
@@ -2527,61 +1394,6 @@ export class NetworkPolicyReconciler {
}
}
- /** Broad, namespace-bounded inventory used only by the safety pass. */
- private async listNamespacePoliciesForSafety(namespace: string): Promise<k8s.V1NetworkPolicy[]> {
- try {
- const response = await this.networkingApi.listNamespacedNetworkPolicy({ namespace })
- return response.items || []
- } catch (error) {
- console.error(`[NetPol] Failed to list broad NetworkPolicy inventory in ${namespace}:`, error)
- throw error
- }
- }
-
- private mergeSafetyLaneInventory(
- selected: k8s.V1NetworkPolicy[],
- broad: k8s.V1NetworkPolicy[],
- lane: SafetyInventoryLane
- ): k8s.V1NetworkPolicy[] {
- const result = new Map<string, k8s.V1NetworkPolicy>()
- const keyFor = (policy: k8s.V1NetworkPolicy, index: number): string =>
- policy.metadata?.uid ?? policy.metadata?.name ?? `unnamed-${index}`
-
- selected.forEach((policy, index) => result.set(keyFor(policy, index), policy))
- broad.forEach((policy, index) => {
- const classification = classifySafetyInventoryPolicy(policy, lane)
- if (classification === 'ambiguous') {
- const name = policy.metadata?.name ?? '<unnamed>'
- throw new Error(
- `Ambiguous NetworkPolicy ownership for reserved HCC ${lane} policy "${name}"`
- )
- }
- if (classification === 'owned' || classification === 'repairable') {
- result.set(keyFor(policy, selected.length + index), policy)
- } else {
- // The broad LIST is later and authoritative for this object identity.
- // Do not retain a selector snapshot after the same object was relabelled
- // out of HCC ownership.
- result.delete(keyFor(policy, selected.length + index))
- }
- })
- return [...result.values()]
- }
-
- /** List all L2 context-allow egress policies in the mcp-host namespace. */
- private async listAllContextEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {
- try {
- const response = await this.networkingApi.listNamespacedNetworkPolicy({
- namespace: config.hostNamespace,
- labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE},${POLICY_TYPE_LABEL}=context-allow`,
- })
- return response.items || []
- } catch (error) {
- console.error('[NetPol] Failed to list all Context egress policies:', error)
- throw error
- }
- }
-
/** List all external-egress policies in the MCP server namespace. */
private async listAllExternalEgressPolicies(): Promise<k8s.V1NetworkPolicy[]> {
try {
@@ -2609,234 +1421,26 @@ export class NetworkPolicyReconciler {
}
/** Create or update a NetworkPolicy. */
- private async applyFixedPolicy(
- name: string,
- namespace: string,
- policy: k8s.V1NetworkPolicy,
- policyType: string
- ): Promise<void> {
- await applyNetworkPolicy(
- this.networkingApi,
- name,
- namespace,
- policy,
- '[NetPol]',
- undefined,
- existing => {
- if (!hasExpectedPolicyOwnership(existing, policyType)) {
- throw new Error(
- `NetworkPolicy "${name}" has conflicting ownership for the ${policyType} lane`
- )
- }
- }
- )
+ private async applyPolicy(name: string, policy: k8s.V1NetworkPolicy): Promise<void> {
+ await applyNetworkPolicy(this.networkingApi, name, config.namespace, policy, '[NetPol]')
}
- /** Create or update a Context ingress NetworkPolicy. */
- private async applyContextIngressPolicy(
- name: string,
- policy: k8s.V1NetworkPolicy,
- isCurrent?: () => boolean
- ): Promise<void> {
- await this.applyOwnedPolicy(name, config.namespace, policy, 'context-ingress', isCurrent)
- }
-
- private async applyOwnedPolicy(
- name: string,
- namespace: string,
- policy: k8s.V1NetworkPolicy,
- lane: SafetyInventoryLane,
- isCurrent?: () => boolean
- ): Promise<void> {
- await applyNetworkPolicy(
- this.networkingApi,
- name,
- namespace,
- policy,
- '[NetPol]',
- isCurrent,
- existing => this.assertPolicyOwnership(name, existing, policy, lane)
- )
- }
-
- private assertPolicyOwnership(
- name: string,
- existing: k8s.V1NetworkPolicy,
- desired: k8s.V1NetworkPolicy,
- lane: SafetyInventoryLane
- ): void {
- const classification = classifySafetyInventoryPolicy(existing, lane)
- const existingLabels = existing.metadata?.labels ?? {}
- const desiredLabels = desired.metadata?.labels ?? {}
- const ownerLabels =
- lane === 'external-egress' ? [MCPSERVER_LABEL] : [CONTEXT_LABEL, MCPSERVER_LABEL]
- if (
- (classification !== 'owned' && classification !== 'repairable') ||
- ownerLabels.some(label => existingLabels[label] !== desiredLabels[label])
- ) {
- throw new Error(`NetworkPolicy "${name}" has conflicting ownership for the ${lane} lane`)
- }
- }
-
- private safetyPolicyIdentity(policy: k8s.V1NetworkPolicy): {
- name: string
- uid: string
- resourceVersion: string
- } {
- const name = policy.metadata?.name
- const uid = policy.metadata?.uid
- const resourceVersion = policy.metadata?.resourceVersion
- if (!name || !uid || !resourceVersion) {
- throw new Error(
- `NetworkPolicy safety mutation requires name, uid, and resourceVersion for "${name ?? '<unnamed>'}"`
- )
- }
- return { name, uid, resourceVersion }
- }
-
- /**
- * Delete exactly the policy version classified by the authoritative safety
- * inventory. Kubernetes rejects the preconditions if the name was recreated
- * or changed after LIST, so HCC cannot delete a new or foreign object.
- */
- private async deleteSafetyPolicySnapshot(
- namespace: string,
- policy: k8s.V1NetworkPolicy,
- isCurrent?: () => boolean,
- onDeleted?: () => void,
- lostFenceOutcome: 'throw' | 'report' = 'throw'
- ): Promise<boolean> {
- if (isCurrent && !isCurrent()) return false
- const { name, uid, resourceVersion } = this.safetyPolicyIdentity(policy)
+ /** Delete a NetworkPolicy in mcp-server namespace. */
+ private async deletePolicy(name: string): Promise<void> {
try {
await this.networkingApi.deleteNamespacedNetworkPolicy({
name,
- namespace,
- body: { preconditions: { uid, resourceVersion } },
+ namespace: config.namespace,
})
- console.log(`[NetPol] Deleted safety-inventory policy "${name}" in ${namespace}`)
- onDeleted?.()
- return true
+ console.log(`[NetPol] Deleted policy "${name}"`)
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[NetPol] Safety-inventory policy "${name}" already gone`)
- return true
- }
- if (getErrorCode(error) === 409 && lostFenceOutcome === 'report') {
- // Kubernetes rejected the uid/resourceVersion preconditions, so the
- // object live under this name is not the one the authoritative
- // inventory classified. Nothing was revoked, and nothing may be
- // adopted. Callers that opt in act on this outcome by declining to
- // certify, which is the expected racy result rather than a failure.
- // Callers that cannot act on it keep the loud default.
- console.warn(
- `[NetPol] Lost the delete fence on safety-inventory policy "${name}" in ${namespace}; it changed identity after the authoritative inventory and was not revoked`
- )
- return false
- }
- console.error(
- `[NetPol] Failed identity-bound delete of safety-inventory policy "${name}" in ${namespace}:`,
- error
- )
- throw error
- }
- }
-
- /**
- * Replace only the exact live object classified by the safety inventory.
- * The fresh read closes ownership drift before mutation; resourceVersion on
- * the replace body closes the remaining read-to-write race.
- */
- private async replaceSafetyPolicySnapshot(
- namespace: string,
- snapshot: k8s.V1NetworkPolicy,
- desired: k8s.V1NetworkPolicy,
- lane: SafetyInventoryLane,
- isCurrent: () => boolean
- ): Promise<boolean> {
- if (!isCurrent()) return false
- const expected = this.safetyPolicyIdentity(snapshot)
- const live = await this.networkingApi.readNamespacedNetworkPolicy({
- name: expected.name,
- namespace,
- })
- if (!isCurrent()) return false
- const actual = this.safetyPolicyIdentity(live)
- if (actual.uid !== expected.uid || actual.resourceVersion !== expected.resourceVersion) {
- throw new Error(
- `NetworkPolicy "${expected.name}" changed identity or ownership after the authoritative safety inventory`
- )
- }
- this.assertPolicyOwnership(expected.name, live, desired, lane)
- await this.networkingApi.replaceNamespacedNetworkPolicy({
- name: expected.name,
- namespace,
- body: {
- ...desired,
- metadata: {
- ...desired.metadata,
- name: expected.name,
- namespace,
- resourceVersion: actual.resourceVersion,
- },
- },
- })
- return isCurrent()
- }
-
- private async createSafetyPolicy(
- namespace: string,
- desired: k8s.V1NetworkPolicy,
- lane: SafetyInventoryLane,
- isCurrent: () => boolean
- ): Promise<boolean> {
- if (!isCurrent()) return false
- let created: k8s.V1NetworkPolicy
- try {
- created = await this.networkingApi.createNamespacedNetworkPolicy({
- namespace,
- body: desired,
- })
- } catch (error: unknown) {
- if (getErrorCode(error) !== 409) throw error
- // This is the only creation route that does not go through
- // applyNetworkPolicy, so a same-name object used to abort the whole
- // safety pass and hold readiness down. Converge it exactly like every
- // other lane does: adopt it only under the ownership guard, which still
- // refuses a foreign or ambiguous object.
- const name = desired.metadata?.name
- if (!name) {
- throw new Error(`Cannot converge a conflicting ${lane} policy without a desired name`)
+ console.log(`[NetPol] Policy "${name}" already gone`)
+ } else {
+ console.error(`[NetPol] Failed to delete policy "${name}":`, error)
+ throw error
}
- console.log(
- `[NetPol] Safety policy "${name}" in ${namespace} already exists — converging it to the desired spec`
- )
- // Deliberately not delegating to applyOwnedPolicy here: that helper owns
- // the create-then-conflict flow, so calling it from inside our own
- // create's catch would issue a second create. Delegating from the top is
- // not an option either — the desired-state fence below needs the object
- // this request created, which applyNetworkPolicy does not return. The
- // wiring below is kept identical to it on purpose.
- await replaceWithConflictRetry<k8s.V1NetworkPolicy>({
- description: `policy "${name}" in ${namespace}`,
- logPrefix: '[NetPol]',
- body: desired,
- read: () => this.networkingApi.readNamespacedNetworkPolicy({ name, namespace }),
- replace: body =>
- this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
- mutationAllowed: isCurrent,
- validateExisting: existing => this.assertPolicyOwnership(name, existing, desired, lane),
- })
- return isCurrent()
}
- if (isCurrent()) return true
-
- // The mutation crossed its desired-state fence. Remove exactly the object
- // this request created, even though the old reconciliation is no longer
- // authoritative. The UID/resourceVersion preconditions ensure a
- // replacement or foreign policy with the same name is never deleted.
- await this.deleteSafetyPolicySnapshot(namespace, created)
- return false
}
/** List L2 egress counterpart policies in mcp-host namespace for a given context. */
@@ -2853,15 +1457,37 @@ export class NetworkPolicyReconciler {
}
}
+ /** Delete a NetworkPolicy in mcp-host namespace (L2 egress counterpart). */
+ private async deleteEgressPolicy(name: string): Promise<void> {
+ try {
+ await this.networkingApi.deleteNamespacedNetworkPolicy({
+ name,
+ namespace: config.hostNamespace,
+ })
+ console.log(`[NetPol] Deleted L2 egress policy "${name}"`)
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) {
+ console.log(`[NetPol] L2 egress policy "${name}" already gone`)
+ } else {
+ console.error(`[NetPol] Failed to delete L2 egress policy "${name}":`, error)
+ throw error
+ }
+ }
+ }
+
// ─── RPC-Proxy Egress (L2) ──────────────────────────────────────────
- private buildRpcProxyEgressPolicy(
+ /**
+ * Create or update an egress policy in the rpc-proxy namespace allowing
+ * rpc-proxy pods to reach a specific MCP server on a given port.
+ */
+ private async ensureRpcProxyEgress(
contextId: string,
serverName: string,
port: number
- ): k8s.V1NetworkPolicy {
+ ): Promise<void> {
const name = `rpc-egress-${contextId}-${serverName}`
- return {
+ const policy: k8s.V1NetworkPolicy = {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
@@ -2892,18 +1518,16 @@ export class NetworkPolicyReconciler {
],
},
}
+ await applyNetworkPolicy(this.networkingApi, name, config.rpcProxyNamespace, policy, '[NetPol]')
}
/** Delete all rpc-proxy egress policies for a given context. */
- private async deleteRpcProxyPoliciesForContext(
- contextId: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ private async deleteRpcProxyPoliciesForContext(contextId: string): Promise<void> {
try {
const policies = await this.listRpcProxyEgressPoliciesForContext(contextId)
for (const policy of policies) {
- if (deleteAllowed && !(await deleteAllowed())) return
- await this.deleteSafetyPolicySnapshot(config.rpcProxyNamespace, policy)
+ const name = policy.metadata?.name || ''
+ await this.deletePolicyInNamespace(config.rpcProxyNamespace, name)
}
} catch (error) {
console.error(
@@ -2947,19 +1571,34 @@ export class NetworkPolicyReconciler {
}
}
+ /** Delete a NetworkPolicy in an arbitrary namespace. */
+ private async deletePolicyInNamespace(namespace: string, name: string): Promise<void> {
+ try {
+ await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
+ console.log(`[NetPol] Deleted rpc-proxy egress policy "${name}"`)
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) {
+ console.log(`[NetPol] rpc-proxy egress policy "${name}" already gone`)
+ } else {
+ console.error(`[NetPol] Failed to delete policy "${name}" in ${namespace}:`, error)
+ throw error
+ }
+ }
+ }
+
/** Delete a formerly static NetworkPolicy that is now generated dynamically. */
private async deleteLegacyStaticPolicy(namespace: string, name: string): Promise<void> {
- let existing: k8s.V1NetworkPolicy
try {
- existing = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
+ await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace })
+ console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)
} catch (error: unknown) {
- if (getErrorCode(error) === 404) return
- throw error
- }
- if (existing.metadata?.labels?.[MANAGED_BY_LABEL] !== MANAGED_BY_VALUE) {
- throw new Error(`NetworkPolicy "${name}" has conflicting ownership for legacy cleanup`)
+ if (getErrorCode(error) !== 404) {
+ console.error(
+ `[NetPol] Failed to delete legacy static policy "${name}" in "${namespace}":`,
+ error
+ )
+ throw error
+ }
}
- await this.deleteSafetyPolicySnapshot(namespace, existing)
- console.log(`[NetPol] Deleted legacy static policy "${name}" in "${namespace}"`)
}
}
diff --git a/host-context-controller/src/readinessGate.test.ts b/host-context-controller/src/readinessGate.test.ts
deleted file mode 100644
index a9b522ab0..000000000
--- a/host-context-controller/src/readinessGate.test.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
-
-describe('resolveProviderAuthoritativeFn (dev-mode readiness gate — B1)', () => {
- it('dev provider (no watcher) is unconditionally authoritative', () => {
- // Regression for R3-B1: main.ts left this undefined for DevMcpServerProvider,
- // so the ContextMapperServer fail-closed default (() => false) pinned /ready
- // and every /api/v1/* endpoint at 503 forever in dev mode. There is no
- // NetworkPolicy inventory to certify in dev, so authority must be granted.
- expect(resolveProviderAuthoritativeFn(null)()).toBe(true)
- })
-
- it('production watcher delegates to the live inventory certification', () => {
- let asked = false
- const watcher = {
- isReadinessInventoryAuthoritative: () => {
- asked = true
- return false
- },
- }
- // Delegates (returns the watcher's verdict) rather than a constant.
- expect(resolveProviderAuthoritativeFn(watcher)()).toBe(false)
- expect(asked).toBe(true)
- watcher.isReadinessInventoryAuthoritative = () => true
- expect(resolveProviderAuthoritativeFn(watcher)()).toBe(true)
- })
-})
-
-describe('resolveHostAuthoritativeFn (dev-mode desktop gate — R2-M1)', () => {
- it('dev provider (no watcher) is unconditionally authoritative', () => {
- // Regression for R2-M1: main.ts left this undefined for DevMcpServerProvider,
- // so the ContextMapperServer fail-closed default (() => false) pinned every
- // /api/v1/desktop/* response at 503 after setReady(true) — where the base
- // answered 200 {status:'inactive'}. There is no Host inventory to certify in
- // dev, so authority must mirror the provider gate and be granted.
- expect(resolveHostAuthoritativeFn(null)()).toBe(true)
- })
-
- it('production watcher delegates to the live Host inventory certification', () => {
- let asked = false
- const watcher = {
- isHostInventoryAuthoritative: () => {
- asked = true
- return false
- },
- }
- // Delegates (returns the watcher's verdict) rather than a constant.
- expect(resolveHostAuthoritativeFn(watcher)()).toBe(false)
- expect(asked).toBe(true)
- watcher.isHostInventoryAuthoritative = () => true
- expect(resolveHostAuthoritativeFn(watcher)()).toBe(true)
- })
-})
diff --git a/host-context-controller/src/readinessGate.ts b/host-context-controller/src/readinessGate.ts
deleted file mode 100644
index 88ff77ee6..000000000
--- a/host-context-controller/src/readinessGate.ts
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Resolve the readiness-authority gate for the server from the provider.
- *
- * Production (McpServerWatcher present): readiness follows the live inventory
- * certification — /ready is 200 only once stale allows are revoked.
- *
- * Dev (DevMcpServerProvider — no watcher): there is NO NetworkPolicy inventory
- * to certify, so authority is unconditional. The ContextMapperServer default
- * for a missing gate is fail-closed (`() => false`), which is correct for a
- * caller that forgets to wire one — but in dev it would pin /ready (and every
- * data endpoint) at 503 forever. Dev must therefore pass an explicit
- * always-authoritative gate; this function is that single, named decision so
- * the affirmative literal never gets copy-pasted into a production path.
- */
-export function resolveProviderAuthoritativeFn(
- watcher: { isReadinessInventoryAuthoritative(): boolean } | null
-): () => boolean {
- return watcher ? () => watcher.isReadinessInventoryAuthoritative() : () => true
-}
-
-/**
- * Resolve the Host-inventory-authority gate the desktop route uses.
- *
- * Production (McpServerWatcher present): desktop status follows the live Host
- * inventory certification — a degraded Host lane 503s desktop rather than
- * answering a stale 200 'inactive'.
- *
- * Dev (DevMcpServerProvider — no watcher): there is NO Host inventory to
- * certify, exactly as there is no NetworkPolicy inventory for the provider
- * gate. The ContextMapperServer default for a missing gate is fail-closed
- * (`() => false`), which is correct for a caller that forgets to wire one — but
- * in dev it would pin every /api/v1/desktop/* response at 503 forever (R2-M1).
- * Dev must therefore mirror the provider gate and pass an explicit
- * always-authoritative gate; this function is that single, named decision so
- * the affirmative literal never gets copy-pasted into a production path.
- */
-export function resolveHostAuthoritativeFn(
- watcher: { isHostInventoryAuthoritative(): boolean } | null
-): () => boolean {
- return watcher ? () => watcher.isHostInventoryAuthoritative() : () => true
-}
diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.ts
deleted file mode 100644
index 4bd14f5e6..000000000
--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts
+++ /dev/null
@@ -1,513 +0,0 @@
-/**
- * Crosses the whole readiness chain with a real NetworkPolicyReconciler.
- *
- * Every other test of this behaviour mocks one side. `k8sClient.test.ts` mocks
- * `./networkPolicyReconciler`, so it can prove the readiness predicate *reads*
- * the certification getter but not that the reconciler ever *degrades* it.
- * `networkPolicyReconciler.test.ts` drives the real fence but never reaches an
- * HTTP response. Each half is verified by mutation; the seam between them was
- * not, and the seam is where the guarantee actually lives.
- *
- * Here `@kubernetes/client-node` is the only mock. The reconciler, the watcher
- * predicate, and the HTTP server are the production classes, so the assertion
- * is end-to-end: a delete that loses its uid/resourceVersion precondition
- * during the additive phase — the 409 a second writer produces, and every
- * rollout opens a two-replica window for one — must turn /ready from 200 into
- * 503, and a later clean pass must turn it back.
- */
-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import * as http from 'http'
-import { McpServerWatcher } from './k8sClient'
-import { ContextMapperServer } from './server'
-
-const mocks = vi.hoisted(() => ({
- listNamespacedNetworkPolicy: vi.fn(),
- deleteNamespacedNetworkPolicy: vi.fn(),
- readNamespacedNetworkPolicy: vi.fn(),
- createNamespacedNetworkPolicy: vi.fn(),
- replaceNamespacedNetworkPolicy: vi.fn(),
- listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
- getNamespacedCustomObject: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
- getNamespacedCustomObjectStatus: vi
- .fn()
- .mockResolvedValue({ metadata: { generation: 1, resourceVersion: 'rv-1' }, status: {} }),
- patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
- watch: vi.fn().mockResolvedValue({ abort: vi.fn() }),
-}))
-
-vi.mock('./config', () => ({
- config: {
- devMode: false,
- namespace: 'mcp-server',
- hostNamespace: 'mcp-host',
- rpcProxyNamespace: 'rpc-proxy',
- channelsNamespace: 'channels',
- port: 0,
- runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],
- hostK8sRequestTimeoutMs: 30_000,
- hostResyncIntervalSec: 0,
- externalEgressResyncIntervalSec: 0,
- controlApiBaseUrl: 'http://control-api.test:8090',
- governedTracingEnabled: false,
- },
-}))
-
-vi.mock('@kubernetes/client-node', () => {
- class CustomObjectsApi {}
- class CoreV1Api {}
- class NetworkingV1Api {}
- class PolicyV1Api {}
- class AppsV1Api {}
- class BatchV1Api {}
- class RbacAuthorizationV1Api {}
- class Watch {
- constructor(_kc: unknown) {}
- watch = mocks.watch
- }
- class KubeConfig {
- loadFromDefault(): void {}
- loadFromOptions(_o: unknown): void {}
- makeApiClient(api: unknown): unknown {
- if (api === NetworkingV1Api) {
- return {
- listNamespacedNetworkPolicy: mocks.listNamespacedNetworkPolicy,
- deleteNamespacedNetworkPolicy: mocks.deleteNamespacedNetworkPolicy,
- readNamespacedNetworkPolicy: mocks.readNamespacedNetworkPolicy,
- createNamespacedNetworkPolicy: mocks.createNamespacedNetworkPolicy,
- replaceNamespacedNetworkPolicy: mocks.replaceNamespacedNetworkPolicy,
- }
- }
- if (api === CustomObjectsApi) {
- return {
- listNamespacedCustomObject: mocks.listNamespacedCustomObject,
- getNamespacedCustomObject: mocks.getNamespacedCustomObject,
- getNamespacedCustomObjectStatus: mocks.getNamespacedCustomObjectStatus,
- patchNamespacedCustomObjectStatus: mocks.patchNamespacedCustomObjectStatus,
- }
- }
- return new Proxy({}, { get: () => vi.fn().mockResolvedValue({ items: [] }) })
- }
- }
- return {
- CustomObjectsApi,
- CoreV1Api,
- NetworkingV1Api,
- PolicyV1Api,
- AppsV1Api,
- BatchV1Api,
- RbacAuthorizationV1Api,
- Watch,
- KubeConfig,
- }
-})
-
-const MANAGED = 'clerum.io/managed-by'
-const POLICY_TYPE = 'clerum.io/policy-type'
-const CONTEXT_LABEL = 'clerum.io/context'
-const SERVER_LABEL = 'clerum.io/mcpserver'
-
-/** A stale context-allow: HCC-owned, for a server the Context no longer lists. */
-const stalePolicy = {
- metadata: {
- name: 'ctx-alpha-ghost',
- namespace: 'mcp-server',
- uid: 'ghost-uid',
- resourceVersion: '7',
- labels: {
- [MANAGED]: 'host-context-controller',
- [POLICY_TYPE]: 'context-allow',
- [CONTEXT_LABEL]: 'alpha',
- [SERVER_LABEL]: 'ghost',
- },
- },
- spec: { podSelector: {}, policyTypes: ['Ingress'] },
-}
-
-const alphaContext = {
- name: 'alpha',
- namespace: 'mcp-server',
- spec: { contextId: 'alpha', mcpServers: [] },
-}
-
-async function readyStatus(server: InstanceType<typeof ContextMapperServer>): Promise<number> {
- const listener = (server as unknown as { server: http.Server | null }).server
- const address = listener?.address()
- if (!address || typeof address === 'string') throw new Error('readiness listener never bound')
- return new Promise((resolve, reject) => {
- http
- .get({ host: '127.0.0.1', port: address.port, path: '/ready' }, res => {
- res.resume()
- res.on('end', () => resolve(res.statusCode ?? 0))
- })
- .on('error', reject)
- })
-}
-
-/** Aligns every revocation counter so only the fence can gate readiness. */
-function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>): void {
- const w = watcher as unknown as Record<string, unknown>
- w.mcpServerCacheSynced = true
- w.contextCacheSynced = true
- w.hostCacheSynced = true
- w.networkPolicyRevocationContextGeneration = w.contextWatchGeneration
- w.networkPolicyRevocationServerGeneration = w.mcpWatchGeneration
- w.networkPolicyRevocationContextRevision = w.contextDesiredRevision
- w.networkPolicyRevocationServerRevision = w.mcpServerDesiredRevision
-}
-
-describe('readiness withdrawal on a real lost delete fence', () => {
- let watcher: InstanceType<typeof McpServerWatcher>
- let server: InstanceType<typeof ContextMapperServer>
-
- beforeEach(async () => {
- vi.clearAllMocks()
- mocks.listNamespacedCustomObject.mockResolvedValue({ items: [] })
- mocks.getNamespacedCustomObject.mockRejectedValue(
- Object.assign(new Error('not found'), { code: 404 })
- )
- mocks.createNamespacedNetworkPolicy.mockResolvedValue({})
- mocks.replaceNamespacedNetworkPolicy.mockResolvedValue({})
- mocks.readNamespacedNetworkPolicy.mockResolvedValue(stalePolicy)
- watcher = new McpServerWatcher()
- server = new ContextMapperServer(watcher, 0, undefined, undefined, () =>
- watcher.isReadinessInventoryAuthoritative()
- )
- await server.start()
- server.setReady(true)
- })
-
- afterEach(async () => {
- await server.stop()
- await watcher.stop()
- })
-
- it('turns /ready from 200 to 503 when the additive phase loses a delete fence, and back on a clean pass', async () => {
- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
- alignRevocationCounters(watcher)
-
- // Baseline: nothing stale anywhere, so the pass certifies and the gate opens.
- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await reconciler.fullReconcile([alphaContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- alignRevocationCounters(watcher)
- expect(await readyStatus(server)).toBe(200)
-
- // The stale allow only becomes visible after the authoritative phase has
- // certified, so the safety phase is clean by construction and the additive
- // re-LIST is what finds it. Its delete then loses the uid/resourceVersion
- // precondition — a 409, exactly what a concurrent writer produces.
- let certified = false
- mocks.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace }: { namespace?: string }) => ({
- items: certified && namespace === 'mcp-server' ? [stalePolicy] : [],
- })
- )
- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('the object has been modified'), { code: 409 })
- )
-
- await expect(
- reconciler.fullReconcile([alphaContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {
- certified = true
- },
- })
- ).rejects.toThrow()
-
- // Nothing moved: no watch generation bumped, no desired revision changed.
- // Only the fence knows this pass left an allow it classified as stale.
- alignRevocationCounters(watcher)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- expect(await readyStatus(server)).toBe(503)
-
- // The retry lands: the allow is gone and the pass certifies again.
- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await reconciler.fullReconcile([alphaContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- alignRevocationCounters(watcher)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
- expect(await readyStatus(server)).toBe(200)
- })
-
- it('withholds readiness when the authoritative pass loses a delete fence under a provided safety snapshot', async () => {
- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
- alignRevocationCounters(watcher)
-
- // Baseline: a clean authoritative pass certifies and the gate opens.
- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await reconciler.fullReconcile([alphaContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- alignRevocationCounters(watcher)
- expect(await readyStatus(server)).toBe(200)
-
- // Unlike the additive-phase test above, the stale allow is visible to the
- // authoritative LISTs from the very start of the pass, so fullReconcile
- // classifies it into the safety snapshot it provides to the revocation
- // lane. Its identity-bound delete then loses the uid/resourceVersion
- // precondition (409). The external-egress selector stays empty so the
- // fence is lost in the provided-snapshot lane and nowhere else.
- mocks.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => ({
- items:
- namespace === 'mcp-server' && !(labelSelector ?? '').includes('external-egress')
- ? [stalePolicy]
- : [],
- })
- )
- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('the object has been modified'), { code: 409 })
- )
-
- // The guarantee under test is that the fence degrades at the point of
- // loss, while the condemned pass is still unwinding — not only when the
- // pass finally reports. Sample it through the authority callback the pass
- // keeps polling: every sample before the lost delete must be certified,
- // and every later sample must already be withdrawn.
- let certifications = 0
- const fenceSeenDuringPass: boolean[] = []
- await expect(
- reconciler.fullReconcile([alphaContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => {
- fenceSeenDuringPass.push(reconciler.hasCertifiedSafetyInventory())
- return true
- },
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {
- certifications += 1
- },
- })
- ).rejects.toThrow()
-
- expect(certifications).toBe(0)
- expect(fenceSeenDuringPass.length).toBeGreaterThan(1)
- expect(fenceSeenDuringPass[0]).toBe(true)
- expect(fenceSeenDuringPass[fenceSeenDuringPass.length - 1]).toBe(false)
-
- alignRevocationCounters(watcher)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- expect(await readyStatus(server)).toBe(503)
-
- // A clean authoritative pass re-certifies and reopens the gate.
- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await reconciler.fullReconcile([alphaContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- alignRevocationCounters(watcher)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(true)
- expect(await readyStatus(server)).toBe(200)
- })
-
- it('short-circuits a scoped delta at the certificate source while the last authoritative pass left the inventory uncertified', async () => {
- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
- const w = watcher as unknown as Record<string, any>
-
- // Register the real Context watch so the production watch handler — the
- // scoped-delta certification lane — can be driven through its callback.
- await w.startContextWatch('ctx-rv-1')
- const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>
- String(path).endsWith('/namespaces/mcp-server/contexts')
- )
- if (!contextWatchCall) throw new Error('Context watch was never registered')
- const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>
-
- // Seed the prior Context so the MODIFIED event below is a same-identity
- // delta: same uid, same contextId, new generation.
- const priorContext = {
- name: 'alpha',
- namespace: 'mcp-server',
- uid: 'ctx-alpha-uid',
- generation: 1,
- spec: { contextId: 'alpha', mcpServers: [] },
- }
- w.contexts.set('alpha', priorContext)
- alignRevocationCounters(watcher)
-
- // Baseline clean authoritative pass: certified, gate open.
- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await reconciler.fullReconcile([priorContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- alignRevocationCounters(watcher)
- expect(await readyStatus(server)).toBe(200)
-
- // The authoritative pass loses its delete fence under the provided
- // snapshot, exactly as in the test above: the broad safety LISTs see the
- // stale allow and its delete keeps 409ing. Every context-scoped LIST
- // stays empty so the scoped delta fired below completes its own
- // revocation cleanly — and because the 409 never clears, the convergence
- // pass the watch event kicks off in the background can never certify
- // either.
- mocks.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- const selector = labelSelector ?? ''
- const scoped =
- selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')
- return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }
- }
- )
- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('the object has been modified'), { code: 409 })
- )
- await expect(
- reconciler.fullReconcile([priorContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- ).rejects.toThrow()
- alignRevocationCounters(watcher)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- expect(await readyStatus(server)).toBe(503)
-
- // Fire a same-identity MODIFIED delta. Its scoped revocation completes
- // (its label-scoped LISTs are clean) and it carries a delta certificate,
- // so the only thing standing between it and
- // recordNetworkPolicySafetyCertificate is the uncertified fence left by
- // the authoritative pass.
- const warnSpy = vi.spyOn(console, 'warn')
- try {
- await fireContextEvent('MODIFIED', {
- metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },
- spec: { contextId: 'alpha', mcpServers: [] },
- })
-
- // M1: the fence was lost before this delta, so
- // currentNetworkPolicySafetyCertificate() returns null at the capture and no
- // delta certificate is ever minted — the delta short-circuits at the source
- // and never reaches the per-delta certification branch. No warn is emitted;
- // nothing is recorded; readiness stays withdrawn via the independent
- // readiness fence even after every counter is realigned.
- expect(
- warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))
- ).toBe(false)
- expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- alignRevocationCounters(watcher)
- expect(await readyStatus(server)).toBe(503)
- } finally {
- warnSpy.mockRestore()
- }
- })
-
- it('declines to certify a scoped delta whose certificate outlived a fence lost mid-flight, and warns', async () => {
- // Coverage preservation for the per-delta fence check (k8sClient.ts :3801):
- // the delta certificate is minted while the fence is still certified, then a
- // CONCURRENT authoritative pass loses its delete fence mid-flight. M1's
- // source short-circuit cannot catch this — only the per-delta check can. This
- // test is green with AND without M1; it guards the TOCTOU window, not M1.
- const reconciler = (watcher as unknown as { netPolReconciler: any }).netPolReconciler
- const w = watcher as unknown as Record<string, any>
-
- await w.startContextWatch('ctx-rv-2')
- const contextWatchCall = mocks.watch.mock.calls.find(([path]) =>
- String(path).endsWith('/namespaces/mcp-server/contexts')
- )
- if (!contextWatchCall) throw new Error('Context watch was never registered')
- const fireContextEvent = contextWatchCall[2] as (type: string, apiObj: unknown) => Promise<void>
-
- const priorContext = {
- name: 'alpha',
- namespace: 'mcp-server',
- uid: 'ctx-alpha-uid',
- generation: 1,
- spec: { contextId: 'alpha', mcpServers: [] },
- }
- w.contexts.set('alpha', priorContext)
- alignRevocationCounters(watcher)
-
- // Baseline clean authoritative pass: certified, gate open.
- mocks.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
- mocks.deleteNamespacedNetworkPolicy.mockResolvedValue({})
- await reconciler.fullReconcile([priorContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- alignRevocationCounters(watcher)
- expect(await readyStatus(server)).toBe(200)
-
- // The delta certificate is captured now, while the fence is still certified.
- // The FIRST scoped LIST the delta issues runs a concurrent authoritative pass
- // that loses its own delete fence — degrading the fence AFTER the delta minted
- // its certificate but BEFORE the delta reaches its certification branch.
- let fenceBrokenMidFlight = false
- mocks.listNamespacedNetworkPolicy.mockImplementation(
- async ({ namespace, labelSelector }: { namespace?: string; labelSelector?: string }) => {
- const selector = labelSelector ?? ''
- const scoped =
- selector.includes(`${CONTEXT_LABEL}=`) || selector.includes('external-egress')
- if (scoped && !fenceBrokenMidFlight) {
- fenceBrokenMidFlight = true
- mocks.deleteNamespacedNetworkPolicy.mockRejectedValue(
- Object.assign(new Error('the object has been modified'), { code: 409 })
- )
- await reconciler
- .fullReconcile([priorContext], [], {
- ensureDefaults: false,
- contextInventoryAuthoritative: () => true,
- serverInventoryAuthoritative: () => true,
- onAuthoritativeRevocationComplete: () => {},
- })
- .then(
- () => {
- throw new Error('interleaved authoritative pass should have lost its fence')
- },
- () => {}
- )
- }
- return { items: namespace === 'mcp-server' && !scoped ? [stalePolicy] : [] }
- }
- )
- const warnSpy = vi.spyOn(console, 'warn')
- try {
- await fireContextEvent('MODIFIED', {
- metadata: { name: 'alpha', namespace: 'mcp-server', uid: 'ctx-alpha-uid', generation: 2 },
- spec: { contextId: 'alpha', mcpServers: [] },
- })
-
- // The interleave actually ran (no vacuous pass) and the per-delta check
- // withheld certification: it warned, recorded nothing, readiness stays 503.
- expect(fenceBrokenMidFlight).toBe(true)
- expect(
- warnSpy.mock.calls.some(call => String(call[0]).includes('Not certifying the scoped delta'))
- ).toBe(true)
- expect(w.networkPolicyRevocationContextRevision).not.toBe(w.contextDesiredRevision)
- expect(reconciler.hasCertifiedSafetyInventory()).toBe(false)
- alignRevocationCounters(watcher)
- expect(await readyStatus(server)).toBe(503)
- } finally {
- warnSpy.mockRestore()
- }
- })
-})
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index 26a7f5ab0..92e74d6e8 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import {
asAppsApi,
@@ -14,17 +14,6 @@ import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './const
import { McpServerReconciler } from './reconciler'
import { McpServerCRD } from './types'
-function deferred<T = void>(): {
- promise: Promise<T>
- resolve: (value: T | PromiseLike<T>) => void
-} {
- let resolve!: (value: T | PromiseLike<T>) => void
- const promise = new Promise<T>(res => {
- resolve = res
- })
- return { promise, resolve }
-}
-
/**
* Tests for the managed:false guard in the reconciler (Risk 1.7).
*
@@ -41,7 +30,6 @@ vi.mock('./config', () => ({
controlPlaneNamespace: 'control-plane',
hostNamespace: 'mcp-host',
rpcProxyNamespace: 'rpc-proxy',
- egressProxyImage: 'clerum/nginx-egress-proxy:test',
devMcpServers: [],
devContexts: [],
devAuthTokens: new Map(),
@@ -76,7 +64,6 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -168,55 +155,6 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
})
- it('completes a remote proxy reconcile without rewriting the desired image', async () => {
- const server = makeServer({
- name: 'remote-api',
- image: 'vendor/original-image:1',
- remote: { baseUrl: 'https://api.example.com' },
- })
- const originalSpec = structuredClone(server.spec)
- let desiredRevisionIsCurrent = true
- customApi.patchNamespacedCustomObject.mockImplementation(
- ({ body }: { body?: { spec?: unknown } }) => {
- if (body?.spec !== undefined) {
- desiredRevisionIsCurrent = false
- }
- return Promise.resolve({})
- }
- )
-
- await reconciler.reconcile(server, {
- isCurrent: () => desiredRevisionIsCurrent,
- })
-
- expect(server.spec).toEqual(originalSpec)
- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(
- expect.objectContaining({ body: expect.objectContaining({ spec: expect.anything() }) }),
- expect.anything()
- )
- expect(coreApi.createNamespacedConfigMap).toHaveBeenCalledOnce()
- expect(coreApi.createNamespacedService).toHaveBeenCalledOnce()
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledOnce()
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith(
- expect.objectContaining({
- body: expect.objectContaining({
- spec: expect.objectContaining({
- template: expect.objectContaining({
- spec: expect.objectContaining({
- containers: [
- expect.objectContaining({
- name: 'egress-proxy',
- image: 'clerum/nginx-egress-proxy:test',
- }),
- ],
- }),
- }),
- }),
- }),
- })
- )
- })
-
it('should log WRC-owned ownership when managed: false', async () => {
const consoleSpy = vi.spyOn(console, 'log')
const server = makeServer({ name: 'workflow-recipes', managed: false })
@@ -285,7 +223,6 @@ describe('PR-B B1 — validateSecret result shape', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -470,12 +407,7 @@ describe('PR-B B1 — writeStatusCondition', () => {
const getPatchedConditions = () => {
const patch = customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0]
- const op = patch.body.find(
- (candidate: { op?: string; path?: string }) =>
- candidate.op === 'add' &&
- (candidate.path === '/status' || candidate.path === '/status/conditions')
- )
- if (!op) throw new Error('status condition add operation was not emitted')
+ const op = patch.body[0]
const conditions = op.path === '/status' ? op.value.conditions : op.value
return { patch, op, conditions }
}
@@ -483,7 +415,6 @@ describe('PR-B B1 — writeStatusCondition', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -578,216 +509,6 @@ describe('PR-B B1 — writeStatusCondition', () => {
expect(types).toContain('Ready')
})
- it('records the reconciled McpServer generation on status conditions', async () => {
- const server = {
- ...makeServer({ name: 'pg' }),
- generation: 17,
- }
-
- await reconciler.writeStatusCondition(server, {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'done',
- })
-
- const { conditions } = getPatchedConditions()
- expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(
- expect.objectContaining({
- status: 'True',
- observedGeneration: 17,
- })
- )
- })
-
- it('re-reads and preserves a concurrent condition after a resourceVersion conflict', async () => {
- const networkReady = {
- type: 'NetworkReady',
- status: 'True',
- reason: 'NetworkPoliciesApplied',
- message: 'network ready',
- lastTransitionTime: '2020-01-01T00:00:00.000Z',
- }
- const externalEgressReady = {
- type: 'ExternalEgressReady',
- status: 'True',
- reason: 'EgressPolicyApplied',
- message: 'external egress ready',
- lastTransitionTime: '2020-01-02T00:00:00.000Z',
- }
- customApi.getNamespacedCustomObjectStatus
- .mockResolvedValueOnce({
- metadata: { resourceVersion: '100' },
- status: { conditions: [networkReady] },
- })
- .mockResolvedValueOnce({
- metadata: { resourceVersion: '101' },
- status: { conditions: [networkReady, externalEgressReady] },
- })
- customApi.patchNamespacedCustomObjectStatus
- .mockRejectedValueOnce(Object.assign(new Error('resourceVersion conflict'), { code: 409 }))
- .mockResolvedValueOnce({})
-
- const server: McpServerCRD = {
- ...makeServer({ name: 'pg' }),
- uid: 'uid-pg',
- generation: 17,
- }
- await reconciler.writeStatusCondition(server, {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'done',
- })
-
- expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
- expect(customApi.patchNamespacedCustomObjectStatus.mock.calls[0][0].body).toContainEqual({
- op: 'test',
- path: '/metadata/resourceVersion',
- value: '100',
- })
- const secondPatch = customApi.patchNamespacedCustomObjectStatus.mock.calls[1][0].body
- expect(secondPatch).toContainEqual({
- op: 'test',
- path: '/metadata/resourceVersion',
- value: '101',
- })
- const conditionPatch = secondPatch.find(
- (candidate: { op?: string; path?: string }) =>
- candidate.op === 'add' && candidate.path === '/status/conditions'
- )
- expect(conditionPatch?.value.map((written: { type: string }) => written.type)).toEqual([
- 'NetworkReady',
- 'ExternalEgressReady',
- 'Ready',
- ])
- })
-
- it('bounds resourceVersion conflict retries', async () => {
- const currentStatus = {
- metadata: { resourceVersion: '100' },
- status: { conditions: [] },
- }
- customApi.getNamespacedCustomObjectStatus
- .mockResolvedValueOnce(currentStatus)
- .mockResolvedValueOnce(currentStatus)
- .mockResolvedValueOnce(currentStatus)
- const conflict = Object.assign(new Error('resourceVersion conflict'), { code: 422 })
- customApi.patchNamespacedCustomObjectStatus
- .mockRejectedValueOnce(conflict)
- .mockRejectedValueOnce(conflict)
- .mockRejectedValueOnce(conflict)
- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
-
- await reconciler.writeStatusCondition(
- {
- ...makeServer({ name: 'pg' }),
- uid: 'uid-pg',
- generation: 17,
- },
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'done',
- }
- )
-
- expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(3)
- expect(warn).toHaveBeenCalledWith(
- expect.stringContaining('Failed to write status condition Ready=True'),
- conflict
- )
- warn.mockRestore()
- })
-
- it('guards the status patch with the captured UID, generation, and resourceVersion', async () => {
- const staleServer: McpServerCRD = {
- ...makeServer({ name: 'recreated' }),
- uid: 'uid-before-recreation',
- generation: 1,
- }
- customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
- metadata: { resourceVersion: '100' },
- status: { conditions: [] },
- })
- const writeFailure = Object.assign(new Error('status API unavailable'), {
- code: 500,
- })
- customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async request => {
- expect(request.body.slice(0, 3)).toEqual([
- {
- op: 'test',
- path: '/metadata/uid',
- value: 'uid-before-recreation',
- },
- {
- op: 'test',
- path: '/metadata/generation',
- value: 1,
- },
- {
- op: 'test',
- path: '/metadata/resourceVersion',
- value: '100',
- },
- ])
- throw writeFailure
- })
- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
-
- await reconciler.writeStatusCondition(staleServer, {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'stale reconcile result',
- })
-
- expect(warn).toHaveBeenCalledWith(
- expect.stringContaining('Failed to write status condition Ready=True'),
- writeFailure
- )
- warn.mockRestore()
- })
-
- it('refreshes observedGeneration even when the condition payload is otherwise unchanged', async () => {
- customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
- status: {
- conditions: [
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'done',
- observedGeneration: 16,
- lastTransitionTime: '2020-01-01T00:00:00.000Z',
- },
- ],
- },
- })
- const server = {
- ...makeServer({ name: 'pg' }),
- generation: 17,
- }
-
- await reconciler.writeStatusCondition(server, {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'done',
- })
-
- const { conditions } = getPatchedConditions()
- expect(conditions.find((condition: { type: string }) => condition.type === 'Ready')).toEqual(
- expect.objectContaining({
- observedGeneration: 17,
- lastTransitionTime: '2020-01-01T00:00:00.000Z',
- })
- )
- })
-
it('creates the status object when the CRD has no status yet', async () => {
customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({})
@@ -932,170 +653,6 @@ describe('PR-B B1 — writeStatusCondition', () => {
})
})
-describe('restart-safe discovery status', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
- let reconciler: McpServerReconciler
-
- beforeEach(() => {
- vi.clearAllMocks()
- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(coreApi),
- customApi: asCustomApi(customApi),
- })
- })
-
- it('uses a persisted Ready condition only when it observes the current generation', () => {
- const server: McpServerCRD = {
- ...makeServer({ name: 'restart-safe' }),
- generation: 7,
- status: {
- conditions: [
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'Deployment created',
- observedGeneration: 7,
- },
- ],
- },
- }
-
- expect(reconciler.getStatus(server)).toEqual({
- deployed: true,
- ready: true,
- message: 'Deployment created',
- authoritative: true,
- })
- })
-
- it.each([
- ['stale', 6],
- ['unversioned', undefined],
- ])('fails closed for a %s persisted Ready condition', (_label, observedGeneration) => {
- const server: McpServerCRD = {
- ...makeServer({ name: 'restart-unsafe' }),
- generation: 7,
- status: {
- conditions: [
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'Deployment created',
- observedGeneration,
- },
- ],
- },
- }
-
- expect(reconciler.getStatus(server)).toEqual({
- deployed: false,
- ready: false,
- message: 'Not reconciled yet',
- authoritative: false,
- })
- })
-
- it('fails closed when persisted Ready conditions conflict for the current generation', () => {
- const server: McpServerCRD = {
- ...makeServer({ name: 'restart-conflict' }),
- generation: 7,
- status: {
- conditions: [
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- observedGeneration: 7,
- },
- {
- type: 'Ready',
- status: 'False',
- reason: 'SecretValidationFailed',
- observedGeneration: 7,
- },
- ],
- },
- }
-
- expect(reconciler.getStatus(server)).toEqual({
- deployed: false,
- ready: false,
- message: 'Not reconciled yet',
- authoritative: false,
- })
- })
-
- it('prefers fresh in-memory status over persisted status from the same generation', async () => {
- const server: McpServerCRD = {
- ...makeServer({ name: 'live-status' }),
- generation: 3,
- status: {
- conditions: [
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- observedGeneration: 3,
- },
- ],
- },
- }
- appsApi.readNamespacedDeployment.mockResolvedValueOnce({
- status: { readyReplicas: 0 },
- })
-
- await reconciler.reconcile(server)
-
- expect(reconciler.getStatus(server)).toEqual(
- expect.objectContaining({
- deployed: true,
- ready: false,
- authoritative: true,
- })
- )
- })
-
- it.each([
- ['generation changes', 'stable-uid', 4],
- ['the same name is recreated', 'replacement-uid', 3],
- ])('does not reuse in-memory status when %s', async (_case, uid, generation) => {
- const original: McpServerCRD = {
- ...makeServer({ name: 'identity-fenced' }),
- uid: 'original-uid',
- generation: 3,
- }
- await reconciler.reconcile(original)
- // #223 (credential rotation) makes readiness immediately False until the poll
- // upgrades it, so the original caches deployed/authoritative here — not ready.
- // This test's subject is the identity fencing asserted below, not readiness
- // timing; it only needs a real (non-default) cached status to prove it is not
- // reused across an identity change.
- expect(reconciler.getStatus(original)).toEqual(
- expect.objectContaining({ deployed: true, ready: false, authoritative: true })
- )
-
- const current: McpServerCRD = {
- ...original,
- uid,
- generation,
- status: undefined,
- }
-
- expect(reconciler.getStatus(current)).toEqual({
- deployed: false,
- ready: false,
- message: 'Not reconciled yet',
- authoritative: false,
- })
- })
-})
-
describe('ownership-safe fail-closed cleanup', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
@@ -1105,7 +662,6 @@ describe('ownership-safe fail-closed cleanup', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -1208,480 +764,6 @@ describe('ownership-safe fail-closed cleanup', () => {
})
})
-describe('full reconciliation inventory authority', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
- let reconciler: McpServerReconciler
-
- beforeEach(() => {
- vi.clearAllMocks()
- customApi.getNamespacedCustomObject.mockRejectedValue(
- Object.assign(new Error('not found'), { code: 404 })
- )
- reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(coreApi),
- customApi: asCustomApi(customApi),
- })
- })
-
- it('rejects when Deployment inventory LIST fails so startup retry remains possible', async () => {
- const inventoryError = new Error('deployment inventory unavailable')
- const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
- appsApi.listNamespacedDeployment.mockRejectedValueOnce(inventoryError)
-
- await expect(reconciler.fullReconcile([])).rejects.toBe(inventoryError)
-
- expect(error).toHaveBeenCalledWith(
- '[Reconciler] Failed to list managed deployments:',
- inventoryError
- )
- error.mockRestore()
- })
-
- it('defaults to fail-closed orphan cleanup until an authority source is wired', async () => {
- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
- appsApi.listNamespacedDeployment.mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-server',
- namespace: 'mcp-server',
- },
- },
- ],
- })
-
- await reconciler.fullReconcile([])
-
- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
- expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))
- warn.mockRestore()
- })
-
- it('finishes desired reconciliation but skips orphan cleanup if inventory authority is lost mid-pass', async () => {
- let authority = { known: true, generation: 7 }
- reconciler.setInventoryAuthority(() => authority)
- const desired = makeServer({ name: 'desired-server' })
- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
- const reconcile = vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {
- authority = { known: false, generation: 8 }
- })
- appsApi.listNamespacedDeployment.mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-server',
- namespace: 'mcp-server',
- },
- },
- ],
- })
-
- await reconciler.fullReconcile([desired])
-
- expect(reconcile).toHaveBeenCalledOnce()
- expect(reconcile).toHaveBeenCalledWith(desired, {
- isCurrent: expect.any(Function),
- })
- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
- expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))
- warn.mockRestore()
- })
-
- it('skips orphan cleanup if the authoritative inventory generation changes mid-pass', async () => {
- let authority = { known: true, generation: 11 }
- reconciler.setInventoryAuthority(() => authority)
- const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
- vi.spyOn(reconciler, 'reconcile').mockImplementation(async () => {
- authority = { known: true, generation: 12 }
- })
- appsApi.listNamespacedDeployment.mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-server',
- namespace: 'mcp-server',
- },
- },
- ],
- })
-
- await reconciler.fullReconcile([makeServer({ name: 'desired-server' })])
-
- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
- expect(warn).toHaveBeenCalledWith(expect.stringContaining('inventory authority'))
- warn.mockRestore()
- })
-
- it('deletes an orphan when inventory authority remains known and stable', async () => {
- reconciler.setInventoryAuthority(() => ({ known: true, generation: 21 }))
- appsApi.listNamespacedDeployment.mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-server',
- namespace: 'mcp-server',
- },
- },
- ],
- })
-
- await reconciler.fullReconcile([])
-
- expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
- name: 'orphan-server',
- namespace: 'mcp-server',
- })
- expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
- name: 'orphan-server',
- namespace: 'mcp-server',
- })
- })
-
- it('skips a startup runtime revision superseded before its keyed effect is admitted', async () => {
- const snapshot: McpServerCRD = {
- ...makeServer({ name: 'updated-server' }),
- generation: 1,
- spec: {
- contextRef: 'default',
- image: 'clerum/updated-server:old',
- transport: { type: 'streamableHttp', port: 3000 },
- },
- }
- const current: McpServerCRD = {
- ...makeServer({ name: 'updated-server' }),
- generation: 2,
- spec: {
- contextRef: 'default',
- image: 'clerum/updated-server:new',
- transport: { type: 'streamableHttp', port: 3000 },
- egressBindings: [{ dns: 'new.example', port: 443 }],
- },
- }
- reconciler.setInventoryAuthority(() => ({ known: true, generation: 40 }))
- reconciler.setResolveCurrentServer(() => current)
- const reconcile = vi.spyOn(reconciler, 'reconcile')
-
- await reconciler.fullReconcile([snapshot])
-
- expect(reconcile).not.toHaveBeenCalled()
- })
-
- it('does not mutate runtime or status after startup authority retires during Secret validation', async () => {
- let authority = { known: true, generation: 51 }
- const server = {
- ...makeServer({
- name: 'secret-authority-race',
- envSecret: {
- name: 'runtime-secret',
- keys: [{ secretKey: 'token', envVar: 'TOKEN' }],
- },
- }),
- uid: 'secret-authority-race-uid',
- generation: 4,
- }
- reconciler.setInventoryAuthority(() => authority)
- reconciler.setResolveCurrentServer(name => (name === server.name ? server : undefined))
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred<{ data: Record<string, string> }>()
- coreApi.readNamespacedSecret.mockImplementationOnce(async () => {
- secretReadStarted.resolve(undefined)
- return releaseSecretRead.promise
- })
-
- const convergence = reconciler.fullReconcile([server])
- await secretReadStarted.promise
-
- authority = { known: true, generation: 53 }
- releaseSecretRead.resolve({ data: { token: 'encoded-token' } })
- await convergence
-
- expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
- expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- })
-
- it('retires a same-watch runtime revision superseded during Secret validation', async () => {
- let current: McpServerCRD
- const original: McpServerCRD = {
- ...makeServer({
- name: 'same-watch-runtime',
- image: 'clerum/same-watch-runtime:v1',
- envSecret: {
- name: 'runtime-secret',
- keys: [{ secretKey: 'token', envVar: 'TOKEN' }],
- },
- }),
- uid: 'same-watch-runtime-uid',
- generation: 1,
- }
- const replacement: McpServerCRD = {
- ...original,
- generation: 2,
- spec: {
- ...original.spec,
- image: 'clerum/same-watch-runtime:v2',
- },
- }
- current = original
- reconciler.setInventoryAuthority(() => ({ known: true, generation: 61 }))
- reconciler.setResolveCurrentServer(name => (name === original.name ? current : undefined))
-
- const originalSecretReadStarted = deferred()
- const releaseOriginalSecretRead = deferred<{ data: Record<string, string> }>()
- coreApi.readNamespacedSecret
- .mockImplementationOnce(async () => {
- originalSecretReadStarted.resolve(undefined)
- return releaseOriginalSecretRead.promise
- })
- .mockResolvedValueOnce({ data: { token: 'encoded-token' } })
-
- const originalReconcile = reconciler.reconcile(original)
- await originalSecretReadStarted.promise
- current = replacement
- const replacementReconcile = reconciler.reconcile(replacement)
- releaseOriginalSecretRead.resolve({ data: { token: 'encoded-token' } })
-
- await Promise.all([originalReconcile, replacementReconcile])
-
- expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith({
- namespace: replacement.namespace,
- body: expect.objectContaining({
- spec: expect.objectContaining({
- template: expect.objectContaining({
- spec: expect.objectContaining({
- containers: expect.arrayContaining([
- expect.objectContaining({
- image: 'clerum/same-watch-runtime:v2',
- }),
- ]),
- }),
- }),
- }),
- }),
- })
- })
-
- it('admits independent desired effects while another server effect is blocked', async () => {
- const blocked = makeServer({ name: 'blocked-server' })
- const independent = makeServer({ name: 'independent-server' })
- let markBlockedStarted!: () => void
- let releaseBlocked!: () => void
- const blockedStarted = new Promise<void>(resolve => {
- markBlockedStarted = resolve
- })
- const blockedRelease = new Promise<void>(resolve => {
- releaseBlocked = resolve
- })
- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
- const fullReconcile = reconciler.fullReconcile([blocked, independent], {
- runEffect: async (serverName, work) => {
- if (serverName === blocked.name) {
- markBlockedStarted()
- await blockedRelease
- }
- await work()
- },
- })
-
- await blockedStarted
- await Promise.resolve()
-
- expect(reconcile).toHaveBeenCalledWith(independent, {
- isCurrent: expect.any(Function),
- })
- expect(reconcile.mock.calls.some(([server]) => server === blocked)).toBe(false)
-
- releaseBlocked()
- await fullReconcile
-
- expect(reconcile).toHaveBeenCalledWith(blocked, {
- isCurrent: expect.any(Function),
- })
- })
-
- it('bounds desired-effect admission without serializing independent siblings', async () => {
- const desired = Array.from({ length: 25 }, (_, index) =>
- makeServer({ name: `bounded-server-${index}` })
- )
- let active = 0
- let maxActive = 0
- let startedBeforeRelease = 0
- let releaseAll!: () => void
- const release = new Promise<void>(resolve => {
- releaseAll = resolve
- })
- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
- const options = {
- maxConcurrency: 3,
- runEffect: async (_serverName: string, work: () => Promise<void>) => {
- active += 1
- startedBeforeRelease += 1
- maxActive = Math.max(maxActive, active)
- await release
- try {
- await work()
- } finally {
- active -= 1
- }
- },
- }
-
- const fullReconcile = reconciler.fullReconcile(desired, options)
- await vi.waitFor(() => expect(active).toBeGreaterThanOrEqual(3))
- await Promise.resolve()
- const observedMaxActive = maxActive
- const observedStartedBeforeRelease = startedBeforeRelease
-
- releaseAll()
- await fullReconcile
-
- expect(observedMaxActive).toBe(3)
- expect(observedStartedBeforeRelease).toBe(3)
- expect(reconcile).toHaveBeenCalledTimes(desired.length)
- })
-
- it('waits for every admitted desired effect before reporting a sibling failure', async () => {
- const failed = makeServer({ name: 'failed-server' })
- const slow = makeServer({ name: 'slow-server' })
- let markSlowStarted!: () => void
- let releaseSlow!: () => void
- const slowStarted = new Promise<void>(resolve => {
- markSlowStarted = resolve
- })
- const slowRelease = new Promise<void>(resolve => {
- releaseSlow = resolve
- })
- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
-
- const fullReconcile = reconciler.fullReconcile([failed, slow], {
- runEffect: async (serverName, work) => {
- if (serverName === failed.name) {
- throw new Error('failed server gate')
- }
- markSlowStarted()
- await slowRelease
- await work()
- },
- })
- let settled = false
- const observed = fullReconcile.then(
- () => undefined,
- error => error
- )
- void observed.then(() => {
- settled = true
- })
-
- await slowStarted
- await Promise.resolve()
- await Promise.resolve()
-
- expect(settled).toBe(false)
-
- releaseSlow()
- const error = await observed
-
- expect(error).toBeInstanceOf(AggregateError)
- expect(error).toMatchObject({
- message: 'Failed to reconcile desired McpServers',
- })
- expect(reconcile).toHaveBeenCalledWith(slow, {
- isCurrent: expect.any(Function),
- })
- })
-
- it('aggregates every desired-effect failure after all bounded workers settle', async () => {
- const failures = [
- new Error('first desired failure'),
- new Error('second desired failure'),
- new Error('third desired failure'),
- ]
- const desired = failures.map((_, index) => makeServer({ name: `failed-${index}` }))
- const reconcile = vi.spyOn(reconciler, 'reconcile').mockResolvedValue(undefined)
-
- const error = await reconciler
- .fullReconcile(desired, {
- maxConcurrency: 2,
- runEffect: async serverName => {
- throw failures[Number(serverName.replace('failed-', ''))]
- },
- })
- .then(
- () => undefined,
- reason => reason
- )
-
- expect(error).toBeInstanceOf(AggregateError)
- expect((error as AggregateError).errors).toEqual(failures)
- expect(reconcile).not.toHaveBeenCalled()
- })
-
- it('does not delete an orphan candidate that was added to the same-generation cache mid-pass', async () => {
- const currentServers = new Map<string, McpServerCRD>()
- ;(reconciler as any).setResolveCurrentServer((name: string) => currentServers.get(name))
- reconciler.setInventoryAuthority(() => ({ known: true, generation: 31 }))
- appsApi.listNamespacedDeployment.mockImplementationOnce(async () => {
- currentServers.set('orphan-server', makeServer({ name: 'orphan-server' }))
- return {
- items: [
- {
- metadata: {
- name: 'orphan-server',
- namespace: 'mcp-server',
- },
- },
- ],
- }
- })
-
- await reconciler.fullReconcile([])
-
- expect(customApi.getNamespacedCustomObject).not.toHaveBeenCalled()
- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
- })
-
- it('does not delete an orphan candidate when a fresh API GET sees a same-name CRD before ADDED arrives', async () => {
- ;(reconciler as any).setResolveCurrentServer(() => undefined)
- reconciler.setInventoryAuthority(() => ({ known: true, generation: 32 }))
- customApi.getNamespacedCustomObject.mockResolvedValueOnce({
- metadata: { name: 'orphan-server', namespace: 'mcp-server', uid: 'recreated-uid' },
- })
- appsApi.listNamespacedDeployment.mockResolvedValueOnce({
- items: [
- {
- metadata: {
- name: 'orphan-server',
- namespace: 'mcp-server',
- },
- },
- ],
- })
-
- await reconciler.fullReconcile([])
-
- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledWith(
- expect.objectContaining({
- namespace: 'mcp-server',
- plural: 'mcpservers',
- name: 'orphan-server',
- })
- )
- expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
- expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
- })
-})
-
describe('updateStatusConditions', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
@@ -1691,7 +773,6 @@ describe('updateStatusConditions', () => {
beforeEach(() => {
vi.clearAllMocks()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
customApi: asCustomApi(customApi),
@@ -1754,53 +835,3 @@ describe('updateStatusConditions', () => {
expect(types).toContain('DeploymentReady')
})
})
-
-describe('unconfigured inventory authority fails closed (G1)', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
- let errorSpy: ReturnType<typeof vi.spyOn>
-
- // Deliberately NO assumeInventoryAuthorityWhenUnconfigured and NO
- // setInventoryAuthority/setResolveCurrentServer wiring: the fence must deny.
- function makeUnwiredReconciler(): McpServerReconciler {
- return new McpServerReconciler({} as k8s.KubeConfig, {
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(coreApi),
- customApi: asCustomApi(customApi),
- })
- }
-
- beforeEach(() => {
- vi.clearAllMocks()
- errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
- })
-
- afterEach(() => {
- errorSpy.mockRestore()
- })
-
- it('T-G1a: reconcile() denies (no Deployment) and warns once when authority is unwired', async () => {
- const reconciler = makeUnwiredReconciler()
- await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))
- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
- expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('fail closed'))
- })
-
- it('T-G1b: fullReconcile() denies when authority is unwired', async () => {
- const reconciler = makeUnwiredReconciler()
- await reconciler.fullReconcile([makeServer({ name: 'g1-mcp', managed: true })])
- expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
- })
-
- it('T-G1c: the explicit opt-in restores assume-current', async () => {
- const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
- appsApi: asAppsApi(appsApi),
- coreApi: asCoreApi(coreApi),
- customApi: asCustomApi(customApi),
- assumeInventoryAuthorityWhenUnconfigured: true,
- })
- await reconciler.reconcile(makeServer({ name: 'g1-mcp', managed: true }))
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
- })
-})
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index 4056c621c..caba6be91 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -16,10 +16,6 @@ import { classifyPluginImage } from '@clerum/image-policy'
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
import { config } from './config'
import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'
-import {
- confirmAuthoritativeMcpServerAbsence,
- sameMcpServerDesiredRevision,
-} from './mcpServerSafety'
import { mcpserverMissingSecret } from './metrics'
import { McpServerCRD, McpServerStatus } from './types'
import {
@@ -66,7 +62,6 @@ export interface McpServerCondition {
reason: string
message: string
lastTransitionTime: string
- observedGeneration?: number
}
type McpServerReconcilerDeps = {
@@ -74,28 +69,6 @@ type McpServerReconcilerDeps = {
coreApi?: k8s.CoreV1Api
customApi?: k8s.CustomObjectsApi
networkingApi?: k8s.NetworkingV1Api
- /**
- * TEST-ONLY escape hatch. When the inventory-authority fence is not wired
- * (setInventoryAuthority / setResolveCurrentServer never called), mutation
- * fences fail CLOSED by default. Unit tests that exercise the reconciler
- * without a watcher opt in to the legacy assume-current behavior here.
- * Production construction (k8sClient) must never set this.
- */
- assumeInventoryAuthorityWhenUnconfigured?: boolean
-}
-
-export type McpServerInventoryAuthoritySnapshot = {
- known: boolean
- generation: number
-}
-
-export type McpServerFullReconcileOptions = {
- runEffect?: (serverName: string, work: () => Promise<void>) => Promise<void>
- maxConcurrency?: number
-}
-
-export type McpServerMutationOptions = {
- isCurrent?: () => boolean
}
// G2: Pre-deploy handshake constants
@@ -115,13 +88,6 @@ const HEADER_NAME_RE = /^[A-Za-z0-9-]+$/
// Bound the value length so a pathological CRD cannot blow up the ConfigMap.
const MAX_AUTH_HEADER_VALUE_LENGTH = 2048
const WRC_OWNED_RUNTIME_READY_REASON = 'WrcOwnedRuntimeRegistered'
-// Keep fleet admission aligned with the external-egress convergence lane:
-// enough parallelism for independent servers without allocating one pending
-// effect promise per fleet member.
-const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10
-// Status is a read/merge/write transaction shared with other controllers.
-// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.
-const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3
function throwCleanupFailures(failures: unknown[], message: string): void {
if (failures.length > 0) {
@@ -129,37 +95,6 @@ function throwCleanupFailures(failures: unknown[], message: string): void {
}
}
-async function mapSettledWithConcurrency<T, TResult>(
- items: readonly T[],
- maxConcurrency: number,
- work: (item: T, index: number) => Promise<TResult>
-): Promise<Array<PromiseSettledResult<TResult>>> {
- if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {
- throw new RangeError('maxConcurrency must be a positive integer')
- }
-
- const results = new Array<PromiseSettledResult<TResult>>(items.length)
- let nextIndex = 0
- const worker = async (): Promise<void> => {
- while (nextIndex < items.length) {
- const index = nextIndex
- nextIndex += 1
- try {
- results[index] = {
- status: 'fulfilled',
- value: await work(items[index], index),
- }
- } catch (reason) {
- results[index] = { status: 'rejected', reason }
- }
- }
- }
-
- const workerCount = Math.min(maxConcurrency, items.length)
- await Promise.all(Array.from({ length: workerCount }, () => worker()))
- return results
-}
-
/**
* State of the readiness poll window for one McpServer (issue #223).
*
@@ -211,35 +146,6 @@ interface ReadinessPollState {
* WaitingForReplicas by the very reconcile the terminal write triggers.
*/
detail: string
- /**
- * Mutation fence and CRD object of the LATEST reconcile that owns this
- * window. pollReadiness() REBINDS both on every call that finds the window
- * live: a superseding reconcile (e.g. the fullReconcile after a routine
- * McpServer watch recovery bumps the inventory-authority generation) becomes
- * the window's owner, so the next tick evaluates the CURRENT fence instead of
- * the arming reconcile's stale closure. Without the rebind the stale fence
- * self-terminates the window with no re-arm and — status-only MODIFIED events
- * being suppressed, with no periodic McpServer resync — DeploymentReady=False
- * sticks forever for a healthy rollout (the #223 rotation-timeout symptom,
- * reintroduced under a routine reconnect).
- *
- * Two ORTHOGONAL supersession guards protect this window; do not assume one
- * covers the other (that assumption is exactly how the F1 race was born):
- * - Identity retirement: beginStatusTracking()/clearStatus() retire the
- * window on any CRD uid or CRD-generation change (spec edit, recreate),
- * so a surviving window always has the caller's status identity — a
- * rebind can never adopt a recreate or a superseded desired revision.
- * - In-tick generation reset + recheckRequested: a credential rotation
- * bumps neither CRD uid nor CRD generation (it changes the Deployment's
- * metadata.generation only), so identity retirement does NOT fire; the
- * rotation is instead guarded by the tick's live Deployment read (a new
- * generation resets attempts/exhausted) and by recheckRequested deferring
- * a mid-flight tick's terminal verdict past the rotation's write.
- * A rebind never touches the budget fields (generation/attempts/exhausted/
- * detail) above.
- */
- isCurrent: () => boolean
- server: McpServerCRD
}
export class McpServerReconciler {
@@ -248,26 +154,8 @@ export class McpServerReconciler {
private readonly customApi: k8s.CustomObjectsApi
private readonly networkingApi?: k8s.NetworkingV1Api
- /**
- * Snapshot the McpServer LIST→WATCH authority owned by McpServerWatcher.
- * The fail-closed default prevents orphan cleanup until the watcher wires
- * the real authority source.
- */
- private inventoryAuthority: () => McpServerInventoryAuthoritySnapshot = () => ({
- known: false,
- generation: 0,
- })
- private inventoryAuthorityConfigured = false
- private resolveCurrentServer?: (name: string) => McpServerCRD | undefined
- /** See McpServerReconcilerDeps: test-only opt-in to assume-current when the
- * inventory-authority fence is unwired. Production leaves it false. */
- private readonly assumeInventoryAuthorityWhenUnconfigured: boolean
- private warnedUnconfiguredAuthority = false
-
/** Tracks the deployment status of each managed McpServer. */
private readonly statusMap: Map<string, McpServerStatus> = new Map()
- private readonly statusIdentities: Map<string, Pick<McpServerCRD, 'uid' | 'generation'>> =
- new Map()
/** G7: Tracks the initial `managed` value per McpServer to enforce immutability. */
private readonly managedSnapshot: Map<string, boolean> = new Map()
@@ -302,120 +190,20 @@ export class McpServerReconciler {
this.networkingApi =
deps?.networkingApi ??
(typeof kc.makeApiClient === 'function' ? kc.makeApiClient(k8s.NetworkingV1Api) : undefined)
- this.assumeInventoryAuthorityWhenUnconfigured =
- deps?.assumeInventoryAuthorityWhenUnconfigured ?? false
- }
-
- /**
- * Late-bound authority source for destructive full-reconcile cleanup.
- * A full pass may finish reconciling desired servers, but it may delete an
- * orphan only while the same authoritative inventory generation remains
- * active from pass start through delete admission.
- */
- setInventoryAuthority(fn: () => McpServerInventoryAuthoritySnapshot): void {
- this.inventoryAuthority = fn
- this.inventoryAuthorityConfigured = true
- }
-
- /**
- * Resolve current cache membership at effect admission. Full-reconcile
- * snapshots select candidates only; they must never overwrite or delete a
- * same-name CRD accepted by the continuing watch.
- */
- setResolveCurrentServer(fn: (name: string) => McpServerCRD | undefined): void {
- this.resolveCurrentServer = fn
- }
-
- /**
- * Verdict for a mutation fence evaluated while the inventory authority is NOT
- * wired. Fail closed (deny) unless a test explicitly opted in; deny loudly
- * once, because an unconfigured fence in a running controller is a
- * construction bug, never a runtime condition.
- */
- private currentWhenUnconfigured(): boolean {
- if (this.assumeInventoryAuthorityWhenUnconfigured) return true
- if (!this.warnedUnconfiguredAuthority) {
- this.warnedUnconfiguredAuthority = true
- console.error(
- '[Reconciler] inventory authority not configured — mutation fences fail closed. ' +
- 'Wire setInventoryAuthority() and setResolveCurrentServer() before reconciling.'
- )
- }
- return false
}
// ─── Status ────────────────────────────────────────────────────────
/**
* Get deployment status for an McpServer.
- * Returns current in-memory status when this process has reconciled the
- * server. After a controller restart, a generation-matched persisted Ready
- * condition is the only safe fallback: a legacy condition without
- * observedGeneration, or a condition for an older spec generation, cannot
- * prove that the current desired runtime is ready.
+ * Returns unknown status if the server hasn't been reconciled yet.
*/
- getStatus(serverOrName: string | McpServerCRD): McpServerStatus {
- const name = typeof serverOrName === 'string' ? serverOrName : serverOrName.name
- const current = this.statusMap.get(name)
- if (current && (typeof serverOrName === 'string' || this.statusIdentityMatches(serverOrName))) {
- return typeof serverOrName === 'string' ? current : { ...current, authoritative: true }
- }
-
- if (typeof serverOrName !== 'string' && serverOrName.generation !== undefined) {
- const readyConditions =
- serverOrName.status?.conditions?.filter(condition => condition.type === 'Ready') ?? []
- const persistedReady =
- readyConditions.length === 1 &&
- readyConditions[0].observedGeneration === serverOrName.generation
- ? readyConditions[0]
- : undefined
- if (persistedReady) {
- const ready = persistedReady.status === 'True'
- return {
- deployed: ready,
- ready,
- message: persistedReady.message,
- authoritative: true,
- }
- }
- }
-
- return {
- deployed: false,
- ready: false,
- message: 'Not reconciled yet',
- authoritative: false,
- }
- }
-
- private statusIdentityMatches(server: McpServerCRD): boolean {
- const identity = this.statusIdentities.get(server.name)
+ getStatus(name: string): McpServerStatus {
return (
- identity !== undefined &&
- identity.uid === server.uid &&
- identity.generation === server.generation
+ this.statusMap.get(name) ?? { deployed: false, ready: false, message: 'Not reconciled yet' }
)
}
- private beginStatusTracking(server: McpServerCRD): void {
- if (!this.statusIdentityMatches(server)) {
- this.statusMap.delete(server.name)
- // A new CRD identity (uid/generation) retires any poll window armed for
- // the previous desired revision: its verdicts would describe a rollout
- // that no longer represents the desired state. The reconcile for the
- // new identity arms its own window after its Deployment write.
- const poll = this.readinessPolls.get(server.name)
- if (poll) {
- if (poll.timer !== undefined) clearTimeout(poll.timer)
- this.readinessPolls.delete(server.name)
- }
- }
- this.statusIdentities.set(server.name, {
- uid: server.uid,
- generation: server.generation,
- })
- }
-
/**
* Update the status of an McpServer after reconciliation.
*/
@@ -428,7 +216,6 @@ export class McpServerReconciler {
*/
private clearStatus(name: string): void {
this.statusMap.delete(name)
- this.statusIdentities.delete(name)
this.managedSnapshot.delete(name)
const poll = this.readinessPolls.get(name)
if (poll) {
@@ -520,26 +307,14 @@ export class McpServerReconciler {
* window with a superseded generation's numbers and the new rollout would
* never be observed (see ReadinessPollState.recheckRequested).
*/
- private pollReadiness(
- server: McpServerCRD,
- intervalMs = 5000,
- maxAttempts = 24,
- isCurrent: () => boolean = () => true
- ): void {
+ private pollReadiness(server: McpServerCRD, intervalMs = 5000, maxAttempts = 24): void {
const { name, namespace } = server
const previous = this.readinessPolls.get(name)
if (previous !== undefined && (previous.timer !== undefined || previous.ticking)) {
- // A window is already running for this server — leave its TIMER and
- // BUDGET alone (re-arming would reset the attempt budget: the E2
- // status-update-loop). But this caller is now the window's owner: rebind
- // the mutation fence (and the same-identity server object) so the next
- // tick evaluates the CURRENT fence instead of the arming reconcile's
- // stale closure, which would otherwise self-terminate the window with no
- // re-arm after a routine watch-generation bump.
- previous.isCurrent = isCurrent
- previous.server = server
- // If its tick is executing RIGHT NOW, the tick's Deployment read may
+ // A window is already running for this server — leave it alone
+ // (re-arming would reset the attempt budget: the E2 status-update-loop).
+ // But if its tick is executing RIGHT NOW, the tick's Deployment read may
// predate the Deployment write that triggered THIS call (a new
// generation), so a silent no-op could let the tick end the window on
// stale data. Record that a re-check is owed; the tick honors it before
@@ -562,8 +337,6 @@ export class McpServerReconciler {
attempts: 0,
exhausted: previous?.exhausted ?? false,
detail: previous?.detail ?? '',
- isCurrent,
- server,
}
this.readinessPolls.set(name, state)
@@ -573,21 +346,7 @@ export class McpServerReconciler {
// can tell "tick running" apart from "window ended" during our awaits.
state.timer = undefined
state.ticking = true
- // Mutation fencing: a superseded reconcile (desired-revision or
- // inventory-authority change) must not keep observing — or writing
- // status — on behalf of a stale desired state. End the window quietly;
- // the current reconcile arms its own window after its Deployment write.
- if (!state.isCurrent()) {
- state.ticking = false
- if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
- return
- }
const rollout = await this.readDeploymentRollout(name, namespace)
- if (!state.isCurrent() || !this.statusIdentityMatches(state.server)) {
- state.ticking = false
- if (this.readinessPolls.get(name) === state) this.readinessPolls.delete(name)
- return
- }
// The McpServer may have been deleted (clearStatus) while we were
// reading. A cancelled window must not keep ticking or write status
@@ -636,20 +395,7 @@ export class McpServerReconciler {
// always false, so this is the ONLY place a converged rollout is ever
// recorded — without it every successful credential rotation would sit
// at DeploymentReady=False forever and the UI would report a timeout.
- await this.updateStatusConditions(state.server, true, undefined, state.isCurrent)
- // Readiness is decoupled from fleet convergence: the poll also
- // upgrades the persisted, generation-matched Ready condition so a
- // controller restart can trust the recorded readiness (see getStatus).
- await this.writeStatusCondition(
- state.server,
- {
- type: 'Ready',
- status: 'True',
- reason: 'ReconcileSuccess',
- message: 'Deployment created',
- },
- state.isCurrent
- )
+ await this.updateStatusConditions(server, true)
return
}
@@ -669,7 +415,7 @@ export class McpServerReconciler {
// Give up loudly, on the CRD: a stuck rollout must be visible to the
// operator with the numbers that prove it, never left as a stale
// condition or a silent Unknown.
- await this.updateStatusConditions(state.server, false, rollout.detail, state.isCurrent)
+ await this.updateStatusConditions(server, false, rollout.detail)
return
}
@@ -933,12 +679,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
/**
* Ensure ConfigMap exists and is up to date (create or replace).
*/
- private async ensureConfigMap(
- server: McpServerCRD,
- isCurrent: () => boolean = () => true
- ): Promise<void> {
+ private async ensureConfigMap(server: McpServerCRD): Promise<void> {
const cm = this.buildNginxConfigMap(server)
- if (!isCurrent()) return
try {
await this.coreApi.createNamespacedConfigMap({
namespace: server.namespace,
@@ -951,7 +693,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
name: `${server.name}-nginx-conf`,
namespace: server.namespace,
})
- if (!isCurrent()) return
const next = preserveObjectAnnotations(
{
...cm,
@@ -962,7 +703,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
existing
)
- if (!isCurrent()) return
await this.coreApi.replaceNamespacedConfigMap({
name: `${server.name}-nginx-conf`,
namespace: server.namespace,
@@ -1070,13 +810,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
private buildDeployment(server: McpServerCRD, credentialsRevision = ''): k8s.V1Deployment {
- // ── Platform hardening: sanitize an ephemeral copy before building PodSpec ──
- // The caller retains this object as the LIST -> WATCH desired-state cache.
- // Mutating it here makes a later status-only watch event look like a
- // user-driven spec revision, which invalidates the authoritative policy
- // revocation marker during startup. Sanitization remains mandatory for the
- // generated PodSpec, but must never rewrite the observed CRD inventory.
- server = structuredClone(server)
+ // ── Platform hardening: sanitize CRD before building PodSpec ──
this.sanitizeCrdSpec(server)
const transportPort = server.spec.transport.port || 3000
@@ -1472,14 +1206,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
/**
* Ensure a Deployment exists and is up to date.
*/
- private async ensureDeployment(
- server: McpServerCRD,
- credentialsRevision = '',
- isCurrent: () => boolean = () => true
- ): Promise<void> {
+ private async ensureDeployment(server: McpServerCRD, credentialsRevision = ''): Promise<void> {
const deployment = this.buildDeployment(server, credentialsRevision)
- if (!isCurrent()) return
try {
await this.appsApi.createNamespacedDeployment({
namespace: server.namespace,
@@ -1514,7 +1243,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
namespace: server.namespace,
body,
}),
- mutationAllowed: isCurrent,
})
}
}
@@ -1522,13 +1250,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
/**
* Ensure a Service exists and is up to date.
*/
- private async ensureService(
- server: McpServerCRD,
- isCurrent: () => boolean = () => true
- ): Promise<void> {
+ private async ensureService(server: McpServerCRD): Promise<void> {
const service = this.buildService(server)
- if (!isCurrent()) return
try {
await this.coreApi.createNamespacedService({
namespace: server.namespace,
@@ -1552,7 +1276,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
namespace: server.namespace,
body,
}),
- mutationAllowed: isCurrent,
})
} else {
throw error
@@ -1565,16 +1288,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* Name matches are not enough: WRC-owned `managed:false` runtimes can share
* names with McpServer CRDs, so every delete verifies HCC ownership labels.
*/
- private async deleteResources(
- name: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ private async deleteResources(name: string, namespace: string): Promise<void> {
const failures: unknown[] = []
for (const cleanup of [
- () => this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed),
- () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
- () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
+ () => this.deleteDeploymentIfHccOwned(name, namespace),
+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name),
+ () => this.deleteServiceIfHccOwned(name, namespace),
]) {
try {
await cleanup()
@@ -1596,11 +1315,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[MCPSERVER_LABEL] === serverName
}
- private async deleteDeploymentIfHccOwned(
- name: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ private async deleteDeploymentIfHccOwned(name: string, namespace: string): Promise<void> {
let deployment: k8s.V1Deployment
try {
deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
@@ -1618,7 +1333,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(`[Reconciler] Skipping Deployment "${name}" delete — not HCC-owned`)
return
}
- if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.appsApi.deleteNamespacedDeployment({ name, namespace })
@@ -1636,8 +1350,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
private async deleteConfigMapIfHccOwned(
configMapName: string,
namespace: string,
- serverName: string,
- deleteAllowed?: () => Promise<boolean>
+ serverName: string
): Promise<void> {
let configMap: k8s.V1ConfigMap
try {
@@ -1656,7 +1369,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(`[Reconciler] Skipping ConfigMap "${configMapName}" delete — not HCC-owned`)
return
}
- if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.coreApi.deleteNamespacedConfigMap({ name: configMapName, namespace })
@@ -1671,11 +1383,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async deleteServiceIfHccOwned(
- name: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ private async deleteServiceIfHccOwned(name: string, namespace: string): Promise<void> {
let service: k8s.V1Service
try {
service = await this.coreApi.readNamespacedService({ name, namespace })
@@ -1693,7 +1401,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(`[Reconciler] Skipping Service "${name}" delete — not HCC-owned`)
return
}
- if (deleteAllowed && !(await deleteAllowed())) return
try {
await this.coreApi.deleteNamespacedService({ name, namespace })
@@ -1708,11 +1415,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async deleteNetworkPoliciesForServer(
- name: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ private async deleteNetworkPoliciesForServer(name: string, namespace: string): Promise<void> {
if (!this.networkingApi) return
const failures: unknown[] = []
const namespaces = [...new Set([namespace, config.hostNamespace, config.rpcProxyNamespace])]
@@ -1726,7 +1429,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
const policyName = policy.metadata.name
if (!policyName) continue
try {
- if (deleteAllowed && !(await deleteAllowed())) return
await this.networkingApi.deleteNamespacedNetworkPolicy({
name: policyName,
namespace: policyNamespace,
@@ -1758,19 +1460,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
throwCleanupFailures(failures, `Failed to delete NetworkPolicies for McpServer "${name}"`)
}
- private async deleteRuntimeResources(
- name: string,
- namespace: string,
- deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {
const failures: unknown[] = []
try {
- await this.deleteResources(name, namespace, deleteAllowed)
+ await this.deleteResources(name, namespace)
} catch (error) {
failures.push(error)
}
try {
- await this.deleteNetworkPoliciesForServer(name, namespace, deleteAllowed)
+ await this.deleteNetworkPoliciesForServer(name, namespace)
} catch (error) {
failures.push(error)
}
@@ -1783,17 +1481,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* G2: Set `clerum.io/network-ready: "true"` annotation on McpServer CRD.
* Completes the pre-deploy handshake so WRC knows NetworkPolicies are applied.
*/
- private async setNetworkReadyAnnotation(
- server: McpServerCRD,
- isCurrent: () => boolean = () => true
- ): Promise<void> {
+ private async setNetworkReadyAnnotation(server: McpServerCRD): Promise<void> {
// Use merge-patch (plain object body) instead of JSON Patch (array body).
// JSON Patch op:"add" on /metadata/annotations/<key> requires /metadata/annotations
// to already exist — it fails with 422 when the CRD was created without annotations.
// Merge-patch creates the map if absent. Errors are re-thrown so the caller (reconcile)
// knows the handshake was not completed — WRC will time out and fail rather than proceed
// silently without NetworkPolicies in place.
- if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObject(
{
group: 'clerum.io',
@@ -1815,6 +1509,74 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
}
+ /**
+ * Keep remote egress proxy CRDs aligned with the platform-owned runtime image.
+ *
+ * Remote servers are rendered as nginx egress proxies and never run
+ * `server.spec.image`, but stale CR specs can mislead scanners or future
+ * tooling. Patch only the declarative image field; Deployment rendering still
+ * uses `config.egressProxyImage` as the authority.
+ *
+ * This stays internally non-throwing because image canonicalization is
+ * best-effort cleanup; a failed CRD patch must not block ConfigMap,
+ * Service, or Deployment reconciliation for the actual proxy runtime.
+ */
+ private async canonicalizeRemoteEgressProxyImage(server: McpServerCRD): Promise<void> {
+ if (!this.isRemote(server)) {
+ return
+ }
+ if (server.spec.image === config.egressProxyImage) {
+ await this.writeStatusCondition(server, {
+ type: 'ImageCanonicalized',
+ status: 'True',
+ reason: 'RemoteEgressProxyImageMatches',
+ message: 'Remote McpServer spec.image matches the platform egress proxy image',
+ })
+ return
+ }
+
+ try {
+ await this.customApi.patchNamespacedCustomObject(
+ {
+ group: 'clerum.io',
+ version: 'v1alpha1',
+ namespace: server.namespace,
+ plural: 'mcpservers',
+ name: server.name,
+ body: {
+ spec: {
+ image: config.egressProxyImage,
+ },
+ },
+ },
+ {
+ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')],
+ }
+ )
+ server.spec.image = config.egressProxyImage
+ console.log(
+ `[Reconciler] Canonicalized remote McpServer "${server.name}" image to ${config.egressProxyImage}`
+ )
+ await this.writeStatusCondition(server, {
+ type: 'ImageCanonicalized',
+ status: 'True',
+ reason: 'RemoteEgressProxyImageMatches',
+ message: 'Remote McpServer spec.image matches the platform egress proxy image',
+ })
+ } catch (err) {
+ console.warn(
+ `[Reconciler] Failed to canonicalize remote McpServer "${server.name}" image to ${config.egressProxyImage}:`,
+ err
+ )
+ await this.writeStatusCondition(server, {
+ type: 'ImageCanonicalized',
+ status: 'False',
+ reason: 'RemoteEgressProxyImagePatchFailed',
+ message: 'Failed to patch remote McpServer spec.image to the platform egress proxy image',
+ })
+ }
+ }
+
/**
* G11: Update status conditions on McpServer CRD.
* Sets NetworkReady and DeploymentReady conditions so WRC can watch
@@ -1828,38 +1590,29 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* message so a stuck rollout reports WHAT is stuck (issue #223). Callers
* that only know the verdict omit it.
*/
- rolloutDetail?: string,
- isCurrent: () => boolean = () => true
+ rolloutDetail?: string
): Promise<void> {
try {
- await this.writeStatusCondition(
- server,
- {
- type: 'NetworkReady',
- status: 'True',
- reason: 'NetworkPoliciesApplied',
- message: 'NetworkPolicies and Service created',
- },
- isCurrent
- )
- await this.writeStatusCondition(
- server,
- {
- type: 'DeploymentReady',
- status: deploymentReady ? 'True' : 'False',
- reason: deploymentReady
- ? 'ReplicasAvailable'
- : rolloutDetail
- ? 'RolloutIncomplete'
- : 'WaitingForReplicas',
- message: rolloutDetail
- ? `Rollout did not converge — ${rolloutDetail}`
- : deploymentReady
- ? 'Deployment has ready replicas'
- : 'Waiting for pods to become ready',
- },
- isCurrent
- )
+ await this.writeStatusCondition(server, {
+ type: 'NetworkReady',
+ status: 'True',
+ reason: 'NetworkPoliciesApplied',
+ message: 'NetworkPolicies and Service created',
+ })
+ await this.writeStatusCondition(server, {
+ type: 'DeploymentReady',
+ status: deploymentReady ? 'True' : 'False',
+ reason: deploymentReady
+ ? 'ReplicasAvailable'
+ : rolloutDetail
+ ? 'RolloutIncomplete'
+ : 'WaitingForReplicas',
+ message: rolloutDetail
+ ? `Rollout did not converge — ${rolloutDetail}`
+ : deploymentReady
+ ? 'Deployment has ready replicas'
+ : 'Waiting for pods to become ready',
+ })
console.log(
`[Reconciler] Updated status conditions on "${server.name}" (DeploymentReady=${deploymentReady})`
)
@@ -1884,29 +1637,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
*
* Failure modes:
* - 404 (CRD deleted mid-reconcile) → log + swallow.
- * - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.
* - Other errors → log + swallow (status writes are best-effort).
*/
async writeStatusCondition(
server: McpServerCRD,
- condition: Omit<McpServerCondition, 'lastTransitionTime'>,
- isCurrent: () => boolean = () => true
+ condition: Omit<McpServerCondition, 'lastTransitionTime'>
): Promise<void> {
- if (!isCurrent()) return
- const now = new Date().toISOString()
-
- // Update the missing-secret gauge for SecretResolved conditions.
- // Do this before any no-op short-circuit so the gauge stays current
- // across reconciles even when we skip the API patch.
- if (condition.type === 'SecretResolved' && server.spec.envSecret) {
- const labels = {
- namespace: server.namespace,
- name: server.name,
- secret_name: server.spec.envSecret.name,
- }
- mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)
- }
-
// Optimistic-concurrency loop (M2). The patch REPLACES the whole
// /status/conditions array with a value built from a snapshot read just
// before. Without a version guard, a concurrent writer — the readiness poll
@@ -1916,11 +1652,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// op on metadata.resourceVersion makes the API server reject the patch (422)
// if anything changed since our read; we then re-read and retry, so no
// writer's condition is lost.
- for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {
- if (!isCurrent()) return
- // Re-read on every optimistic-conflict retry. Merging against the
- // latest resourceVersion preserves conditions written by peer
- // controllers between our read and patch.
+ const MAX_ATTEMPTS = 4
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+ const now = new Date().toISOString()
+
+ // Fetch current status (and its resourceVersion) to preserve existing
+ // conditions and to fence the write below.
let existingConditions: McpServerCondition[] = []
let hasStatusObject = false
let resourceVersion: string | undefined
@@ -1935,9 +1672,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
metadata?: { resourceVersion?: string }
status?: { conditions?: McpServerCondition[] }
}
+ resourceVersion = current.metadata?.resourceVersion
hasStatusObject = typeof current.status === 'object' && current.status !== null
existingConditions = current.status?.conditions ?? []
- resourceVersion = current.metadata?.resourceVersion
} catch (error) {
if (getErrorCode(error) === 404) {
console.warn(
@@ -1951,16 +1688,26 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
return
}
- if (!isCurrent()) return
const prior = existingConditions.find(c => c.type === condition.type)
- const observedGeneration = condition.observedGeneration ?? server.generation
const unchanged =
prior !== undefined &&
prior.status === condition.status &&
prior.reason === condition.reason &&
- prior.message === condition.message &&
- prior.observedGeneration === observedGeneration
+ prior.message === condition.message
+
+ // Update the missing-secret gauge for SecretResolved conditions. Do this
+ // BEFORE the no-op short-circuit so the gauge stays current across
+ // reconciles even when we skip the API patch. Idempotent, so a retry
+ // re-running it is harmless.
+ if (condition.type === 'SecretResolved' && server.spec.envSecret) {
+ const labels = {
+ namespace: server.namespace,
+ name: server.name,
+ secret_name: server.spec.envSecret.name,
+ }
+ mcpserverMissingSecret.set(labels, condition.status === 'False' ? 1 : 0)
+ }
// Skip the patch when the condition is identical to what's already on
// the CRD. Without this, every reconcile bumps the McpServer's
@@ -1980,49 +1727,26 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
reason: condition.reason,
message: condition.message,
lastTransitionTime,
- ...(observedGeneration !== undefined && { observedGeneration }),
}
const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]
- const writePreconditions: Array<{
- op: 'test'
- path: string
- value: string | number
- }> = []
- if (server.uid !== undefined) {
- writePreconditions.push({
- op: 'test',
- path: '/metadata/uid',
- value: server.uid,
- })
- }
- if (server.generation !== undefined) {
- writePreconditions.push({
- op: 'test',
- path: '/metadata/generation',
- value: server.generation,
- })
- }
- if (resourceVersion !== undefined) {
- writePreconditions.push({
- op: 'test',
- path: '/metadata/resourceVersion',
- value: resourceVersion,
- })
- }
+ // The `test` op fences the write to the exact revision we read. If a
+ // concurrent writer bumped resourceVersion in between, the API server
+ // fails the patch (422) and we retry with a fresh read.
const statusPatch = [
- ...writePreconditions,
- ...(hasStatusObject
- ? [{ op: 'add', path: '/status/conditions', value: nextConditions }]
- : [{ op: 'add', path: '/status', value: { conditions: nextConditions } }]),
+ ...(resourceVersion
+ ? [{ op: 'test', path: '/metadata/resourceVersion', value: resourceVersion }]
+ : []),
+ hasStatusObject
+ ? { op: 'add', path: '/status/conditions', value: nextConditions }
+ : { op: 'add', path: '/status', value: { conditions: nextConditions } },
]
try {
// The generated client prefers JSON Patch for this endpoint. Send an
// actual patch document so status writes keep working across client
// upgrades instead of relying on merge-patch object bodies.
- if (!isCurrent()) return
await this.customApi.patchNamespacedCustomObjectStatus({
group: 'clerum.io',
version: 'v1alpha1',
@@ -2033,25 +1757,22 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
})
return
} catch (error) {
- const errorCode = getErrorCode(error)
- if (errorCode === 404) {
+ if (getErrorCode(error) === 404) {
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`
)
return
}
- // 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer
- // won the race. Re-read and retry rather than lose our condition. Any
- // other error, or the last attempt, falls through to the best-effort
- // log-and-swallow below.
- if (
- (errorCode === 409 || errorCode === 422) &&
- attempt < STATUS_CONDITION_WRITE_MAX_ATTEMPTS
- ) {
+ // 409 (Conflict) or 422 (the resourceVersion `test` op failed) mean a
+ // concurrent writer won the race. Re-read and retry rather than lose
+ // our condition. Any other error, or the last attempt, falls through to
+ // the best-effort log-and-swallow below.
+ const code = getErrorCode(error)
+ if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) {
continue
}
console.warn(
- `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,
+ `[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${MAX_ATTEMPTS}):`,
error
)
return
@@ -2069,24 +1790,10 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* A prior failure does not block the next call; the chain absorbs it so
* the next reconcile still runs.
*/
- async reconcile(server: McpServerCRD, options: McpServerMutationOptions = {}): Promise<void> {
- const capturedAuthority = this.inventoryAuthority()
- const callerIsCurrent = options.isCurrent ?? (() => true)
- const isCurrent = () => {
- if (!callerIsCurrent()) return false
- if (!this.inventoryAuthorityConfigured) return this.currentWhenUnconfigured()
- if (!this.authorityMatches(capturedAuthority)) return false
- const current = this.resolveCurrentServer?.(server.name)
- return current !== undefined && sameMcpServerDesiredRevision(server, current)
- }
+ async reconcile(server: McpServerCRD): Promise<void> {
const key = server.name
const prev = this.inFlight.get(key) ?? Promise.resolve()
- const next = prev
- .catch(() => undefined)
- .then(async () => {
- if (!isCurrent()) return
- await this.performReconcile(server, isCurrent)
- })
+ const next = prev.catch(() => undefined).then(() => this.performReconcile(server))
this.inFlight.set(key, next)
try {
await next
@@ -2097,12 +1804,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
}
- private async performReconcile(server: McpServerCRD, isCurrent: () => boolean): Promise<void> {
- if (!isCurrent()) return
- this.beginStatusTracking(server)
-
+ private async performReconcile(server: McpServerCRD): Promise<void> {
// G7: Enforce managed field immutability — once set, cannot change
- if (!isCurrent()) return
const currentManaged = server.spec.managed ?? true
const previousManaged = this.managedSnapshot.get(server.name)
if (previousManaged !== undefined && previousManaged !== currentManaged) {
@@ -2110,7 +1813,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
`[Reconciler] McpServer "${server.name}": managed field changed from ${previousManaged} to ${currentManaged}. ` +
`This is not allowed — delete and recreate the McpServer to change ownership.`
)
- if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -2118,7 +1820,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
})
return
}
- if (!isCurrent()) return
this.managedSnapshot.set(server.name, currentManaged)
// If disabled, respect ownership before any cleanup. managed:false means
@@ -2128,72 +1829,57 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.log(
`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`
)
- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
+ await this.deleteRuntimeResources(server.name, server.namespace)
} else {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`
)
}
- if (!isCurrent()) return
this.setStatus(server.name, { deployed: false, ready: false, message: 'Disabled' })
- await this.writeStatusCondition(
- server,
- {
- type: 'Ready',
- status: 'False',
- reason: 'Disabled',
- message: 'McpServer is disabled',
- },
- isCurrent
- )
+ await this.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'False',
+ reason: 'Disabled',
+ message: 'McpServer is disabled',
+ })
return
}
if (!currentManaged) {
- await this.reconcileWrcOwnedServer(server, isCurrent)
+ await this.reconcileWrcOwnedServer(server)
return
}
// Validate secret before creating/updating
const secretResult = await this.validateSecret(server)
- if (!isCurrent()) return
if (!secretResult.ok) {
console.error(
`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
)
if (this.shouldFailClosedForSecretFailure(secretResult)) {
- await this.deleteRuntimeResources(server.name, server.namespace, async () => isCurrent())
+ await this.deleteRuntimeResources(server.name, server.namespace)
} else {
console.warn(
`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`
)
}
- if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
message: secretResult.message,
})
- await this.writeStatusCondition(
- server,
- {
- type: 'SecretResolved',
- status: 'False',
- reason: secretResult.reason,
- message: secretResult.message,
- },
- isCurrent
- )
- await this.writeStatusCondition(
- server,
- {
- type: 'Ready',
- status: 'False',
- reason: 'SecretValidationFailed',
- message: secretResult.message,
- },
- isCurrent
- )
+ await this.writeStatusCondition(server, {
+ type: 'SecretResolved',
+ status: 'False',
+ reason: secretResult.reason,
+ message: secretResult.message,
+ })
+ await this.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'False',
+ reason: 'SecretValidationFailed',
+ message: secretResult.message,
+ })
return
}
@@ -2201,11 +1887,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
try {
// Remote egress proxy: create nginx ConfigMap before Deployment
if (this.isRemote(server)) {
- // The CRD image remains desired-state input owned by its author. Remote
- // workloads use the platform-owned proxy image selected in
- // buildDeployment(); rewriting spec.image here would bump generation
- // and retire this reconcile before it can create the runtime resources.
- await this.ensureConfigMap(server, isCurrent)
+ await this.canonicalizeRemoteEgressProxyImage(server)
+ await this.ensureConfigMap(server)
} else {
// Phase 2.3: plugin image-host allowlist. Audit mode (default) logs a
// would-be denial and still builds; enforce mode blocks the workload.
@@ -2217,17 +1900,12 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
const message = `Image "${server.spec.image}" is not permitted by the plugin image allowlist (${decision.reason})`
if (config.enforcePluginImageAllowlist) {
console.warn(`[Reconciler] ${message} — blocking "${server.name}" (enforce mode)`)
- await this.writeStatusCondition(
- server,
- {
- type: 'Ready',
- status: 'False',
- reason: 'ImageNotAllowed',
- message,
- },
- isCurrent
- )
- if (!isCurrent()) return
+ await this.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'False',
+ reason: 'ImageNotAllowed',
+ message,
+ })
this.setStatus(server.name, { deployed: false, ready: false, message })
return
}
@@ -2236,12 +1914,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
}
}
- if (!isCurrent()) return
- await this.ensureService(server, isCurrent)
- if (!isCurrent()) return
- await this.ensureDeployment(server, secretResult.revision, isCurrent)
+ await this.ensureService(server)
+ await this.ensureDeployment(server, secretResult.revision)
} catch (err) {
- if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -2258,9 +1933,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
server.annotations?.[NETWORK_READY_ANNOTATION] !== 'true'
) {
try {
- await this.setNetworkReadyAnnotation(server, isCurrent)
+ await this.setNetworkReadyAnnotation(server)
} catch (err) {
- if (!isCurrent()) return
this.setStatus(server.name, {
deployed: false,
ready: false,
@@ -2272,7 +1946,6 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// Check readiness after reconciliation
const rollout = await this.readDeploymentRollout(server.name, server.namespace)
- if (!isCurrent()) return
const ready = rollout.ready
this.setStatus(server.name, {
deployed: true,
@@ -2297,36 +1970,28 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
pollState.generation === rollout.generation
? pollState.detail
: undefined
- await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)
+ await this.updateStatusConditions(server, ready, exhaustedDetail)
// PR-B B1: record SecretResolved=True + Ready=True after a successful
// reconcile. This is additive to updateStatusConditions (NetworkReady /
// DeploymentReady); the merge logic in writeStatusCondition preserves
// the existing conditions by type.
if (server.spec.envSecret) {
- await this.writeStatusCondition(
- server,
- {
- type: 'SecretResolved',
- status: 'True',
- reason: 'SecretFound',
- message: 'Secret resolved and validated',
- },
- isCurrent
- )
+ await this.writeStatusCondition(server, {
+ type: 'SecretResolved',
+ status: 'True',
+ reason: 'SecretFound',
+ message: 'Secret resolved and validated',
+ })
}
- await this.writeStatusCondition(
- server,
- {
- type: 'Ready',
- status: ready ? 'True' : 'Unknown',
- reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',
- message: ready
- ? 'Deployment created'
- : 'Deployment created, waiting for pods to become ready',
- },
- isCurrent
- )
+ await this.writeStatusCondition(server, {
+ type: 'Ready',
+ status: ready ? 'True' : 'Unknown',
+ reason: ready ? 'ReconcileSuccess' : 'WaitingForReplicas',
+ message: ready
+ ? 'Deployment created'
+ : 'Deployment created, waiting for pods to become ready',
+ })
// If not ready yet, poll until it becomes ready. With generation-aware
// readiness this is the normal path right after any Deployment write —
@@ -2343,8 +2008,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// is a deployment whose replica tuple perpetually OSCILLATES, and that is
// bounded to one reconcile per poll window (~120s) — not a tight loop, and (per
// pollReadiness's atomic check-then-set) not a timer leak.
- if (!ready && isCurrent()) {
- this.pollReadiness(server, 5000, 24, isCurrent)
+ if (!ready) {
+ this.pollReadiness(server)
}
}
@@ -2358,18 +2023,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
)
}
- private async reconcileWrcOwnedServer(
- server: McpServerCRD,
- isCurrent: () => boolean
- ): Promise<void> {
- if (!isCurrent()) return
+ private async reconcileWrcOwnedServer(server: McpServerCRD): Promise<void> {
console.log(
`[Reconciler] McpServer "${server.name}" is WRC-owned (managed:false); ` +
'skipping HCC runtime creation/deletion and updating discovery status.'
)
const secretResult = await this.validateSecret(server)
- if (!isCurrent()) return
if (!secretResult.ok) {
this.setStatus(server.name, {
deployed: true,
@@ -2377,58 +2037,41 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
message: secretResult.message,
})
if (server.spec.envSecret) {
- await this.writeStatusCondition(
- server,
- {
- type: 'SecretResolved',
- status: 'False',
- reason: secretResult.reason,
- message: secretResult.message,
- },
- isCurrent
- )
- }
- await this.writeStatusCondition(
- server,
- {
- type: 'Ready',
+ await this.writeStatusCondition(server, {
+ type: 'SecretResolved',
status: 'False',
- reason: 'SecretValidationFailed',
+ reason: secretResult.reason,
message: secretResult.message,
- },
- isCurrent
- )
+ })
+ }
+ await this.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'False',
+ reason: 'SecretValidationFailed',
+ message: secretResult.message,
+ })
return
}
- if (!isCurrent()) return
this.setStatus(server.name, {
deployed: true,
ready: true,
message: 'WRC-owned runtime registered',
})
if (server.spec.envSecret) {
- await this.writeStatusCondition(
- server,
- {
- type: 'SecretResolved',
- status: 'True',
- reason: 'SecretFound',
- message: 'Secret resolved and validated',
- },
- isCurrent
- )
- }
- await this.writeStatusCondition(
- server,
- {
- type: 'Ready',
+ await this.writeStatusCondition(server, {
+ type: 'SecretResolved',
status: 'True',
- reason: WRC_OWNED_RUNTIME_READY_REASON,
- message: 'WRC-owned runtime registered for HCC discovery',
- },
- isCurrent
- )
+ reason: 'SecretFound',
+ message: 'Secret resolved and validated',
+ })
+ }
+ await this.writeStatusCondition(server, {
+ type: 'Ready',
+ status: 'True',
+ reason: WRC_OWNED_RUNTIME_READY_REASON,
+ message: 'WRC-owned runtime registered for HCC discovery',
+ })
}
/**
@@ -2436,55 +2079,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
*/
async reconcileDelete(name: string, namespace: string): Promise<void> {
console.log(`[Reconciler] McpServer "${name}" deleted — cleaning up resources`)
- const capturedAuthority = this.inventoryAuthority()
- await this.deleteRuntimeResources(name, namespace, () =>
- this.orphanDeleteAllowed(name, namespace, capturedAuthority)
- )
+ await this.deleteRuntimeResources(name, namespace)
this.clearStatus(name)
}
- private authorityMatches(captured: McpServerInventoryAuthoritySnapshot): boolean {
- const current = this.inventoryAuthority()
- return captured.known && current.known && captured.generation === current.generation
- }
-
- /**
- * A cache miss is not proof of absence: the API watch may have accepted an
- * ADDED event that has not reached this process yet. Confirm a 404 from the
- * API and then recheck both cache membership and LIST→WATCH authority
- * immediately before every destructive delete.
- */
- private async orphanDeleteAllowed(
- name: string,
- namespace: string,
- capturedAuthority: McpServerInventoryAuthoritySnapshot
- ): Promise<boolean> {
- return confirmAuthoritativeMcpServerAbsence({
- inventoryAuthoritative: () => this.authorityMatches(capturedAuthority),
- resolveCurrent: () => this.resolveCurrentServer?.(name),
- readCurrent: () =>
- this.customApi.getNamespacedCustomObject({
- group: 'clerum.io',
- version: 'v1alpha1',
- namespace,
- plural: 'mcpservers',
- name,
- }),
- })
- }
-
/**
* Full reconciliation pass — sync desired state with actual state.
* Called on startup to handle CRDs that were created while the skill-mapper was down.
*/
- async fullReconcile(
- desiredServers: McpServerCRD[],
- options: McpServerFullReconcileOptions = {}
- ): Promise<void> {
+ async fullReconcile(desiredServers: McpServerCRD[]): Promise<void> {
console.log(
`[Reconciler] Running full reconciliation for ${desiredServers.length} McpServer(s)`
)
- const capturedAuthority = this.inventoryAuthority()
// Get all deployments managed by skill-mapper
const existingDeployments = await this.listManagedDeployments()
@@ -2494,62 +2100,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
.map(s => s.name)
)
- const runEffect =
- options.runEffect ?? ((_serverName: string, work: () => Promise<void>) => work())
- const maxConcurrency = options.maxConcurrency ?? DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY
-
- // Create or update desired servers independently. runEffect owns each
- // per-server lane, so a slow fleet member cannot become a process-wide
- // startup barrier. The finite worker pool admits no more than the configured
- // number of effects at once, while still waiting for every desired result
- // before reporting aggregated failures or moving to orphan cleanup.
- const desiredResults = await mapSettledWithConcurrency(desiredServers, maxConcurrency, server =>
- runEffect(server.name, async () => {
- const current = this.resolveCurrentServer ? this.resolveCurrentServer(server.name) : server
- // Cache mutation happens synchronously before its keyed watch effect is
- // queued. If this startup item was superseded, skip it and let the
- // newer queued event perform external-egress-before-runtime using its
- // own spec; reconciling the newer runtime here could otherwise race
- // ahead of that safety policy.
- if (!current || !sameMcpServerDesiredRevision(server, current)) return
- const isCurrent = () => {
- if (!this.inventoryAuthorityConfigured) {
- if (!this.currentWhenUnconfigured()) return false
- } else if (!this.authorityMatches(capturedAuthority)) {
- return false
- }
- const latest = this.resolveCurrentServer?.(current.name)
- return latest
- ? sameMcpServerDesiredRevision(current, latest)
- : this.resolveCurrentServer === undefined && this.currentWhenUnconfigured()
- }
- await this.reconcile(current, { isCurrent })
- })
- )
- throwCleanupFailures(
- desiredResults
- .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
- .map(result => result.reason),
- 'Failed to reconcile desired McpServers'
- )
+ // Create or update desired servers
+ for (const server of desiredServers) {
+ await this.reconcile(server)
+ }
// Delete orphaned deployments (managed by us but no longer desired)
for (const deployment of existingDeployments) {
const name = deployment.metadata?.name || ''
if (!desiredNames.has(name)) {
const namespace = deployment.metadata?.namespace || config.namespace
- await runEffect(name, async () => {
- if (!(await this.orphanDeleteAllowed(name, namespace, capturedAuthority))) {
- console.warn(
- `[Reconciler] Deferring orphan cleanup for "${name}": McpServer exists or inventory authority is unknown or changed`
- )
- return
- }
- console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
- await this.deleteRuntimeResources(name, namespace, () =>
- this.orphanDeleteAllowed(name, namespace, capturedAuthority)
- )
- })
+ console.log(`[Reconciler] Removing orphaned resources for "${name}"`)
+ await this.deleteRuntimeResources(name, namespace)
}
}
@@ -2568,7 +2130,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
return response.items || []
} catch (error) {
console.error('[Reconciler] Failed to list managed deployments:', error)
- throw error
+ return []
}
}
}
diff --git a/host-context-controller/src/server.test.ts b/host-context-controller/src/server.test.ts
index 0ab7d03aa..633a48992 100644
--- a/host-context-controller/src/server.test.ts
+++ b/host-context-controller/src/server.test.ts
@@ -1,6 +1,5 @@
-import { afterEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, describe, expect, it } from 'vitest'
import { McpServerProvider } from './k8sClient'
-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
import { ContextMapperServer } from './server'
class FakeProvider implements McpServerProvider {
@@ -76,30 +75,14 @@ describe('ContextMapperServer readiness', () => {
afterEach(async () => {
await server?.stop()
server = null
- vi.restoreAllMocks()
- })
-
- it('withholds readiness when no inventory-authority gate is wired', async () => {
- // The gate decides whether the provider's inventory is authoritative. A
- // caller that forgets to wire it must not get a permanently Ready server:
- // this endpoint is the contract that no stale allow is live, so the default
- // has to fail closed. There is no type error to catch the omission.
- server = new ContextMapperServer(new FakeProvider(), 0)
- server.setReady(true)
-
- const response = await invoke(server, '/ready')
- expect(response.statusCode).toBe(503)
})
it('serves health immediately but gates readiness and API responses until warm-up completes', async () => {
- server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
+ server = new ContextMapperServer(new FakeProvider(), 0)
let response = await invoke(server, '/health')
expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toEqual({
- status: 'ok',
- ready: false,
- })
+ expect(JSON.parse(response.body)).toEqual({ status: 'ok', ready: false })
response = await invoke(server, '/ready')
expect(response.statusCode).toBe(503)
@@ -125,188 +108,4 @@ describe('ContextMapperServer readiness', () => {
contextRef: '*',
})
})
-
- it('tracks provider authority dynamically after warm-up and fails API requests closed', async () => {
- let providerAuthoritative = true
- server = new ContextMapperServer(
- new FakeProvider(),
- 0,
- undefined,
- undefined,
- () => providerAuthoritative
- )
-
- let response = await invoke(server, '/ready')
- expect(response.statusCode).toBe(503)
- expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })
-
- server.setReady(true)
-
- response = await invoke(server, '/ready')
- expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
-
- providerAuthoritative = false
-
- response = await invoke(server, '/ready')
- expect(response.statusCode).toBe(503)
- expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
-
- response = await invoke(server, '/health')
- expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toEqual({
- status: 'ok',
- ready: false,
- })
-
- response = await invoke(server, '/api/v1/mcpservers')
- expect(response.statusCode).toBe(503)
- expect(JSON.parse(response.body)).toEqual({
- error: 'Service Unavailable',
- message: 'Context mapper provider inventory is not authoritative',
- })
-
- providerAuthoritative = true
-
- response = await invoke(server, '/ready')
- expect(response.statusCode).toBe(200)
- expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
-
- response = await invoke(server, '/api/v1/mcpservers')
- expect(response.statusCode).toBe(200)
- })
-
- it('fails readiness closed when the provider authority check throws', async () => {
- const authorityError = new Error('authority unavailable')
- const errorLog = vi.spyOn(console, 'error').mockImplementation(() => undefined)
- server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => {
- throw authorityError
- })
- server.setReady(true)
-
- const response = await invoke(server, '/ready')
-
- expect(response.statusCode).toBe(503)
- expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
- expect(errorLog).toHaveBeenCalledWith(
- '[Server] Provider authority readiness check failed:',
- authorityError
- )
- })
-
- it('withdraws readiness as soon as shutdown begins', async () => {
- server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => true)
- server.setReady(true)
-
- expect((await invoke(server, '/ready')).statusCode).toBe(200)
-
- await server.stop()
-
- const response = await invoke(server, '/ready')
- expect(response.statusCode).toBe(503)
- expect(JSON.parse(response.body)).toEqual({ status: 'starting', ready: false })
- })
-})
-
-describe('ContextMapperServer desktop status authority', () => {
- let server: ContextMapperServer | null = null
-
- // A Host reconciler that reports a running desktop, so the only thing that can
- // withhold a 200 is the Host-authority gate under test.
- const runningReconciler = {
- getStatus: () => ({ deployed: true, ready: true, message: '' }),
- hasDesktop: () => true,
- } as unknown as ConstructorParameters<typeof ContextMapperServer>[2]
- const hasDesktopFn = () => true
-
- afterEach(async () => {
- await server?.stop()
- server = null
- vi.restoreAllMocks()
- })
-
- it('serves desktop status on Host authority alone even when the provider inventory is not authoritative', async () => {
- // The heart of H1: desktop needs only Host authority. A degraded
- // McpServer/Context lane (providerAuthoritative=false) must NOT 503 desktop,
- // while /api/v1/mcpservers still 503s on the same server.
- server = new ContextMapperServer(
- new FakeProvider(),
- 0,
- runningReconciler,
- hasDesktopFn,
- () => false, // provider inventory NOT authoritative
- () => true // Host inventory authoritative
- )
- server.setReady(true)
-
- const desktop = await invoke(server, '/api/v1/desktop/host-a')
- expect(desktop.statusCode).toBe(200)
- expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })
-
- const mcpservers = await invoke(server, '/api/v1/mcpservers')
- expect(mcpservers.statusCode).toBe(503)
- })
-
- it('fails desktop closed (503, never 200 inactive) when Host inventory is not authoritative', async () => {
- server = new ContextMapperServer(
- new FakeProvider(),
- 0,
- runningReconciler,
- hasDesktopFn,
- () => true, // provider authoritative — must not rescue desktop
- () => false // Host inventory NOT authoritative
- )
- server.setReady(true)
-
- const desktop = await invoke(server, '/api/v1/desktop/host-a')
- expect(desktop.statusCode).toBe(503)
- expect(JSON.parse(desktop.body)).toEqual({
- error: 'Service Unavailable',
- message: 'Host inventory is not authoritative',
- })
- })
-
- it('serves desktop status 200 in the dev wiring (no watcher), never 503 (R2-M1)', async () => {
- // Wiring regression for R2-M1: main.ts resolves both authority gates from the
- // same watcher. In dev (DevMcpServerProvider, watcher=null) the OLD wiring left
- // the Host gate undefined, so the server's fail-closed default pinned every
- // /api/v1/desktop/* at 503 after setReady(true) — where the base answered 200
- // {status:'inactive'}. Exercise the REAL resolvers with a null watcher, exactly
- // as main.ts does, so a revert to the undefined wiring turns this 200 into 503.
- const providerAuthoritativeFn = resolveProviderAuthoritativeFn(null)
- const hostAuthoritativeFn = resolveHostAuthoritativeFn(null)
- server = new ContextMapperServer(
- new FakeProvider(),
- 0,
- runningReconciler,
- hasDesktopFn,
- providerAuthoritativeFn,
- hostAuthoritativeFn
- )
- server.setReady(true)
-
- const desktop = await invoke(server, '/api/v1/desktop/host-a')
- expect(desktop.statusCode).toBe(200)
- expect(JSON.parse(desktop.body)).toEqual({ status: 'running', hostRef: 'host-a' })
- })
-
- it('fails desktop closed when no Host-authority gate is wired (default)', async () => {
- // Omitting the 6th arg must default to fail-closed, exactly like the
- // provider gate — no type error catches the omission.
- server = new ContextMapperServer(
- new FakeProvider(),
- 0,
- runningReconciler,
- hasDesktopFn,
- () => true
- )
- server.setReady(true)
-
- const desktop = await invoke(server, '/api/v1/desktop/host-a')
- expect(desktop.statusCode).toBe(503)
- expect(JSON.parse(desktop.body)).toEqual({
- error: 'Service Unavailable',
- message: 'Host inventory is not authoritative',
- })
- })
})
diff --git a/host-context-controller/src/server.ts b/host-context-controller/src/server.ts
index 03f576dcf..7be800a8e 100644
--- a/host-context-controller/src/server.ts
+++ b/host-context-controller/src/server.ts
@@ -8,53 +8,24 @@ import { McpServerProvider } from './k8sClient'
import { registry } from './metrics'
import { ErrorResponse, McpServersResponse } from './types'
-type ReadinessState =
- | { ready: true; status: 'ready' }
- | { ready: false; status: 'starting' | 'degraded'; message: string }
-
export class ContextMapperServer {
private server: http.Server | null = null
private provider: McpServerProvider
private port: number
private hostReconciler: HostReconciler | null
private hasDesktopFn: ((hostRef: string) => boolean) | null
- private providerAuthoritativeFn: () => boolean
- private hostAuthoritativeFn: () => boolean
private ready = false
constructor(
provider: McpServerProvider,
port: number = config.port,
hostReconciler?: HostReconciler,
- hasDesktopFn?: (hostRef: string) => boolean,
- // Fails closed. /ready is the assertion that no stale allow is live, and a
- // caller that omits this gate gets no type error, so the default must
- // withhold readiness rather than grant it.
- providerAuthoritativeFn: () => boolean = () => false,
- // Desktop status needs ONLY Host inventory authority, not the full
- // readiness inventory. Fails closed like providerAuthoritativeFn: a caller
- // that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.
- hostAuthoritativeFn: () => boolean = () => false
+ hasDesktopFn?: (hostRef: string) => boolean
) {
this.provider = provider
this.port = port
this.hostReconciler = hostReconciler ?? null
this.hasDesktopFn = hasDesktopFn ?? null
- this.providerAuthoritativeFn = providerAuthoritativeFn
- this.hostAuthoritativeFn = hostAuthoritativeFn
- }
-
- /**
- * Host inventory authority, fail-closed on error — the desktop route's only
- * authority gate. Same shape as the provider check in getReadinessState().
- */
- private safeHostAuthoritative(): boolean {
- try {
- return this.hostAuthoritativeFn()
- } catch (err) {
- console.error('[Server] Host authority check failed:', err)
- return false
- }
}
/**
@@ -64,30 +35,6 @@ export class ContextMapperServer {
this.ready = ready
}
- private getReadinessState(): ReadinessState {
- if (!this.ready) {
- return {
- ready: false,
- status: 'starting',
- message: 'Context mapper is still starting',
- }
- }
-
- try {
- if (this.providerAuthoritativeFn()) {
- return { ready: true, status: 'ready' }
- }
- } catch (err) {
- console.error('[Server] Provider authority readiness check failed:', err)
- }
-
- return {
- ready: false,
- status: 'degraded',
- message: 'Context mapper provider inventory is not authoritative',
- }
- }
-
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// CORS headers
res.setHeader('Access-Control-Allow-Origin', '*')
@@ -110,20 +57,15 @@ export class ContextMapperServer {
// Health check
if (req.method === 'GET' && url.pathname === '/health') {
- const readiness = this.getReadinessState()
- this.sendJson(res, 200, {
- status: 'ok',
- ready: readiness.ready,
- })
+ this.sendJson(res, 200, { status: 'ok', ready: this.ready })
return
}
// Readiness check
if (req.method === 'GET' && url.pathname === '/ready') {
- const readiness = this.getReadinessState()
- this.sendJson(res, readiness.ready ? 200 : 503, {
- status: readiness.status,
- ready: readiness.ready,
+ this.sendJson(res, this.ready ? 200 : 503, {
+ status: this.ready ? 'ready' : 'starting',
+ ready: this.ready,
})
return
}
@@ -136,30 +78,10 @@ export class ContextMapperServer {
return
}
- // Desktop status: Host inventory is the ONLY authority this route needs, so
- // it is gated here — ABOVE the blanket readiness gate — on Host authority
- // alone. A degraded McpServer/Context lane must NOT 503 desktop, and a
- // degraded Host lane must 503 it (never answer 200 'inactive' from a stale
- // cache). See the E3 design note's endpoint→authority matrix.
- if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {
- if (!this.ready || !this.safeHostAuthoritative()) {
- this.sendJson(res, 503, {
- error: 'Service Unavailable',
- message: 'Host inventory is not authoritative',
- })
- return
- }
- if (!this.checkDesktopAuth(req, res)) return
- const hostRef = url.pathname.replace('/api/v1/desktop/', '')
- await this.handleDesktopStatus(res, hostRef)
- return
- }
-
- const readiness = this.getReadinessState()
- if (!readiness.ready) {
+ if (!this.ready) {
this.sendJson(res, 503, {
error: 'Service Unavailable',
- message: readiness.message,
+ message: 'Context mapper is still starting',
})
return
}
@@ -188,6 +110,14 @@ export class ContextMapperServer {
return
}
+ // Desktop status (requires service token)
+ if (req.method === 'GET' && url.pathname.startsWith('/api/v1/desktop/')) {
+ if (!this.checkDesktopAuth(req, res)) return
+ const hostRef = url.pathname.replace('/api/v1/desktop/', '')
+ await this.handleDesktopStatus(res, hostRef)
+ return
+ }
+
// 404
this.sendJson(res, 404, { error: 'Not Found', message: 'Endpoint not found' })
}
@@ -268,8 +198,7 @@ export class ContextMapperServer {
endpoints: {
'GET /': 'This information page',
'GET /health': "Health check endpoint - returns { status: 'ok', ready: boolean }",
- 'GET /ready':
- 'Readiness endpoint - returns 200 after warm-up while provider inventory is authoritative',
+ 'GET /ready': 'Readiness endpoint - returns 200 only after warm-up completes',
'GET /api/v1/mcpservers': 'List all McpServer resources',
'GET /api/v1/mcpservers/context/:contextRef':
'List McpServers filtered by contextRef (only enabled servers)',
@@ -417,12 +346,9 @@ export class ContextMapperServer {
* Stop the server.
*/
stop(): Promise<void> {
- this.ready = false
- const activeServer = this.server
- this.server = null
return new Promise(resolve => {
- if (activeServer) {
- activeServer.close(() => {
+ if (this.server) {
+ this.server.close(() => {
console.log('[Server] Context Mapper stopped')
resolve()
})
diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.ts
index f731391ca..81542832a 100644
--- a/host-context-controller/src/statelessLifecycleExecutor.ts
+++ b/host-context-controller/src/statelessLifecycleExecutor.ts
@@ -42,21 +42,6 @@ const ACTIVE_COMMUNICATION_CHANNELS_REASON = 'ActiveCommunicationChannels'
*/
const WAKE_REQUESTED_ANNOTATION = 'clerum.io/wake-requested'
-/**
- * H2: reflect a committed lifecycle outcome onto the CURRENT canonical cache
- * entry for `name` — but only if that entry still has the admitted `uid`.
- * The receiver (McpServerWatcher) owns the lookup and the uid guard; the
- * executor never touches the cache directly. A same-name recreation with a
- * different uid MUST be skipped silently (the mutation already committed
- * server-side under the old uid; corrupting the new object would be worse
- * than staleness, which the watch/resync repairs).
- */
-export type ReflectHostOutcomeFn = (
- name: string,
- uid: string | undefined,
- apply: (target: HostCRD) => void
-) => void
-
/** Constructor-injected seams from HostReconciler (late-bound where needed). */
export interface StatelessLifecycleExecutorDeps {
appsApi: k8s.AppsV1Api
@@ -86,16 +71,7 @@ export interface StatelessLifecycleExecutorDeps {
* serialization slot, so re-entering the per-host chain would
* self-deadlock.
*/
- reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>
- /**
- * Capture Host inventory authority at heartbeat dispatch time and return an
- * admission closure that revalidates that authority and resolves the current
- * same-identity Host inside this executor's per-Host serializer. The two-step
- * shape is intentional: capturing only after the queue drains cannot detect
- * an authority generation change that happened while the work was waiting.
- */
- prepareHostMutationAdmission?: (action: string, host: HostCRD) => () => HostCRD
- reflectHostOutcome?: ReflectHostOutcomeFn
+ reconcileCore: (host: HostCRD) => Promise<void>
onLifecycleStatusCommitted?: (host: HostCRD, lifecycle: HostLifecycleStatus) => void
}
@@ -106,9 +82,7 @@ export class StatelessLifecycleExecutor {
private readonly now: () => Date
private readonly countCommunicationChannels: (hostName: string) => number
private readonly isCommunicationChannelCacheSynced: () => boolean
- private readonly reconcileCore: (host: HostCRD, revalidate?: () => HostCRD) => Promise<void>
- private readonly prepareHostMutationAdmission: (action: string, host: HostCRD) => () => HostCRD
- private readonly reflectHostOutcome: ReflectHostOutcomeFn
+ private readonly reconcileCore: (host: HostCRD) => Promise<void>
private readonly onLifecycleStatusCommitted: (
host: HostCRD,
lifecycle: HostLifecycleStatus
@@ -158,9 +132,6 @@ export class StatelessLifecycleExecutor {
this.countCommunicationChannels = deps.countCommunicationChannels
this.isCommunicationChannelCacheSynced = deps.isCommunicationChannelCacheSynced
this.reconcileCore = deps.reconcileCore
- this.prepareHostMutationAdmission =
- deps.prepareHostMutationAdmission ?? ((_action, host) => () => host)
- this.reflectHostOutcome = deps.reflectHostOutcome ?? (() => {})
this.onLifecycleStatusCommitted = deps.onLifecycleStatusCommitted ?? (() => {})
StatelessLifecycleExecutor.warnSingleReplicaInvariantOnce()
}
@@ -700,19 +671,15 @@ export class StatelessLifecycleExecutor {
metadata: {
name: string
namespace?: string
- uid?: string
- generation?: number
resourceVersion?: string
annotations?: Record<string, string>
}
spec: HostCRD['spec']
status?: HostCRD['status']
}
- const fresh: HostCRD = {
+ return {
name: obj.metadata.name,
namespace: obj.metadata.namespace ?? host.namespace,
- uid: obj.metadata.uid,
- generation: obj.metadata.generation,
// Preserve resourceVersion so heartbeat-path status writers can pass it
// back as an optimistic-concurrency precondition (D3). Dropping it here
// would force every write to be an unconditioned last-write-wins patch.
@@ -721,8 +688,6 @@ export class StatelessLifecycleExecutor {
spec: obj.spec,
status: obj.status,
}
- this.requireSameHostIdentity(host, fresh)
- return fresh
}
/**
@@ -751,24 +716,17 @@ export class StatelessLifecycleExecutor {
reason: string,
entryWakeHandledGeneration: number
): Promise<SuspendFromHeartbeatOutcome> {
- return this.serializeHeartbeatMutation(
- 'heartbeat suspension',
- host,
- (admittedHost, revalidate) =>
- this.suspendHostFromHeartbeatCore(
- admittedHost,
- reason,
- entryWakeHandledGeneration,
- revalidate
- )
+ // L2: serialize on the host chain so a tracker-driven suspend never
+ // interleaves with a watch-driven wake reconcile for the same host.
+ return this.serializeByHost(host.name, () =>
+ this.suspendHostFromHeartbeatCore(host, reason, entryWakeHandledGeneration)
)
}
private async suspendHostFromHeartbeatCore(
host: HostCRD,
reason: string,
- entryWakeHandledGeneration: number,
- revalidate?: () => HostCRD
+ entryWakeHandledGeneration: number
): Promise<SuspendFromHeartbeatOutcome> {
// FRESH GET (see readFreshHost): guard against the CURRENT server-side
// lifecycle, not the caller's watch-cache snapshot. The guard is re-run
@@ -783,17 +741,6 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
- // UID proves object identity, not decision freshness. A same-identity
- // Host may have advanced from stateless to stateful while the watch
- // cache trails the API. Re-evaluate the complete effective stateless
- // policy from every fresh read/retry before honoring old heartbeat
- // evidence.
- if (
- !this.sameHostSpecRevision(host, fresh) ||
- !this.effectiveLifecycleFromCache(fresh).stateless
- ) {
- return { skip: true }
- }
const freshLifecycle = fresh.status?.lifecycle
if (freshLifecycle?.state !== 'draining') {
// Every legitimate suspend follows a durable `draining` write (the
@@ -855,36 +802,12 @@ export class StatelessLifecycleExecutor {
writtenStatus = { ...(fresh.status ?? {}), lifecycle }
return this.fullStatusOps(writtenStatus)
},
- initialFresh,
- revalidate
+ initialFresh
)
if ('skipped' in result || writtenStatus === undefined) {
return skipOutcome
}
- // Revalidate authority/UID immediately before follow-on effects, then read
- // once more from the API. The successful PATCH attempt's fresh object is
- // the exact spec on which the status write committed, but a wake/spec
- // update can land immediately AFTER that commit and BEFORE runtime
- // reconciliation. A linearizable follow-on GET observes both our durable
- // suspended write and any such newer same-UID update; the watch cache can
- // legitimately trail either one.
- const currentAtFollowOn = revalidate?.()
- if (currentAtFollowOn) this.requireSameHostIdentity(host, currentAtFollowOn)
- this.reflectOutcome(host, target => {
- target.status = writtenStatus
- })
- const reconcileHost = await this.readFreshHost(result.fresh)
- if (!this.sameHostSpecRevision(host, reconcileHost)) {
- const error = new Error(
- `Host spec generation changed before heartbeat suspension follow-on for "${host.name}"`
- )
- error.name = 'HostMutationSpecRevisionChangedError'
- throw error
- }
- const currentAfterFollowOnRead = revalidate?.()
- if (currentAfterFollowOnRead) {
- this.requireSameHostIdentity(host, currentAfterFollowOnRead)
- }
+ host.status = writtenStatus
this.lastWrittenLifecycleStatus.delete(host.name)
this.logSuspendedApplied(host.name)
console.log(
@@ -893,32 +816,18 @@ export class StatelessLifecycleExecutor {
// L2: call reconcileCore (not the serialized reconcile) — this method
// already runs inside this host's serialization slot, so re-entering the
// per-host chain here would self-deadlock.
- await this.reconcileCore(reconcileHost, revalidate)
- // Preserve the historical caller-visible reflection contract even when
- // follow-on work correctly used a fresher API object. The tracker may hold
- // the originally admitted instance until the watch delivers the next
- // MODIFIED event; reflect the final authoritative outcome immediately so
- // it cannot make a second decision from the pre-follow-on status.
- this.reflectOutcome(host, target => {
- target.uid = reconcileHost.uid
- target.generation = reconcileHost.generation
- target.resourceVersion = reconcileHost.resourceVersion
- target.annotations = reconcileHost.annotations
- target.spec = reconcileHost.spec
- target.status = reconcileHost.status
- })
+ await this.reconcileCore(host)
// The wake fast-path / drained-pre-scale guard inside reconcile() may
// have flipped the host back to active (a wake raced the suspension) —
// in that case no scale-down happened, so no phase/metric is emitted.
const wakePending =
- this.wakeRequestedGeneration(reconcileHost) >
- (reconcileHost.status?.lifecycle?.wakeHandledGeneration ?? 0)
- if (reconcileHost.status?.lifecycle?.state !== 'suspended' || wakePending) {
+ this.wakeRequestedGeneration(host) > (host.status?.lifecycle?.wakeHandledGeneration ?? 0)
+ if (host.status?.lifecycle?.state !== 'suspended' || wakePending) {
// The durable suspended write DID commit (a racing wake revived the
// host afterwards), so the drained evidence was honored: 'suspended'.
return 'suspended'
}
- this.recordSuspendedApplied(reconcileHost.name)
+ this.recordSuspendedApplied(host.name)
return 'suspended'
}
@@ -929,19 +838,12 @@ export class StatelessLifecycleExecutor {
* Throws on API failure — the tracker logs it loudly.
*/
async publishSuspendBlockedReason(host: HostCRD, reason: string): Promise<void> {
- return this.serializeHeartbeatMutation(
- 'heartbeat suspend-blocked reason publication',
- host,
- (admittedHost, revalidate) =>
- this.publishSuspendBlockedReasonCore(admittedHost, reason, revalidate)
- )
+ // L2: serialize on the host chain (see serializeByHost) so this status
+ // write never interleaves with a concurrent reconcile for the same host.
+ return this.serializeByHost(host.name, () => this.publishSuspendBlockedReasonCore(host, reason))
}
- private async publishSuspendBlockedReasonCore(
- host: HostCRD,
- reason: string,
- revalidate?: () => HostCRD
- ): Promise<void> {
+ private async publishSuspendBlockedReasonCore(host: HostCRD, reason: string): Promise<void> {
// FRESH GET (see readFreshHost): this is a suspend-BLOCKED reason
// ANNOTATOR, not a state writer — it must NEVER change state, only stamp
// the reason on top of the CURRENT server-side state. AP-1 (the 9th
@@ -964,12 +866,6 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
- if (
- !this.sameHostSpecRevision(host, fresh) ||
- !this.effectiveLifecycleFromCache(fresh).stateless
- ) {
- return { skip: true }
- }
const freshLifecycle = fresh.status?.lifecycle
const freshState = freshLifecycle?.state ?? 'active'
if (freshState !== 'active') {
@@ -993,16 +889,12 @@ export class StatelessLifecycleExecutor {
writtenLifecycle = lifecycle
return [{ op: 'add', path: '/status/lifecycle', value: lifecycle }]
},
- initialFresh,
- revalidate
+ initialFresh
)
if ('skipped' in result || writtenLifecycle === undefined) {
return
}
- const reflectedStatus = { ...(host.status ?? {}), lifecycle: writtenLifecycle }
- this.reflectOutcome(host, target => {
- target.status = reflectedStatus
- })
+ host.status = { ...(host.status ?? {}), lifecycle: writtenLifecycle }
this.lastWrittenLifecycleStatus.delete(host.name)
}
@@ -1034,18 +926,17 @@ export class StatelessLifecycleExecutor {
host: HostCRD,
entryWakeHandledGeneration: number
): Promise<void> {
- return this.serializeHeartbeatMutation(
- 'heartbeat draining transition',
- host,
- (admittedHost, revalidate) =>
- this.markHostDrainingFromHeartbeatCore(admittedHost, entryWakeHandledGeneration, revalidate)
+ // L2: serialize on the host chain (see serializeByHost) so the drain
+ // status write never interleaves with a concurrent reconcile for the
+ // same host.
+ return this.serializeByHost(host.name, () =>
+ this.markHostDrainingFromHeartbeatCore(host, entryWakeHandledGeneration)
)
}
private async markHostDrainingFromHeartbeatCore(
host: HostCRD,
- entryWakeHandledGeneration: number,
- revalidate?: () => HostCRD
+ entryWakeHandledGeneration: number
): Promise<void> {
// FRESH GET (see readFreshHost): a stale `active` snapshot over an
// actually-`suspended` CR would write `draining` (a running state) and
@@ -1057,12 +948,6 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
- if (
- !this.sameHostSpecRevision(host, fresh) ||
- !this.effectiveLifecycleFromCache(fresh).stateless
- ) {
- return { skip: true }
- }
const freshLifecycle = fresh.status?.lifecycle
if ((freshLifecycle?.state ?? 'active') !== 'active') {
return { skip: true }
@@ -1098,15 +983,12 @@ export class StatelessLifecycleExecutor {
writtenStatus = { ...(fresh.status ?? {}), lifecycle }
return this.fullStatusOps(writtenStatus)
},
- initialFresh,
- revalidate
+ initialFresh
)
if ('skipped' in result || writtenStatus === undefined) {
return
}
- this.reflectOutcome(host, target => {
- target.status = writtenStatus
- })
+ host.status = writtenStatus
this.lastWrittenLifecycleStatus.delete(host.name)
console.log(
`[HostReconciler] Marked stateless Host "${host.name}" draining — control-api now answers {drain:true} from Host.status`
@@ -1128,17 +1010,13 @@ export class StatelessLifecycleExecutor {
* heartbeat retries.
*/
async markHostActiveFromHeartbeat(host: HostCRD): Promise<void> {
- return this.serializeHeartbeatMutation(
- 'heartbeat cancel-drain transition',
- host,
- (admittedHost, revalidate) => this.markHostActiveFromHeartbeatCore(admittedHost, revalidate)
- )
+ // L2: serialize on the host chain (see serializeByHost) so the
+ // cancel-drain status write never interleaves with a concurrent
+ // reconcile for the same host.
+ return this.serializeByHost(host.name, () => this.markHostActiveFromHeartbeatCore(host))
}
- private async markHostActiveFromHeartbeatCore(
- host: HostCRD,
- revalidate?: () => HostCRD
- ): Promise<void> {
+ private async markHostActiveFromHeartbeatCore(host: HostCRD): Promise<void> {
// FRESH GET (see readFreshHost): a stale `draining` snapshot over an
// actually-`suspended` CR must never resurrect it (and would regress
// wakeHandledGeneration). Proceed only when the CURRENT server-side
@@ -1151,12 +1029,6 @@ export class StatelessLifecycleExecutor {
const result = await this.patchStatusWithPrecondition(
host,
fresh => {
- if (
- !this.sameHostSpecRevision(host, fresh) ||
- !this.effectiveLifecycleFromCache(fresh).stateless
- ) {
- return { skip: true }
- }
const freshLifecycle = fresh.status?.lifecycle
if (freshLifecycle?.state !== 'draining') {
skippedNotDraining = freshLifecycle?.state ?? 'active'
@@ -1169,8 +1041,7 @@ export class StatelessLifecycleExecutor {
writtenStatus = { ...(fresh.status ?? {}), lifecycle }
return this.fullStatusOps(writtenStatus)
},
- initialFresh,
- revalidate
+ initialFresh
)
if ('skipped' in result || writtenStatus === undefined) {
console.log(
@@ -1178,9 +1049,7 @@ export class StatelessLifecycleExecutor {
)
return
}
- this.reflectOutcome(host, target => {
- target.status = writtenStatus
- })
+ host.status = writtenStatus
this.lastWrittenLifecycleStatus.delete(host.name)
console.log(
`[StatelessLifecycle] host=${host.name} cancel-drain: activity evidence while draining — reverting status to active`
@@ -1296,10 +1165,8 @@ export class StatelessLifecycleExecutor {
// Reflect the fresh truth on the in-memory object so the heavy
// reconcile body derives replicas from the current durable state
// instead of the stale event payload.
- this.reflectOutcome(host, target => {
- target.annotations = initialFresh.annotations
- target.status = initialFresh.status
- })
+ host.annotations = initialFresh.annotations
+ host.status = initialFresh.status
return false
}
@@ -1358,10 +1225,8 @@ export class StatelessLifecycleExecutor {
initialFresh
)
if ('skipped' in result || writtenLifecycle === undefined) {
- this.reflectOutcome(host, target => {
- target.annotations = latestFresh.annotations
- target.status = latestFresh.status
- })
+ host.annotations = latestFresh.annotations
+ host.status = latestFresh.status
return false
}
} catch (err) {
@@ -1380,11 +1245,8 @@ export class StatelessLifecycleExecutor {
// (assessLifecycle reads host.status) derives replicas from the woken
// state instead of re-suspending from the stale event payload. The base
// is the FRESH status the write committed over, never the cached spread.
- const reflectedStatus = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }
- this.reflectOutcome(host, target => {
- target.annotations = latestFresh.annotations
- target.status = reflectedStatus
- })
+ host.annotations = latestFresh.annotations
+ host.status = { ...(latestFresh.status ?? {}), lifecycle: writtenLifecycle }
console.log(
`[StatelessWake] host=${host.name} generation=${writtenLifecycle.wakeHandledGeneration} phase=status_flipped ts=${this.now().getTime()}`
)
@@ -1555,31 +1417,19 @@ export class StatelessLifecycleExecutor {
private async patchStatusWithPrecondition(
host: HostCRD,
build: (fresh: HostCRD) => Array<{ op: 'add'; path: string; value: unknown }> | { skip: true },
- initialFresh?: HostCRD,
- revalidate?: () => HostCRD
+ initialFresh?: HostCRD
): Promise<{ fresh: HostCRD } | { skipped: true }> {
const maxAttempts = 5
let fresh = initialFresh
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
- const currentAtAttempt = revalidate?.()
- if (currentAtAttempt) this.requireSameHostIdentity(host, currentAtAttempt)
if (fresh === undefined) {
fresh = await this.readFreshHost(host)
}
- if (host.uid !== undefined && fresh.resourceVersion === undefined) {
- const error = new Error(
- `Host status mutation for "${host.name}" has no fresh resourceVersion; refusing an unconditioned write`
- )
- error.name = 'HostMutationPreconditionUnavailableError'
- throw error
- }
const built = build(fresh)
if ('skip' in built) {
return { skipped: true }
}
try {
- const currentBeforePatch = revalidate?.()
- if (currentBeforePatch) this.requireSameHostIdentity(host, currentBeforePatch)
await this.customApi.patchNamespacedCustomObjectStatus({
group: GROUP,
version: VERSION,
@@ -1649,79 +1499,6 @@ export class StatelessLifecycleExecutor {
return result
}
- /**
- * Capture admission before enqueueing, then resolve the current Host only
- * after this mutation reaches the front of its own serializer. Keeping the
- * check here avoids wrapping a lifecycle call in HostReconciler's serializer,
- * which would recursively acquire the same Host chain and deadlock.
- */
- private serializeHeartbeatMutation<T>(
- action: string,
- host: HostCRD,
- work: (admittedHost: HostCRD, revalidate: () => HostCRD) => Promise<T>
- ): Promise<T> {
- const revalidate = this.prepareHostMutationAdmission(action, host)
- return this.serializeByHost(host.name, () => work(revalidate(), revalidate))
- }
-
- /**
- * A resourceVersion conflict can be caused by ordinary writes or by a
- * delete/recreate. Every fresh read, including every bounded 409 retry, must
- * remain on the Kubernetes UID that was admitted. Name equality is not an
- * identity fence. Production watch snapshots always carry UID; if only one
- * side carries it, fail closed rather than guessing.
- */
- /**
- * H2: apply a committed outcome to BOTH visible surfaces:
- * 1. the local working object (`host`) — with a resolver wired this is the
- * admitted CLONE (serializeHeartbeatMutation), so this keeps same-pass
- * logic (assessLifecycle, drained-pre-scale guard, follow-on reconcile)
- * reading the outcome exactly as before; standalone callers (no resolver)
- * get the legacy caller-visible reflection unchanged;
- * 2. the CURRENT cache entry, via the uid-guarded reflector (no-op unless
- * McpServerWatcher wired it).
- * `host.uid` is the admitted identity (admission proved requested.uid ===
- * current.uid; readFreshHost re-proves it), so passing it after mutate() ran
- * is equivalent to passing it before.
- *
- * `mutate` closures MUST only ASSIGN precomputed values — never derive from
- * `target` — so both surfaces receive the identical outcome.
- */
- private reflectOutcome(host: HostCRD, mutate: (target: HostCRD) => void): void {
- mutate(host)
- this.reflectHostOutcome(host.name, host.uid, mutate)
- }
-
- private requireSameHostIdentity(expected: HostCRD, current: HostCRD): void {
- if (expected.uid === undefined && current.uid === undefined) return
- if (expected.uid === undefined || current.uid === undefined || expected.uid !== current.uid) {
- const error = new Error(
- `Host identity changed while mutating "${expected.name}"; refusing to cross a same-name recreation`
- )
- error.name = 'HostMutationIdentityChangedError'
- throw error
- }
- }
-
- /**
- * Heartbeat evidence belongs to one Host spec epoch. UID equality allows
- * status-only updates and retries, but a generation change can replace the
- * runtime inputs (context, Secret, image, lifecycle policy, etc.) while
- * remaining stateless. Never apply an older runtime's heartbeat decision to
- * that new spec. UID-less standalone fixtures retain their legacy behavior;
- * production objects (which carry UID) fail closed on a missing generation.
- */
- private sameHostSpecRevision(expected: HostCRD, current: HostCRD): boolean {
- if (expected.generation === undefined && current.generation === undefined) {
- return expected.uid === undefined && current.uid === undefined
- }
- return (
- expected.generation !== undefined &&
- current.generation !== undefined &&
- expected.generation === current.generation
- )
- }
-
/**
* Drained-pre-scale guard (Stage 4.3): a pending wake must abort the
* suspension IMMEDIATELY before replicas:0 derives from the assessment.
@@ -1802,10 +1579,8 @@ export class StatelessLifecycleExecutor {
// A wake is pending in the fresh read — run the wake transition
// instead of the scale-down. Reflect the fresh wake generation on the
// in-memory object so downstream writes preserve it.
- this.reflectOutcome(host, target => {
- target.annotations = fresh.annotations
- target.status = fresh.status
- })
+ host.annotations = fresh.annotations
+ host.status = fresh.status
return {
effective: { stateless: true, state: 'active' },
lifecycle: { state: 'active', wakeHandledGeneration: freshWakeRequested },
@@ -1831,10 +1606,8 @@ export class StatelessLifecycleExecutor {
// Reflect fresh onto the in-memory host (mirroring the wake fast-path's
// epoch guard) so every downstream write derives from the current
// server-side truth instead of the stale event payload.
- this.reflectOutcome(host, target => {
- target.annotations = fresh.annotations
- target.status = fresh.status
- })
+ host.annotations = fresh.annotations
+ host.status = fresh.status
const freshReason = fresh.status?.lifecycle?.reason
const preservedReason = isHeartbeatManagedLifecycleReason(freshReason) ? freshReason : undefined
return {
diff --git a/host-context-controller/src/types.ts b/host-context-controller/src/types.ts
index 552ed3f0f..a7488dc7d 100644
--- a/host-context-controller/src/types.ts
+++ b/host-context-controller/src/types.ts
@@ -350,8 +350,6 @@ export interface ContextStatus {
export interface ContextCRD {
name: string
namespace: string
- uid?: string
- generation?: number
spec: ContextSpec
status?: ContextStatus
}
@@ -511,12 +509,6 @@ export interface McpServerStatus {
ready: boolean
/** Human-readable status message. */
message?: string
- /**
- * Whether HCC can tie this status to the current McpServer identity and
- * generation. False means discovery is authoritative but runtime status has
- * not yet been re-established after controller startup.
- */
- authoritative?: boolean
}
/**
diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.ts
index a6992eeb8..22d63b8b2 100644
--- a/host-context-controller/src/utils.ts
+++ b/host-context-controller/src/utils.ts
@@ -36,10 +36,6 @@ export async function replaceWithConflictRetry<
* when a meaningful change is still required.
*/
isUpToDate?: (body: T, existing: T) => boolean
- /** Reject an object whose identity or ownership is unsafe to replace. */
- validateExisting?: (existing: T) => void
- /** Rechecked immediately before every Kubernetes write attempt. */
- mutationAllowed?: () => boolean
maxAttempts?: number
}): Promise<void> {
const {
@@ -51,13 +47,10 @@ export async function replaceWithConflictRetry<
replace,
mergeExisting,
isUpToDate,
- validateExisting,
- mutationAllowed,
maxAttempts = 3,
} = opts
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
const existing = await read()
- validateExisting?.(existing)
const desired = resolveBody ? await resolveBody() : body
const base: T = {
...desired,
@@ -65,7 +58,6 @@ export async function replaceWithConflictRetry<
}
const next = mergeExisting ? mergeExisting(base, existing) : base
if (isUpToDate?.(next, existing)) return
- if (mutationAllowed && !mutationAllowed()) return
try {
await replace(next)
const suffix = attempt > 1 ? ` (after ${attempt} attempts)` : ''
@@ -179,11 +171,8 @@ export async function applyNetworkPolicy(
name: string,
namespace: string,
policy: k8s.V1NetworkPolicy,
- logPrefix = '[NetPol]',
- mutationAllowed?: () => boolean,
- validateExisting?: (existing: k8s.V1NetworkPolicy) => void
+ logPrefix = '[NetPol]'
): Promise<void> {
- if (mutationAllowed && !mutationAllowed()) return
try {
await api.createNamespacedNetworkPolicy({ namespace, body: policy })
console.log(`${logPrefix} Created policy "${name}" in ${namespace}`)
@@ -199,8 +188,6 @@ export async function applyNetworkPolicy(
body: policy,
read: () => api.readNamespacedNetworkPolicy({ name, namespace }),
replace: body => api.replaceNamespacedNetworkPolicy({ name, namespace, body }),
- mutationAllowed,
- validateExisting,
})
}
diff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.ts
index 17ac53f0f..4e6593024 100644
--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts
+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts
@@ -136,20 +136,6 @@ function createReconciler() {
return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }
}
-function deferred<T = void>(): {
- promise: Promise<T>
- resolve: (value: T) => void
- reject: (reason?: unknown) => void
-} {
- let resolve!: (value: T) => void
- let reject!: (reason?: unknown) => void
- const promise = new Promise<T>((res, rej) => {
- resolve = res
- reject = rej
- })
- return { promise, resolve, reject }
-}
-
/** The mcp-host Deployment body sent to the K8s API (excludes channel-reader). */
function hostDeploymentBody(appsApi: MockAppsApi, name: string): k8s.V1Deployment {
const calls = [
@@ -221,755 +207,7 @@ function freshHostRead(lifecycle?: {
}
}
-describe('HostReconciler heartbeat mutation admission', () => {
- const heartbeatMutations = [
- {
- name: 'suspend',
- core: 'suspendHostFromHeartbeatCore',
- coreArgs: ['idle', 0] as const,
- freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,
- invoke: (reconciler: HostReconciler, host: HostCRD) =>
- reconciler.suspendHostFromHeartbeat(host, 'idle', 0),
- },
- {
- name: 'suspend-blocked reason',
- core: 'publishSuspendBlockedReasonCore',
- coreArgs: ['suspend-blocked: activeTask'] as const,
- freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,
- invoke: (reconciler: HostReconciler, host: HostCRD) =>
- reconciler.publishSuspendBlockedReason(host, 'suspend-blocked: activeTask'),
- },
- {
- name: 'draining',
- core: 'markHostDrainingFromHeartbeatCore',
- coreArgs: [0] as const,
- freshLifecycle: { state: 'active', wakeHandledGeneration: 0 } as const,
- invoke: (reconciler: HostReconciler, host: HostCRD) =>
- reconciler.markHostDrainingFromHeartbeat(host, 0),
- },
- {
- name: 'cancel-drain',
- core: 'markHostActiveFromHeartbeatCore',
- coreArgs: [] as const,
- freshLifecycle: { state: 'draining', wakeHandledGeneration: 0 } as const,
- invoke: (reconciler: HostReconciler, host: HostCRD) =>
- reconciler.markHostActiveFromHeartbeat(host),
- },
- ] as const
-
- it.each(heartbeatMutations)(
- 'rejects queued $name work when Host authority changes before serializer admission',
- async ({ core, invoke }) => {
- const { reconciler } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 1 }
- const authority = { current: { known: true, generation: 7 } }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => authority.current)
- const lifecycle = (reconciler as any).lifecycle
- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = lifecycle.serializeByHost(host.name, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }) as Promise<void>
- await blockerStarted.promise
-
- const pending = invoke(reconciler, host)
- authority.current = { known: false, generation: 8 }
- releaseBlocker.resolve(undefined)
- await occupied
-
- await expect(pending).rejects.toThrow(/Host inventory authority/)
- expect(coreSpy).not.toHaveBeenCalled()
- }
- )
-
- it.each(heartbeatMutations)(
- 'rejects queued $name work when the Host UID changes before serializer admission',
- async ({ core, invoke }) => {
- const { reconciler } = createReconciler()
- const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-old-uid', generation: 4 }
- let current = requested
- const authority = { current: { known: true, generation: 9 } }
- reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => authority.current)
- const lifecycle = (reconciler as any).lifecycle
- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = lifecycle.serializeByHost(requested.name, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }) as Promise<void>
- await blockerStarted.promise
-
- const pending = invoke(reconciler, requested)
- current = { ...requested, uid: 'heartbeat-host-new-uid', generation: 1 }
- releaseBlocker.resolve(undefined)
- await occupied
-
- await expect(pending).rejects.toThrow(/Host identity/)
- expect(coreSpy).not.toHaveBeenCalled()
- }
- )
-
- it.each(heartbeatMutations)(
- 'admits the latest same-UID same-spec Host snapshot for $name inside the heartbeat serializer',
- async ({ core, coreArgs, invoke }) => {
- const { reconciler } = createReconciler()
- const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- const current = {
- ...requested,
- resourceVersion: 'cache-rv-latest',
- status: { lifecycle: { state: 'active' as const, wakeHandledGeneration: 2 } },
- }
- reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 12 }))
- const lifecycle = (reconciler as any).lifecycle
- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
-
- await invoke(reconciler, requested)
-
- expect(coreSpy).toHaveBeenCalledOnce()
- expect(coreSpy).toHaveBeenCalledWith(current, ...coreArgs, expect.any(Function))
- }
- )
-
- it.each(heartbeatMutations)(
- 'rejects a $name 409 retry that resolves to a recreated same-name Host',
- async ({ freshLifecycle, invoke }) => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),
- uid: 'heartbeat-host-old-uid',
- generation: 4,
- }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 15 }))
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead(freshLifecycle),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-old',
- },
- })
- .mockResolvedValueOnce({
- ...freshHostRead(freshLifecycle),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: 'heartbeat-host-new-uid',
- generation: 1,
- resourceVersion: 'rv-new',
- },
- })
- customApi.patchNamespacedCustomObjectStatus
- .mockRejectedValueOnce({ code: 409 })
- .mockResolvedValueOnce(undefined)
-
- await expect(invoke(reconciler, host)).rejects.toThrow(/Host identity/)
- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
- }
- )
-
- it.each(heartbeatMutations)(
- 'rejects queued $name work when the same Host advances to a new spec generation',
- async ({ core, invoke }) => {
- const { reconciler } = createReconciler()
- const requested = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- let current = requested
- reconciler.setResolveCurrentHost(name => (name === requested.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 13 }))
- const lifecycle = (reconciler as any).lifecycle
- const coreSpy = vi.spyOn(lifecycle, core).mockResolvedValue(undefined)
- const blockerStarted = deferred()
- const releaseBlocker = deferred()
- const occupied = lifecycle.serializeByHost(requested.name, async () => {
- blockerStarted.resolve(undefined)
- await releaseBlocker.promise
- }) as Promise<void>
- await blockerStarted.promise
-
- const pending = invoke(reconciler, requested)
- current = {
- ...requested,
- generation: 5,
- spec: { ...requested.spec, contextRef: 'new-runtime-context' },
- }
- releaseBlocker.resolve(undefined)
- await occupied
-
- await expect(pending).rejects.toThrow(/Host spec generation/)
- expect(coreSpy).not.toHaveBeenCalled()
- }
- )
-
- it.each(heartbeatMutations)(
- 'does not apply stale $name evidence to a newer same-UID stateless spec',
- async ({ freshLifecycle, invoke }) => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({ status: { lifecycle: freshLifecycle } }),
- uid: 'heartbeat-host-uid',
- generation: 4,
- }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 14 }))
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead(freshLifecycle),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-new-spec',
- },
- spec: { ...host.spec, contextRef: 'new-runtime-context' },
- })
- const reconcileCore = vi
- .spyOn(reconciler as any, 'reconcileCore')
- .mockResolvedValue(undefined)
-
- await invoke(reconciler, host)
-
- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- expect(reconcileCore).not.toHaveBeenCalled()
- }
- )
-
- it.each(heartbeatMutations)(
- 'does not apply stale $name evidence after the same-UID Host becomes stateful',
- async ({ invoke }) => {
- const { reconciler, customApi } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 16 }))
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-stateful',
- },
- spec: {
- ...host.spec,
- lifecycle: { stateless: false },
- },
- })
- const reconcileCore = vi
- .spyOn(reconciler as any, 'reconcileCore')
- .mockResolvedValue(undefined)
-
- await invoke(reconciler, host)
-
- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- expect(reconcileCore).not.toHaveBeenCalled()
- }
- )
-
- it('does not retry stale heartbeat evidence after a 409 reveals a same-UID stateful spec', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-stateless',
- },
- })
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-stateful',
- },
- spec: {
- ...host.spec,
- lifecycle: { stateless: false },
- },
- })
- customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })
-
- await reconciler.markHostDrainingFromHeartbeat(host, 0)
-
- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
- })
-
- it('does not retry stale heartbeat evidence after a 409 reveals a newer stateless spec', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-old-spec',
- },
- })
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-new-spec',
- },
- spec: { ...host.spec, contextRef: 'new-runtime-context' },
- })
- customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce({ code: 409 })
-
- await reconciler.markHostDrainingFromHeartbeat(host, 0)
-
- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
- })
-
- it('revalidates authority after the fresh read and before the first status patch', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- const authority = { current: { known: true, generation: 18 } }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => authority.current)
- customApi.getNamespacedCustomObject.mockImplementation(async () => {
- authority.current = { known: false, generation: 19 }
- return {
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- resourceVersion: 'rv-authority-lost',
- },
- }
- })
-
- await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
- /Host inventory authority/
- )
- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- })
-
- it('revalidates authority before a 409 retry performs another fresh read', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- const authority = { current: { known: true, generation: 21 } }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => authority.current)
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-before-conflict',
- },
- })
- customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {
- authority.current = { known: false, generation: 22 }
- throw { code: 409 }
- })
-
- await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
- /Host inventory authority/
- )
- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
- })
-
- it('does not reconcile runtime effects after suspend when the Host UID changes', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
- }),
- uid: 'heartbeat-host-old-uid',
- generation: 4,
- }
- let current = host
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 24 }))
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-suspend',
- },
- })
- customApi.patchNamespacedCustomObjectStatus.mockImplementationOnce(async () => {
- current = { ...host, uid: 'heartbeat-host-new-uid', generation: 1 }
- })
-
- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
- /Host identity/
- )
- expect(reconcileCore).not.toHaveBeenCalled()
- })
-
- it('reconciles suspend follow-on effects from the successful fresh same-UID Host spec', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({
- spec: { contextRef: 'latest-context' },
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
- }),
- uid: 'heartbeat-host-uid',
- generation: 5,
- }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 26 }))
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-suspend-current',
- },
- spec: { ...host.spec, contextRef: 'latest-context' },
- })
- .mockResolvedValue({
- ...freshHostRead({
- state: 'suspended',
- wakeHandledGeneration: 0,
- reason: 'idle',
- }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-suspend-committed',
- },
- spec: { ...host.spec, contextRef: 'latest-context' },
- })
-
- await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
-
- expect(reconcileCore).toHaveBeenCalledOnce()
- const reconciledHost = reconcileCore.mock.calls[0][0] as HostCRD
- expect(reconciledHost).toMatchObject({
- uid: host.uid,
- generation: 5,
- spec: expect.objectContaining({ contextRef: 'latest-context' }),
- })
- expect(reconciledHost.status?.lifecycle).toEqual({
- state: 'suspended',
- wakeHandledGeneration: 0,
- reason: 'idle',
- })
- })
-
- it('rejects a known-identity status write without a fresh resourceVersion', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = { ...makeStatelessHost(), uid: 'heartbeat-host-uid', generation: 4 }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 28 }))
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: { name: host.name, namespace: host.namespace, uid: host.uid },
- })
-
- await expect(reconciler.markHostDrainingFromHeartbeat(host, 0)).rejects.toThrow(
- /no fresh resourceVersion/
- )
- expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
- })
-
- it('retries a 409 on the same UID and uses the latest Host for suspend follow-on effects', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
- }),
- uid: 'heartbeat-host-uid',
- generation: 4,
- }
- let current = host
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 30 }))
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-1',
- },
- })
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-2',
- },
- spec: host.spec,
- })
- .mockResolvedValueOnce({
- ...freshHostRead({
- state: 'suspended',
- wakeHandledGeneration: 0,
- reason: 'idle',
- }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-3',
- },
- spec: host.spec,
- })
- customApi.patchNamespacedCustomObjectStatus
- .mockImplementationOnce(async () => {
- current = {
- ...host,
- resourceVersion: 'cache-rv-after-conflict',
- }
- throw { code: 409 }
- })
- .mockResolvedValueOnce(undefined)
-
- await reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
-
- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(3)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
- expect(reconcileCore.mock.calls[0][0]).toMatchObject({
- uid: host.uid,
- generation: 4,
- spec: expect.objectContaining({ contextRef: 'context-a' }),
- })
- })
-
- it('does not apply or record suspension when only the follow-on GET observes a newer spec', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
- }),
- uid: 'heartbeat-host-uid',
- generation: 4,
- }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 33 }))
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-suspend-write',
- },
- })
- .mockResolvedValueOnce({
- ...freshHostRead({
- state: 'suspended',
- wakeHandledGeneration: 0,
- reason: 'idle',
- }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 5,
- resourceVersion: 'rv-new-spec',
- },
- spec: {
- ...host.spec,
- contextRef: 'replacement-context',
- lifecycle: { stateless: false },
- },
- })
- const reconcileCore = vi.spyOn(reconciler as any, 'reconcileCore').mockResolvedValue(undefined)
- const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')
-
- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).rejects.toThrow(
- /Host spec generation/
- )
-
- expect(reconcileCore).not.toHaveBeenCalled()
- expect(recordSuspended).not.toHaveBeenCalled()
- })
-
- it('rejects stale suspend follow-on work when the Host spec changes during Secret validation', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
- }),
- uid: 'heartbeat-host-uid',
- generation: 4,
- }
- let current = host
- reconciler.setResolveCurrentHost(name => (name === host.name ? current : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 34 }))
- customApi.getNamespacedCustomObject
- .mockResolvedValueOnce({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-suspend-write',
- },
- })
- .mockResolvedValueOnce({
- ...freshHostRead({
- state: 'suspended',
- wakeHandledGeneration: 0,
- reason: 'idle',
- }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-suspend-follow-on',
- },
- })
- const secretReadStarted = deferred()
- const releaseSecretRead = deferred()
- vi.spyOn(reconciler as any, 'validateHostSecret').mockImplementation(async () => {
- secretReadStarted.resolve(undefined)
- await releaseSecretRead.promise
- return { ok: true }
- })
- const staleMutation = vi
- .spyOn(reconciler as any, 'ensureHostServiceAccount')
- .mockRejectedValue(new Error('stale follow-on mutation was admitted'))
-
- const pending = reconciler.suspendHostFromHeartbeat(host, 'idle', 0)
- await secretReadStarted.promise
- current = {
- ...host,
- generation: 5,
- spec: {
- ...host.spec,
- contextRef: 'replacement-context',
- lifecycle: { stateless: false },
- },
- }
- releaseSecretRead.resolve(undefined)
-
- await expect(pending).rejects.toThrow(/Host spec generation/)
- expect(staleMutation).not.toHaveBeenCalled()
- })
-
- it('does not record a suspended scale-down when follow-on reconcile handles a wake', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 0 } },
- }),
- uid: 'heartbeat-host-uid',
- generation: 4,
- }
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 32 }))
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'draining', wakeHandledGeneration: 0 }),
- metadata: {
- name: host.name,
- namespace: host.namespace,
- uid: host.uid,
- generation: 4,
- resourceVersion: 'rv-wake-follow-on',
- },
- })
- vi.spyOn(reconciler as any, 'reconcileCore').mockImplementation(async (...args: unknown[]) => {
- const reconcileHost = args[0] as HostCRD
- reconcileHost.annotations = { 'clerum.io/wake-requested': '1' }
- reconcileHost.status = {
- lifecycle: { state: 'active', wakeHandledGeneration: 1 },
- }
- })
- const recordSuspended = vi.spyOn((reconciler as any).lifecycle, 'recordSuspendedApplied')
-
- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 0)).resolves.toBe('suspended')
-
- expect(recordSuspended).not.toHaveBeenCalled()
- })
-})
-
describe('HostReconciler.suspendHostFromHeartbeat', () => {
- it('does not mutate the admitted watch-cache snapshot while applying a suspension', async () => {
- const { reconciler, customApi } = createReconciler()
- const host = makeStatelessHost({
- status: { lifecycle: { state: 'draining', wakeHandledGeneration: 2 } },
- })
- host.uid = 'stateless-host-uid'
- host.generation = 1
- host.resourceVersion = 'cache-rv'
- const snapshot = structuredClone(host)
- Object.freeze(host)
- reconciler.setResolveCurrentHost(name => (name === host.name ? host : undefined))
- reconciler.setHostMutationAuthority(() => ({
- known: true,
- hostGeneration: 1,
- contextGeneration: 1,
- }))
- const draining = freshHostRead({ state: 'draining', wakeHandledGeneration: 2 })
- Object.assign(draining.metadata, {
- uid: host.uid,
- generation: host.generation,
- resourceVersion: 'draining-rv',
- })
- const suspended = freshHostRead({
- state: 'suspended',
- wakeHandledGeneration: 2,
- reason: 'idle',
- })
- Object.assign(suspended.metadata, {
- uid: host.uid,
- generation: host.generation,
- resourceVersion: 'suspended-rv',
- })
- customApi.getNamespacedCustomObject.mockResolvedValueOnce(draining).mockResolvedValue(suspended)
-
- await expect(reconciler.suspendHostFromHeartbeat(host, 'idle', 2)).resolves.toBe('suspended')
- expect(host).toEqual(snapshot)
- })
-
it("writes state='suspended' + reason 'idle' durably, then reconciles to replicas=0", async () => {
const { reconciler, appsApi, customApi } = createReconciler()
const host = makeStatelessHost({
@@ -2075,89 +1313,3 @@ describe('HostReconciler reconcile — stateless replicas derive from FRESH stat
}
})
})
-
-describe('H2: guarded cache reflection', () => {
- it('reflects a committed lifecycle outcome onto the current cache entry without a watch event', async () => {
- const { reconciler, customApi } = createReconciler()
- const entry = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
- }),
- uid: 'uid-1',
- generation: 1,
- resourceVersion: 'rv-1',
- }
- const hosts = new Map<string, HostCRD>([[entry.name, entry]])
- reconciler.setResolveCurrentHost(name => hosts.get(name))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
- // Exactly the k8sClient.ts wiring shape (uid-guarded apply onto the cache).
- reconciler.setReflectHostOutcome((name, uid, apply) => {
- const cached = hosts.get(name)
- if (!cached || uid === undefined || cached.uid !== uid) return
- apply(cached)
- })
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: entry.name,
- namespace: entry.namespace,
- uid: 'uid-1',
- generation: 1,
- resourceVersion: 'fresh-rv',
- },
- })
-
- await reconciler.markHostDrainingFromHeartbeat(entry, 0)
-
- // The status write committed to the API…
- expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')
- // …and — the H2 fix — the committed outcome is visible on the CANONICAL cache
- // entry without waiting for a watch MODIFIED. With the resolver wired, the
- // heartbeat core mutates the admitted clone (Shape A), so this can ONLY be
- // true if the guarded reflector routed the outcome onto the cache entry.
- expect(hosts.get(entry.name)?.status?.lifecycle?.state).toBe('draining')
- })
-
- it('never reflects a committed outcome onto a same-name recreation with a different uid', async () => {
- const { reconciler, customApi } = createReconciler()
- const entry = {
- ...makeStatelessHost({
- status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
- }),
- uid: 'uid-1',
- generation: 1,
- resourceVersion: 'rv-1',
- }
- const hosts = new Map<string, HostCRD>([[entry.name, entry]])
- const replacement = { ...makeStatelessHost({ name: entry.name }), uid: 'uid-2' }
- reconciler.setResolveCurrentHost(name => hosts.get(name))
- reconciler.setHostWatchAuthority(() => ({ known: true, generation: 17 }))
- reconciler.setReflectHostOutcome((name, uid, apply) => {
- const cached = hosts.get(name)
- if (!cached || uid === undefined || cached.uid !== uid) return
- apply(cached)
- })
- customApi.getNamespacedCustomObject.mockResolvedValue({
- ...freshHostRead({ state: 'active', wakeHandledGeneration: 0 }),
- metadata: {
- name: entry.name,
- namespace: entry.namespace,
- uid: 'uid-1',
- generation: 1,
- resourceVersion: 'fresh-rv',
- },
- })
- // A watch DELETE+ADD recreates the entry (new uid) mid-commit.
- customApi.patchNamespacedCustomObjectStatus.mockImplementation(async () => {
- hosts.set(entry.name, replacement)
- return {}
- })
-
- await reconciler.markHostDrainingFromHeartbeat(entry, 0)
-
- // The write committed…
- expect(lifecycleStatusWrites(customApi).at(-1)?.lifecycle?.state).toBe('draining')
- // …but the recreation (uid-2) was NOT corrupted by the stale reflection.
- expect(replacement.status?.lifecycle).toBeUndefined()
- })
-})
diff --git a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts
index f8696e57e..a06465eeb 100644
--- a/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.k8sApiEgress.test.ts
@@ -34,12 +34,9 @@ describe('ensureK8sApiEgress', () => {
mockApply.mockClear()
// ensureDefaultPolicies() also prunes legacy static policies via the
// networking API directly (not applyNetworkPolicy), so the mocked client
- // must expose the read-before-delete legacy cleanup path.
+ // must expose deleteNamespacedNetworkPolicy.
const mockKc = {
makeApiClient: () => ({
- readNamespacedNetworkPolicy: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
}),
} as unknown as k8s.KubeConfig
diff --git a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts
index 04e7b3b33..6a89edc00 100644
--- a/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.minimalInfra.test.ts
@@ -26,9 +26,7 @@ const mockApply = vi.mocked(applyNetworkPolicy)
function createMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
diff --git a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts
index 9d0db5860..ae59d2dbd 100644
--- a/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.nodeLocalDns.test.ts
@@ -32,9 +32,7 @@ const mockApply = vi.mocked(applyNetworkPolicy)
function createMockNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
- readNamespacedNetworkPolicy: vi
- .fn()
- .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: {} }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
diff --git a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts
index 6acb165b3..a93e20eed 100644
--- a/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts
+++ b/host-context-controller/test/networkPolicyReconciler.rpcEgress.test.ts
@@ -83,7 +83,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('generates egress policy in rpc-proxy namespace when context has servers', async () => {
const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])
- await reconciler.reconcileContext(ctx, { isCurrent: () => true })
+ await reconciler.reconcileContext(ctx)
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)
@@ -91,7 +91,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('egress policy targets correct server pod by mcpserver label', async () => {
const ctx = makeContext('ctx-alpha', ['mongodb-mcp'])
- await reconciler.reconcileContext(ctx, { isCurrent: () => true })
+ await reconciler.reconcileContext(ctx)
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
expect(rpcProxyCalls.length).toBeGreaterThanOrEqual(1)
@@ -103,7 +103,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('egress policy uses correct port from McpServer spec', async () => {
const ctx = makeContext('ctx-beta', ['airtable-mcp'])
- await reconciler.reconcileContext(ctx, { isCurrent: () => true })
+ await reconciler.reconcileContext(ctx)
const rpcProxyCalls = mockApply.mock.calls.filter(
call => call[1] === 'rpc-egress-ctx-beta-airtable-mcp'
@@ -122,8 +122,6 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
metadata: {
name: 'rpc-egress-ctx-gamma-mongodb-mcp',
namespace: 'rpc-proxy',
- uid: 'rpc-delete-uid',
- resourceVersion: '7',
labels: {
'clerum.io/managed-by': 'host-context-controller',
'clerum.io/policy-type': 'rpc-proxy-egress',
@@ -146,22 +144,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('deletes orphaned egress policies when server removed from context', async () => {
// First reconcile with both servers
const ctx1 = makeContext('ctx-delta', ['mongodb-mcp', 'airtable-mcp'])
- await reconciler.reconcileContext(ctx1, { isCurrent: () => true })
- const existingRpcProxyPolicies = mockApply.mock.calls
- .filter(call => call[2] === 'rpc-proxy')
- .map(call => {
- const policy = call[3] as k8s.V1NetworkPolicy
- const name = policy.metadata?.name
- return {
- ...policy,
- metadata: {
- ...policy.metadata,
- uid: `rpc-proxy:${name}:uid`,
- resourceVersion: '1',
- },
- }
- })
- expect(existingRpcProxyPolicies).toHaveLength(2)
+ await reconciler.reconcileContext(ctx1)
vi.clearAllMocks()
mockNetApi.listNamespacedNetworkPolicy.mockImplementation(
@@ -171,7 +154,10 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
String(labelSelector).includes('clerum.io/context=ctx-delta')
) {
return {
- items: existingRpcProxyPolicies,
+ items: [
+ { metadata: { name: 'rpc-egress-ctx-delta-mongodb-mcp' } },
+ { metadata: { name: 'rpc-egress-ctx-delta-airtable-mcp' } },
+ ],
}
}
return { items: [] }
@@ -180,7 +166,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
// Now reconcile with only one server — orphaned rpc-proxy policy should be cleaned
const ctx2 = makeContext('ctx-delta', ['mongodb-mcp'])
- await reconciler.reconcileContext(ctx2, { isCurrent: () => true })
+ await reconciler.reconcileContext(ctx2)
// applyNetworkPolicy should be called for mongodb-mcp in rpc-proxy but not airtable-mcp
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
@@ -190,12 +176,6 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
expect(mockNetApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'rpc-egress-ctx-delta-airtable-mcp',
namespace: 'rpc-proxy',
- body: {
- preconditions: {
- uid: 'rpc-proxy:rpc-egress-ctx-delta-airtable-mcp:uid',
- resourceVersion: '1',
- },
- },
})
expect(mockNetApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
name: 'rpc-egress-ctx-delta-mongodb-mcp',
@@ -205,7 +185,7 @@ describe('NetworkPolicyReconciler rpc-proxy egress', () => {
it('skips egress generation when server not found in cache', async () => {
const ctx = makeContext('ctx-epsilon', ['nonexistent-server'])
- await reconciler.reconcileContext(ctx, { isCurrent: () => true })
+ await reconciler.reconcileContext(ctx)
const rpcProxyCalls = mockApply.mock.calls.filter(call => call[2] === 'rpc-proxy')
expect(rpcProxyCalls.length).toBe(0)
diff --git a/mcp-host/package-lock.json b/mcp-host/package-lock.json
index 8519e148d..15f46834e 100644
--- a/mcp-host/package-lock.json
+++ b/mcp-host/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.239",
+ "version": "0.1.240",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-mcp-host",
- "version": "0.1.239",
+ "version": "0.1.240",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.0",
diff --git a/mcp-host/package.json b/mcp-host/package.json
index 2dcb39dec..0fd114fe5 100644
--- a/mcp-host/package.json
+++ b/mcp-host/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-mcp-host",
- "version": "0.1.239",
+ "version": "0.1.240",
"description": "MCP Host - reads Host CRD and provides LLM access via OpenAI/Claude",
"main": "dist/main.js",
"engines": {
diff --git a/mcp-host/src/__tests__/main.mcpInitialization.test.ts b/mcp-host/src/__tests__/main.mcpInitialization.test.ts
index 02e9e9c75..c90bad869 100644
--- a/mcp-host/src/__tests__/main.mcpInitialization.test.ts
+++ b/mcp-host/src/__tests__/main.mcpInitialization.test.ts
@@ -22,12 +22,7 @@ function readyServer(overrides: Partial<McpServerInfo> = {}): McpServerInfo {
transport: { type: 'streamableHttp', url: 'http://secured-server.test/mcp' },
auth: { type: 'bearer', secretRef: 'secured-server-auth' },
enabled: true,
- // Mirror the real producer contract: HCC's reconciler.getStatus() (surfaced
- // through k8sClient.toServerInfo) always stamps `authoritative` explicitly —
- // a ready, identity-matched snapshot carries authoritative: true. The legacy
- // shape that omitted it is never emitted for a live server, so the default
- // fixture must carry it too.
- status: { deployed: true, ready: true, authoritative: true },
+ status: { deployed: true, ready: true },
...overrides,
}
}
diff --git a/nginx-egress-proxy/README.md b/nginx-egress-proxy/README.md
index 372f8e5aa..b4c205974 100644
--- a/nginx-egress-proxy/README.md
+++ b/nginx-egress-proxy/README.md
@@ -53,10 +53,10 @@ reference is platform-controlled:
| `CONTEXT_MAPPER_EGRESS_PROXY_IMAGE` | host-context-controller | `clerum/nginx-egress-proxy:0.1.0` |
| `CONTROL_API_REMOTE_MCP_EGRESS_PROXY_IMAGE` | control-api (registry install stamps `spec.image` for remote entries) | `clerum/nginx-egress-proxy:0.1.0` |
-HCC does not rewrite `spec.image` during reconciliation: that field remains
-desired state owned by the CRD author. Deployment rendering always stamps the
-configured platform image for remote egress proxies, so a remote CRD's
-`spec.image` is never what runs.
+HCC additionally canonicalizes `spec.image` of remote McpServers back to the
+platform image (`canonicalizeRemoteEgressProxyImage`) as best-effort cleanup;
+the enforcement is that Deployment rendering always stamps the configured
+platform image, so a CRD's `spec.image` is never what runs.
## Ports
diff --git a/rpc-proxy/package-lock.json b/rpc-proxy/package-lock.json
index 38afc5704..fde59308e 100644
--- a/rpc-proxy/package-lock.json
+++ b/rpc-proxy/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.65",
+ "version": "0.1.64",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-rpc-proxy",
- "version": "0.1.65",
+ "version": "0.1.64",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.21.2",
diff --git a/rpc-proxy/package.json b/rpc-proxy/package.json
index a17ac4862..2c7a4f3ed 100644
--- a/rpc-proxy/package.json
+++ b/rpc-proxy/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-rpc-proxy",
- "version": "0.1.65",
+ "version": "0.1.64",
"description": "External-facing RPC proxy for tenant-scoped MCP access",
"main": "dist/main.js",
"engines": {
diff --git a/rpc-proxy/src/config.hccBaseUrl.test.ts b/rpc-proxy/src/config.hccBaseUrl.test.ts
index bc3543a8d..b1cedb606 100644
--- a/rpc-proxy/src/config.hccBaseUrl.test.ts
+++ b/rpc-proxy/src/config.hccBaseUrl.test.ts
@@ -27,33 +27,16 @@ describe('rpc-proxy HCC base URL default', () => {
)
const configmapMatch = configmap.match(/RPC_PROXY_HCC_BASE_URL:\s*"?([^"\n]+)"?/)
- // Derive the real endpoint from the api-gateway manifest itself, so renaming
- // the Service or moving its port breaks this test unless the dev default and
- // configmap are updated in lockstep — no hand-copied literal to drift.
- const gatewayManifest = readFileSync(
- path.join(repoRoot, 'deploy/base/control-plane/host-context-controller-api-gateway.yaml'),
- 'utf8'
- )
- const serviceDoc = gatewayManifest.split(/^---$/m).find(doc => /\bkind:\s*Service\b/.test(doc))
- const serviceName = serviceDoc?.match(/\bmetadata:[\s\S]*?\bname:\s*([\w.-]+)/)?.[1]
- const serviceNamespace = serviceDoc?.match(/\bnamespace:\s*([\w.-]+)/)?.[1]
- const servicePort = serviceDoc?.match(/\bports:[\s\S]*?\bport:\s*(\d+)/)?.[1]
-
- it('parses all three sources', () => {
+ it('parses both sources', () => {
expect(defaultMatch?.[1]).toBeTruthy()
expect(configmapMatch?.[1]).toBeTruthy()
- expect(serviceName).toBeTruthy()
- expect(serviceNamespace).toBeTruthy()
- expect(servicePort).toBeTruthy()
})
it('dev default matches the deployed configmap endpoint', () => {
expect(defaultMatch?.[1]).toBe(configmapMatch?.[1]?.trim())
})
- it('targets the api-gateway Service exactly as the gateway manifest declares it', () => {
- const url = new URL(defaultMatch![1])
- expect(url.hostname).toBe(`${serviceName}.${serviceNamespace}.svc.cluster.local`)
- expect(url.port).toBe(servicePort)
+ it('targets the api-gateway in control-plane, not a nonexistent Service', () => {
+ expect(defaultMatch?.[1]).toContain('host-context-controller-api-gateway.control-plane')
})
})
diff --git a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
index ecb97cbdc..bc77ee0f1 100755
--- a/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
@@ -1,82 +1,28 @@
#!/usr/bin/env bash
-# Safety helpers shared by disruptive HCC fault-injection fixtures.
-
-profile_env_value() {
- local key=$1 file=$2
- awk -v key="$key" '
- index($0, key "=") == 1 {
- print substr($0, length(key) + 2)
- exit
- }
- ' "$file"
-}
+# Safety helpers for the HCC CommunicationChannel watch-recovery fault fixture.
require_branch_owned_hcc_gate() {
- local marker_namespace="${1:-control-plane}"
- local actual_worktree_id actual_head actual_cluster_fingerprint actual_gate
- local worktree_dirty expected_branch profile_env
- local pre_gate_state_root cluster_fingerprint_file expected_cluster_fingerprint
+ local repo_root expected_worktree_id expected_head actual_worktree_id actual_head worktree_dirty
is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||
die "fault injection requires a generated branch-scoped context, got '${E2E_KUBECONTEXT}'"
- HCC_BRANCH_GATE_REPO_ROOT="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"
- worktree_dirty="$(
- git -C "$HCC_BRANCH_GATE_REPO_ROOT" status --porcelain --untracked-files=normal
- )"
+ repo_root="$(git -C "${SCRIPT_DIR}/../.." rev-parse --show-toplevel)"
+ worktree_dirty="$(git -C "$repo_root" status --porcelain --untracked-files=normal)"
[ -z "$worktree_dirty" ] ||
die "worktree has uncommitted changes; commit and re-sync before runtime proof"
- HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID="$(
- printf '%s' "$HCC_BRANCH_GATE_REPO_ROOT" | shasum | awk '{print $1}'
- )"
- HCC_BRANCH_GATE_EXPECTED_HEAD="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" rev-parse HEAD)"
- expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"
- [ -n "$expected_branch" ] ||
- die "branch-owned runtime proof requires a named branch, not detached HEAD"
- [ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ] ||
- die "MINIKUBE_PROFILE must explicitly select context ${E2E_KUBECONTEXT}, got ${MINIKUBE_PROFILE:-missing}"
-
- profile_env="${E2E_BRANCH_PROFILE_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/profile.env}"
- [ -r "$profile_env" ] ||
- die "branch-profile helper evidence is missing at ${profile_env}; run a state-writing branch-profile helper action from this worktree"
- [ "$(profile_env_value PROFILE "$profile_env")" = "$E2E_KUBECONTEXT" ] ||
- die "branch-profile helper output does not select context ${E2E_KUBECONTEXT}"
- [ "$(profile_env_value REPO_DIR "$profile_env")" = "$HCC_BRANCH_GATE_REPO_ROOT" ] ||
- die "branch-profile helper output belongs to a different worktree"
- [ "$(profile_env_value BRANCH "$profile_env")" = "$expected_branch" ] ||
- die "branch-profile helper output belongs to a different branch"
- [ "$(profile_env_value DIRTY "$profile_env")" = false ] ||
- die "branch-profile helper recorded a dirty worktree; refresh it after committing"
-
- pre_gate_state_root="${E2E_PRE_GATE_STATE_ROOT:-${TMPDIR:-/tmp}/clerum-pre-gate-sync}"
- cluster_fingerprint_file="${pre_gate_state_root}/${HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID}/cluster.sha"
- [ -r "$cluster_fingerprint_file" ] ||
- die "pre-gate fingerprint evidence is missing at ${cluster_fingerprint_file}"
- expected_cluster_fingerprint="$(sed -n '1p' "$cluster_fingerprint_file")"
- [[ "$expected_cluster_fingerprint" =~ ^[0-9a-f]{40}$ ]] ||
- die "pre-gate fingerprint evidence is malformed"
- [ -n "${E2E_EXPECTED_PRE_GATE_GATE:-}" ] ||
- die "E2E_EXPECTED_PRE_GATE_GATE must name the gate recorded by the branch-owned pre-gate sync"
-
- HCC_BRANCH_GATE_SYNC_MARKER="$(
- kctl get configmap clerum-pre-gate-sync-state -n "$marker_namespace" -o json 2>/dev/null
- )" || die "cluster has no readable pre-gate sync marker"
- actual_worktree_id="$(jq -r '.data.worktreeId // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
- actual_head="$(jq -r '.data.gitHead // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
- actual_cluster_fingerprint="$(
- jq -r '.data.clusterFingerprint // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER"
- )"
- actual_gate="$(jq -r '.data.gate // ""' <<<"$HCC_BRANCH_GATE_SYNC_MARKER")"
+ expected_worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"
+ expected_head="$(git -C "$repo_root" rev-parse HEAD)"
+ actual_worktree_id="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \
+ -o jsonpath='{.data.worktreeId}' 2>/dev/null || true)"
+ actual_head="$(kctl get configmap clerum-pre-gate-sync-state -n control-plane \
+ -o jsonpath='{.data.gitHead}' 2>/dev/null || true)"
- [ "$actual_worktree_id" = "$HCC_BRANCH_GATE_EXPECTED_WORKTREE_ID" ] ||
+ [ "$actual_worktree_id" = "$expected_worktree_id" ] ||
die "cluster ownership marker does not match this worktree; run minikube-pre-gate-sync first"
- [ "$actual_head" = "$HCC_BRANCH_GATE_EXPECTED_HEAD" ] ||
- die "cluster HEAD marker ${actual_head:-missing} does not match ${HCC_BRANCH_GATE_EXPECTED_HEAD}"
- [ "$actual_cluster_fingerprint" = "$expected_cluster_fingerprint" ] ||
- die "cluster fingerprint marker does not match this worktree's last pre-gate sync"
- [ "$actual_gate" = "$E2E_EXPECTED_PRE_GATE_GATE" ] ||
- die "cluster gate marker ${actual_gate:-missing} does not match expected ${E2E_EXPECTED_PRE_GATE_GATE}"
+ [ "$actual_head" = "$expected_head" ] ||
+ die "cluster HEAD marker ${actual_head:-missing} does not match ${expected_head}"
}
create_hcc_api_proxy() {
@@ -338,10 +284,10 @@ delete_hcc_proxy_fixture() {
}
restore_hcc_after_fault_injection() {
- local host_override port_override api_cidrs_override host_aliases desired ready
+ local host_override port_override host_aliases desired ready
kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
- KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1
+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- >/dev/null || return 1
kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=merge \
-p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1
kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null || return 1
@@ -350,12 +296,10 @@ restore_hcc_after_fault_injection() {
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"
port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"
- api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"
host_aliases="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')"
desired="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"
ready="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.status.readyReplicas}')"
- [ -z "$host_override$port_override$api_cidrs_override$host_aliases" ] && [ "$desired" = "$ready" ]
+ [ -z "$host_override$port_override$host_aliases" ] && [ "$desired" = "$ready" ]
}
print_hcc_repair_instructions() {
@@ -364,7 +308,7 @@ HCC restoration failed. Proxy resources were retained for repair.
Context: ${E2E_KUBECONTEXT}
Proxy: ${HCC_NS}/${PROXY_NAME}
Repair commands:
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS-
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT-
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} patch deployment/${HCC_DEPLOY} --type=merge -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}'
kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s
After HCC is healthy, delete ${PROXY_NAME}, ${PROXY_EGRESS_NP}, ${HCC_PROXY_NP}, and ${PROBE_EGRESS_NP} in ${HCC_NS}.
diff --git a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh b/scripts/e2e/_lib/hcc-watch-recovery-logs.sh
index f0857c3d5..b227248a6 100755
--- a/scripts/e2e/_lib/hcc-watch-recovery-logs.sh
+++ b/scripts/e2e/_lib/hcc-watch-recovery-logs.sh
@@ -2,18 +2,6 @@
# Log capture and causal recovery-cycle assertions for the HCC watch gate.
-hcc_log_snapshot_contains() {
- local snapshot=$1 marker=$2
- grep -Fq -- "$marker" <<<"$snapshot"
-}
-
-hcc_initial_pass_snapshot_is_active() {
- local snapshot=$1 start_marker=$2 complete_marker=$3 fail_marker=$4
- hcc_log_snapshot_contains "$snapshot" "$start_marker" &&
- ! hcc_log_snapshot_contains "$snapshot" "$complete_marker" &&
- ! hcc_log_snapshot_contains "$snapshot" "$fail_marker"
-}
-
hcc_recovery_logs() {
[ -n "$START_TIME" ] || return 0
grep -E 'CommunicationChannel|Listing all Hosts|Reconciling [0-9]+ Host|Completed Host reconciliation|Periodic resync' \
@@ -71,8 +59,7 @@ recovery_cycle_used_fresh_host_inventory() {
hcc_recovery_logs | awk \
-v target="$cycle" -v host_namespace="$HOST_NS" -v expected="$expected_count" '
BEGIN {
- channel_inventory = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"
- host_inventory = "Reconciling " expected " Host(s) after Host watch recovery convergence"
+ inventory_line = "Reconciling " expected " Host(s) for lifecycle after CommunicationChannel recovery"
succeeded = 0
}
/CommunicationChannel watch ended;/ {
@@ -81,7 +68,7 @@ recovery_cycle_used_fresh_host_inventory() {
if (inside) {
listed = 0
recovered = 0
- completion_mode = ""
+ inventory_matched = 0
}
next
}
@@ -91,20 +78,11 @@ recovery_cycle_used_fresh_host_inventory() {
inside && /Recovered [0-9]+ CommunicationChannel\(s\) into cache/ {
recovered = 1
}
- inside && listed && recovered && index($0, channel_inventory) > 0 {
- completion_mode = "channel"
+ inside && index($0, inventory_line) > 0 {
+ inventory_matched = 1
}
- inside && listed && recovered && index($0, host_inventory) > 0 {
- completion_mode = "host"
- }
- inside && completion_mode == "channel" &&
- /Completed Host reconciliation after CommunicationChannel recovery$/ {
- succeeded = 1
- exit
- }
- inside && completion_mode == "host" &&
- /Completed Host reconciliation after Host watch recovery convergence$/ {
- succeeded = 1
+ inside && /Completed Host reconciliation after CommunicationChannel recovery$/ {
+ succeeded = listed && recovered && inventory_matched
exit
}
END {
@@ -112,15 +90,3 @@ recovery_cycle_used_fresh_host_inventory() {
}
'
}
-
-wait_recovery_cycle_fresh_host_inventory() {
- local cycle=$1 expected_count=$2 timeout=$3 elapsed=0
- while [ "$elapsed" -lt "$timeout" ]; do
- require_hcc_recovery_log_stream || return 1
- recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_count" && return 0
- sleep 2
- elapsed=$((elapsed + 2))
- done
- hcc_recovery_logs >&2
- return 1
-}
diff --git a/scripts/e2e/e2e-crd-field-injection.sh b/scripts/e2e/e2e-crd-field-injection.sh
index d4fe358f8..af8a25829 100755
--- a/scripts/e2e/e2e-crd-field-injection.sh
+++ b/scripts/e2e/e2e-crd-field-injection.sh
@@ -55,7 +55,6 @@ require_safe_kube_context || exit 1
cleanup_e2e_resources
ensure_e2e_resources_absent ||
fail "Stale e2e-sec resources remain after cleanup; refusing stale-positive E2E run"
-# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.
[ "$e2e_fail" -gt 0 ] && exit 1
# ═══ Test 1: imagePullPolicy override prevention ═════════════════════
@@ -79,11 +78,8 @@ spec:
url: http://e2e-sec-pullpolicy.mcp-server.svc:3000/mcp
EOF
-if wait_for_deployment "$MCP_NS" e2e-sec-pullpolicy "$TIMEOUT_POD"; then
- ok "imagePullPolicy fixture rendered a ready Deployment"
-else
- fail "imagePullPolicy fixture did not render a ready Deployment"
-fi
+log "Waiting for HCC to create Deployment..."
+sleep 15
PULL_POLICY=$(kctl get deployment e2e-sec-pullpolicy -n "$MCP_NS" \
-o jsonpath='{.spec.template.spec.containers[0].imagePullPolicy}' 2>/dev/null || echo "NOT_FOUND")
@@ -241,11 +237,7 @@ spec:
value: postgres://safe
EOF
-if wait_for_deployment "$MCP_NS" e2e-sec-envvars "$TIMEOUT_POD"; then
- ok "Dangerous-env fixture rendered a ready Deployment"
-else
- fail "Dangerous-env fixture did not render a ready Deployment"
-fi
+sleep 15
ENV_JSON=$(kctl get deployment e2e-sec-envvars -n "$MCP_NS" \
-o jsonpath='{.spec.template.spec.containers[0].env}' 2>/dev/null || echo "[]")
@@ -284,11 +276,7 @@ spec:
- DAC_OVERRIDE
EOF
-if wait_for_deployment "$MCP_NS" e2e-sec-uid70 "$TIMEOUT_POD"; then
- ok "Non-root UID fixture rendered a ready Deployment"
-else
- fail "Non-root UID fixture did not render a ready Deployment"
-fi
+sleep 15
UID_70=$(kctl get deployment e2e-sec-uid70 -n "$MCP_NS" \
-o jsonpath='{.spec.template.spec.securityContext.runAsUser}' 2>/dev/null || echo "NOT_FOUND")
@@ -304,6 +292,5 @@ ensure_e2e_resources_absent && ok "All e2e-sec resources cleaned up" || fail "e2
# ═══ Summary ═════════════════════════════════════════════════════════
header "Summary"
-# shellcheck disable=SC2154 # e2e-lib.sh owns the shared result counters.
echo -e " ${GREEN}PASS${NC}: ${e2e_pass} ${RED}FAIL${NC}: ${e2e_fail} Total: ${e2e_total}"
[ "$e2e_fail" -gt 0 ] && exit 1 || exit 0
diff --git a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
index 480b9abd9..d1a38ed5b 100755
--- a/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
+++ b/scripts/e2e/e2e-hcc-communicationchannel-watch-recovery.sh
@@ -35,15 +35,13 @@ command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 1; }
exit 1
}
kctl get nodes -o json | jq -e \
- --arg context "$E2E_KUBECONTEXT" \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null || {
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] != null)' >/dev/null || {
echo "Refusing HCC fault injection: target is not a minikube cluster." >&2
exit 1
}
HCC_NS="${HCC_NS:-control-plane}"
HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
-MCP_NS="${MCP_NS:-mcp-server}"
SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm-stateless}"
HOST_NS="${MCP_HOST_NS:-mcp-host}"
CHANNEL_NS="${CHANNELS_NS:-channels}"
@@ -63,7 +61,7 @@ HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-stream.XXXXXX")"
NON_FIXTURE_BASELINE="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-baseline.XXXXXX")"
NON_FIXTURE_AFTER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-after.XXXXXX")"
CLEANUP_KINDS="deployments,services,secrets,serviceaccounts,roles,rolebindings,networkpolicies,persistentvolumeclaims"
-START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY="" ORIGINAL_REPLICAS=""
+START_TIME="" HCC_UID="" HCC_RESTARTS="" CONTROL_IDENTITY=""
HCC_LOG_STREAM_PID=""
# Mirrors COMMUNICATION_CHANNEL_CACHE_RECOVERY_RETRY_MS in k8sClient.ts.
CC_RECOVERY_RETRY_SECONDS=5
@@ -77,35 +75,6 @@ HCC_GATE_FINALIZATION_FAILURE=""
die() { fail "$*"; exit 1; }
now_rfc3339() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
-# This gate deliberately changes only the HCC Kubernetes API path. If its
-# rollout fails before the normal recovery log stream starts, preserve the
-# bounded pod state and both container log buffers that distinguish a proxy,
-# policy, or process-startup failure. Cleanup still restores HCC afterwards.
-print_hcc_proxy_rollout_diagnostics() {
- local pod
-
- echo "HCC proxy rollout diagnostics (before restoration):" >&2
- kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o json |
- jq -r '
- .items[] |
- .metadata.name as $pod |
- .status.phase as $phase |
- .status.containerStatuses[]? |
- "pod=\($pod) phase=\($phase // "unknown") ready=\(.ready) restarts=\(.restartCount) waiting=\(.state.waiting.reason // "none") terminated=\(.state.terminated.reason // "none") exitCode=\(.state.terminated.exitCode // "none") lastTerminated=\(.lastState.terminated.reason // "none") lastExitCode=\(.lastState.terminated.exitCode // "none")"
- ' >&2 || true
-
- while IFS= read -r pod; do
- [ -n "$pod" ] || continue
- echo "HCC proxy rollout previous logs for pod/${pod}:" >&2
- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \
- --previous --tail=120 >&2 || true
- echo "HCC proxy rollout current logs for pod/${pod}:" >&2
- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller \
- --tail=120 >&2 || true
- done < <(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
- -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
-}
-
cleanup() {
local status=$? cleanup_failed=0 restore_ok=1 host remaining
set +e
@@ -157,8 +126,6 @@ cleanup() {
else
cleanup_failed=1
print_hcc_repair_instructions
- echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}" >&2
- echo " kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s" >&2
fi
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
rm -f "$HCC_LOG_BUFFER" "$NON_FIXTURE_BASELINE" "$NON_FIXTURE_AFTER"
@@ -176,35 +143,19 @@ header "HCC CommunicationChannel watch recovery"
require_branch_owned_hcc_gate
acquire_hcc_watch_gate_lock || die "another HCC watch-recovery gate owns this profile"
kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" >/dev/null
-ORIGINAL_REPLICAS="$(
- kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}'
-)"
-[ "$ORIGINAL_REPLICAS" = 1 ] ||
- die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
kctl get host "$SOURCE_HOST" -n "$HOST_NS" >/dev/null
host_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_HOST")].name}')"
port_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="KUBERNETES_SERVICE_PORT")].name}')"
-api_cidrs_override="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CONTEXT_MAPPER_K8S_API_CIDRS")].name}')"
-[ -z "$host_override$port_override$api_cidrs_override" ] ||
- die "HCC already has an explicit Kubernetes service or API CIDR override"
+[ -z "$host_override$port_override" ] || die "HCC already has an explicit Kubernetes service override"
[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||
die "HCC already has hostAliases; refusing a non-restorable fault injection"
HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
-o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"
-K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
- -c host-context-controller -- printenv KUBERNETES_SERVICE_HOST)" ||
- die "could not read the Kubernetes-injected API service IP"
-kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \
- node -e 'const { isIP } = require("node:net"); process.exit(isIP(process.argv[1]) === 4 ? 0 : 1)' \
- "$K8S_API_SERVICE_HOST" >/dev/null ||
- die "Kubernetes-injected API service host is not an IPv4 address"
-K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"
log "Creating an isolated TCP proxy for the Kubernetes API"
create_hcc_api_proxy
@@ -213,27 +164,19 @@ verify_hcc_proxy_network_policy
PROXY_IP="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"
[ -n "$PROXY_IP" ] || die "proxy Service has no ClusterIP"
HCC_PATCHED=1
-hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" --arg api_cidr "$K8S_API_CIDR" '{spec:{template:{spec:{
+hcc_proxy_patch="$(jq -cn --arg ip "$PROXY_IP" '{spec:{template:{spec:{
hostAliases:[{ip:$ip,hostnames:["kubernetes.default.svc"]}],
containers:[{name:"host-context-controller",env:[
{name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},
- {name:"KUBERNETES_SERVICE_PORT",value:"443"},
- {name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr}
+ {name:"KUBERNETES_SERVICE_PORT",value:"443"}
]}]
}}}}')"
kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$hcc_proxy_patch" >/dev/null
-if ! kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null; then
- print_hcc_proxy_rollout_diagnostics
+kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||
die "HCC did not become ready through the API proxy"
-fi
initial_hcc_identity="$(wait_for_hcc_identity 30)" || die "could not capture HCC pod identity"
read -r HCC_UID HCC_RESTARTS <<<"$initial_hcc_identity"
[ -n "$HCC_UID" ] && [ "$HCC_UID" != invalid ] || die "could not capture HCC pod identity"
-proxy_policy_cidrs="$(kctl get networkpolicy "allow-k8s-api-egress-${MCP_NS}" -n "$MCP_NS" -o json |
- jq -r '[.spec.egress[0].to[]?.ipBlock.cidr] | join(",")')" ||
- die "could not read the proxy-run API egress policy"
-[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ] ||
- die "proxy-run API egress policy did not retain the injected Kubernetes API CIDR"
ok "HCC is healthy through the proxy (pod=${HCC_UID}, restarts=${HCC_RESTARTS})"
log "Creating one stateful control and two stateless Host fixtures"
@@ -271,13 +214,14 @@ ok "isolated Host fleet reached its baseline"
run_recovery_cycle() {
local cycle=$1 cycle_logs recovery_logs outage_started outage_seconds max_retry_failures list_pattern
- local host_list_pattern ended_before started_before recovered_before failed_before
+ local host_list_pattern ended_before started_before recovered_before completed_before failed_before
local list_before host_list_before expected_host_count
local failed_after retry_failures
cycle_logs="$(hcc_recovery_logs)"
ended_before="$(log_count_from "$cycle_logs" 'CommunicationChannel watch ended;')"
started_before="$(log_count_from "$cycle_logs" 'Starting CommunicationChannel watch')"
recovered_before="$(log_count_from "$cycle_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"
+ completed_before="$(log_count_from "$cycle_logs" 'Completed Host reconciliation after CommunicationChannel recovery$')"
failed_before="$(log_count_from "$cycle_logs" 'cache recovery failed;')"
list_pattern="Listing all CommunicationChannels in namespace ${CHANNEL_NS}"
host_list_pattern="Listing all Hosts in namespace ${HOST_NS}"
@@ -321,7 +265,10 @@ run_recovery_cycle() {
max_retry_failures=$((
(outage_seconds + CC_RECOVERY_RETRY_SECONDS - 1) / CC_RECOVERY_RETRY_SECONDS + 2
))
- wait_recovery_cycle_fresh_host_inventory "$cycle" "$expected_host_count" 180 ||
+ wait_log_count 'Completed Host reconciliation after CommunicationChannel recovery$' \
+ "$((completed_before + 1))" 180 ||
+ die "Host fleet did not converge after snapshot recovery in cycle ${cycle}"
+ recovery_cycle_used_fresh_host_inventory "$cycle" "$expected_host_count" ||
die "cycle ${cycle} did not reconcile the fresh ${expected_host_count}-Host inventory"
recovery_logs="$(hcc_recovery_logs)"
failed_after="$(log_count_from "$recovery_logs" 'cache recovery failed;')"
@@ -369,18 +316,14 @@ ended="$(log_count_from "$final_logs" 'CommunicationChannel watch ended;')"
started="$(log_count_from "$final_logs" 'Starting CommunicationChannel watch')"
recovered="$(log_count_from "$final_logs" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache')"
failed="$(log_count_from "$final_logs" 'cache recovery failed;')"
-fleet_passes="$(
- log_count_from "$final_logs" \
- 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'
-)"
+fleet_passes="$(log_count_from "$final_logs" 'Completed Host reconciliation after CommunicationChannel')"
[ "$ended" = 2 ] && [ "$started" = 2 ] && [ "$recovered" = 2 ] && \
[ "$failed" = "$TOTAL_RETRY_FAILURES" ] && [ "$failed" -le "$TOTAL_RETRY_FAILURE_LIMIT" ] ||
die "unexpected watch lifecycle counts: ended=${ended}, started=${started}, recovered=${recovered}, failed=${failed}"
[ "$fleet_passes" -ge 2 ] && [ "$fleet_passes" -le 4 ] ||
die "unexpected successful fleet reconcile count ${fleet_passes}; expected 2..4 passes"
-# Each recovered snapshot requires one causally ordered successful pass; a
-# Host-watch recovery pass may cover the same fresh CC+Host inventories. At
-# most one already queued covering pass may also finish per cycle. The deterministic
+# Each recovered snapshot requires one successful pass; at most one already
+# queued fail-closed pass may also finish per cycle. The deterministic
# max-concurrency=1 proof stays in k8sClient unit tests where overlapping passes
# can be barrier-controlled.
diff --git a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh b/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
deleted file mode 100755
index cccb17538..000000000
--- a/scripts/e2e/e2e-hcc-mcp-context-readiness.sh
+++ /dev/null
@@ -1,1790 +0,0 @@
-#!/usr/bin/env bash
-# Proves that HCC readiness is independent from the exact McpServer ->
-# external-egress -> runtime -> Context NetworkPolicy initial convergence path.
-#
-# The gate runs only on an exact-HEAD, branch-owned Minikube profile. It routes
-# HCC DNS through an isolated in-cluster proxy that forwards every query except
-# one fixture hostname. The gate first lets HCC create a real external-egress
-# policy. A same-intent restart then proves the safety pass RETAINS that
-# identity-stable DNS-derived allow without attempting DNS and certifies global
-# readiness with the allow still present — no deny window — while the affected
-# runtime stays blocked. DNS release lets the additive lane refresh the current
-# TCP policy and runtime. A second phase changes that binding's protocol while
-# HCC is stopped and proves the divergent contract: an identity-changing
-# TCP-to-UDP replacement IS revoked before readiness. A real Context watch update
-# still converges its three
-# NetworkPolicies. The fleet is wider than the bounded worker width, covering
-# the original large-fleet symptom.
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=scripts/e2e/e2e-lib.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/e2e-lib.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
-
-[ -n "$E2E_KUBECONTEXT" ] || {
- echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2
- exit 1
-}
-is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
- echo "Refusing HCC MCP readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
- exit 1
-}
-require_safe_kube_context
-[ "${E2E_HCC_MCP_READINESS_FAULT_INJECTION:-0}" = 1 ] || {
- echo "Set E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC DNS fault injection." >&2
- exit 1
-}
-command -v jq >/dev/null 2>&1 || {
- echo "jq is required" >&2
- exit 1
-}
-
-HCC_NS="${HCC_NS:-control-plane}"
-HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
-MCP_NS="${MCP_SERVER_NS:-mcp-server}"
-HOST_NS="${MCP_HOST_NS:-mcp-host}"
-RPC_PROXY_NS="${RPC_PROXY_NS:-rpc-proxy}"
-RUN_ID="$(date +%s)-$$"
-SUITE_NAME="hcc-mcp-readiness"
-RUN_LABEL="$(truncate_rfc1123 "$RUN_ID")"
-MCP_NAME="$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}")"
-CONTEXT_NAME="$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}")"
-CONTEXT_ID="$CONTEXT_NAME"
-MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"
-TARGET_DNS="$(truncate_rfc1123 "hcc-readiness-${RUN_ID}").example.com"
-DNS_BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-dns-${RUN_ID}")"
-DNS_BLOCKER_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-policy")"
-HCC_DNS_POLICY="$(truncate_rfc1123 "${DNS_BLOCKER_NAME}-from-hcc")"
-CONTEXT_POLICY="ctx-${CONTEXT_ID}-${MCP_NAME}"
-CONTEXT_EGRESS_POLICY="${CONTEXT_POLICY}-egress"
-RPC_EGRESS_POLICY="rpc-egress-${CONTEXT_ID}-${MCP_NAME}"
-MANAGED_BY_LABEL="clerum.io/managed-by"
-MANAGED_BY_VALUE="host-context-controller"
-MCP_SERVER_LABEL="clerum.io/mcpserver"
-POLICY_TYPE_LABEL="clerum.io/policy-type"
-CONTEXT_LABEL="clerum.io/context"
-EXTERNAL_EGRESS_CIDR="93.184.216.34/32"
-EXTERNAL_EGRESS_PORT=443
-INITIAL_EXTERNAL_EGRESS_PROTOCOL=TCP
-EXTERNAL_EGRESS_PROTOCOL=UDP
-ORIGINAL_REPLICAS=""
-ORIGINAL_DNS_POLICY=""
-ORIGINAL_DNS_CONFIG=""
-DNS_BLOCKER_IP=""
-DNS_RELEASE_COUNT_AT_HOLD=0
-DNS_HOLD_COUNT_AT_ARM=0
-DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=0
-UPSTREAM_DNS_IP=""
-NEW_HCC_POD=""
-HCC_UID=""
-HCC_RESTARTS=""
-HCC_PORT=""
-HCC_MUTATED=0
-DNS_BLOCKER_CREATED=0
-MCP_CREATED=0
-CONTEXT_CREATED=0
-PEER_FLEET_CREATED=0
-MCP_FLEET_NAMES=("$MCP_NAME")
-CONTEXT_FLEET_NAMES=("$CONTEXT_NAME")
-# Read and mutated by the sourced compare-and-swap lock helper.
-# shellcheck disable=SC2034
-HCC_GATE_LOCK_ACQUIRED=0
-# shellcheck disable=SC2034
-HCC_GATE_LOCK_NAME=""
-# shellcheck disable=SC2034
-HCC_GATE_LOCK_UID=""
-# shellcheck disable=SC2034
-HCC_GATE_FINALIZATION_FAILURE=""
-HCC_READY_PROBE_DIAGNOSTIC=""
-BASELINE_POLICY_SNAPSHOT=""
-SAME_INTENT_EGRESS_SNAPSHOT=""
-
-# Clean-window negative markers. Consumed ONLY while the DNS fault injection is
-# NOT applied (pre-injection pre-checks, post-restore cleanup): under the
-# injection the additive NetworkPolicy lane legitimately fails and schedules
-# retries, so these negatives false-fail inside the injected window (removed
-# there by 1abcfb1/15997353/e7064a7f). Inventoried in
-# scripts/tests/test-hcc-watch-recovery-gate.sh marker_bindings.
-NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'
-NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'
-NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'
-NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '
-clean_window_checks_run=0
-
-[[ "$MCP_FLEET_SIZE" =~ ^[0-9]+$ ]] && [ "$MCP_FLEET_SIZE" -gt 10 ] &&
- [ "$MCP_FLEET_SIZE" -le 24 ] || {
- echo "E2E_HCC_MCP_FLEET_SIZE must be an integer from 11 through 24; got ${MCP_FLEET_SIZE}." >&2
- exit 1
-}
-for ((index = 2; index <= MCP_FLEET_SIZE; index++)); do
- MCP_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-mcp-${RUN_ID}-${index}")")
- CONTEXT_FLEET_NAMES+=("$(truncate_rfc1123 "e2e-hcc-context-${RUN_ID}-${index}")")
-done
-
-die() {
- if [ -n "$NEW_HCC_POD" ]; then
- echo "Recent HCC logs from ${NEW_HCC_POD}:" >&2
- kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller \
- --tail=300 >&2 || true
- fi
- if [ "$DNS_BLOCKER_CREATED" = 1 ]; then
- echo "Recent DNS blocker logs:" >&2
- kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" --tail=120 >&2 || true
- fi
- fail "$*"
- exit 1
-}
-
-wait_until() {
- local timeout=$1 description=$2
- shift 2
- local deadline now
- deadline=$(( $(date +%s) + timeout ))
- while :; do
- "$@" && return 0
- now="$(date +%s)"
- [ "$now" -lt "$deadline" ] || break
- sleep 1
- done
- echo "Timed out after ${timeout}s waiting for ${description}" >&2
- return 1
-}
-
-wait_until_fast() {
- local timeout=$1 description=$2
- shift 2
- local deadline now rc
- deadline=$(( $(date +%s) + timeout ))
- while :; do
- "$@" && return 0
- rc=$?
- # rc=2 is a fatal, non-retryable ordering violation (e.g. 200 observed with
- # a stale policy still present): stop polling immediately so a later clean
- # poll cannot mask it. rc=1 is benign "not ready yet".
- if [ "$rc" -eq 2 ]; then
- echo "FATAL ordering violation while waiting for ${description}: ${HCC_READY_PROBE_DIAGNOSTIC:-unset}" >&2
- return 2
- fi
- now="$(date +%s)"
- [ "$now" -lt "$deadline" ] || break
- sleep 0.5
- done
- echo "Timed out after ${timeout}s waiting for ${description}" >&2
- return 1
-}
-
-running_hcc_pod() {
- local rows
- rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
- --field-selector=status.phase=Running \
- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
- 2>/dev/null)" || return 1
- awk -F '\t' '$1 != "" && $4 == "" { print $1; exit }' <<<"$rows"
-}
-
-capture_hcc() {
- NEW_HCC_POD="$(running_hcc_pod)"
- [ -n "$NEW_HCC_POD" ]
-}
-
-hcc_pods_absent() {
- local pods
- pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||
- return 1
- [ -z "$pods" ]
-}
-
-hcc_log_contains() {
- local marker=$1 logs
- [ -n "$NEW_HCC_POD" ] || return 1
- logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \
- -c host-context-controller 2>/dev/null)" || return 1
- grep -Fq "$marker" <<<"$logs"
-}
-
-dns_log_contains() {
- local marker=$1 logs
- logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
- return 1
- grep -Fq "$marker" <<<"$logs"
-}
-
-dns_release_count() {
- local marker="released DNS A ${TARGET_DNS}" logs
- logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
- return 1
- grep -Fc "$marker" <<<"$logs" || true
-}
-
-dns_hold_count() {
- local marker="holding DNS A ${TARGET_DNS}" logs
- logs="$(kctl logs "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" ||
- return 1
- grep -Fc "$marker" <<<"$logs" || true
-}
-
-dns_hold_observed_since_arm() {
- [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_ARM" ]
-}
-
-dns_has_not_released_since_hold() {
- [ "$(dns_release_count)" = "$DNS_RELEASE_COUNT_AT_HOLD" ]
-}
-
-dns_released_since_hold() {
- [ "$(dns_release_count)" -gt "$DNS_RELEASE_COUNT_AT_HOLD" ]
-}
-
-external_egress_retry_progress_is_observed() {
- local logs marker
- [ -n "$NEW_HCC_POD" ] || return 1
- logs="$(kctl logs "pod/${NEW_HCC_POD}" -n "$HCC_NS" \
- -c host-context-controller 2>/dev/null)" || return 1
- marker="Scheduling external egress retry "
- grep -Fq \
- "Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};" <<<"$logs" &&
- awk \
- -v marker="$marker" \
- -v server="for McpServer \"${MCP_NAME}\"" '
- index($0, marker) && index($0, server) { found = 1 }
- END { exit found ? 0 : 1 }
- ' <<<"$logs"
-}
-
-dns_retry_query_observed_since_schedule() {
- [ "$(dns_hold_count)" -gt "$DNS_HOLD_COUNT_AT_RETRY_SCHEDULE" ]
-}
-
-hcc_identity_is_stable() {
- local row uid restarts
- row="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \
- -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}' \
- 2>/dev/null)" || return 1
- read -r uid restarts <<<"$row"
- [ "$uid" = "$HCC_UID" ] && [ "$restarts" = "$HCC_RESTARTS" ]
-}
-
-hcc_kubernetes_readiness_is_exact() {
- [ -n "$NEW_HCC_POD" ] && [ -n "$HCC_UID" ] || return 1
- kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" -o json 2>/dev/null |
- jq -e --arg uid "$HCC_UID" '
- .metadata.uid == $uid and
- any(.status.conditions[]?; .type == "Ready" and .status == "True") and
- any(.status.containerStatuses[]?;
- .name == "host-context-controller" and .ready == true
- )
- ' >/dev/null || return 1
- kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json 2>/dev/null |
- jq -e '
- .status.observedGeneration == .metadata.generation and
- (.spec.replicas // 0) == 1 and
- (.status.updatedReplicas // 0) == 1 and
- (.status.readyReplicas // 0) == 1 and
- (.status.availableReplicas // 0) == 1 and
- (.status.unavailableReplicas // 0) == 0
- ' >/dev/null
-}
-
-resource_absent() {
- local kind=$1 name=$2 namespace=$3 found
- found="$(
- kctl get "$kind" "$name" -n "$namespace" --ignore-not-found -o name 2>/dev/null
- )" || return 1
- [ -z "$found" ]
-}
-
-fixture_mcp_runtime_absent() {
- local external_egress_policies
- fixture_runtime_absent || return 1
- external_egress_policies="$(
- kctl get networkpolicy -n "$MCP_NS" \
- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
- -o name 2>/dev/null
- )" || return 1
- [ -z "$external_egress_policies" ]
-}
-
-fixture_runtime_absent() {
- resource_absent deployment "$MCP_NAME" "$MCP_NS" &&
- resource_absent service "$MCP_NAME" "$MCP_NS"
-}
-
-mcpserver_current_ready() {
- local name=$1
- kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null |
- jq -e '
- .metadata.generation as $generation |
- any(
- .status.conditions[]?;
- .type == "Ready" and .status == "True" and
- (.observedGeneration // -1) == $generation
- )
- ' >/dev/null
-}
-
-mcpserver_current_ready_absent() {
- local name=$1 resource
- resource="$(kctl get mcpserver "$name" -n "$MCP_NS" -o json 2>/dev/null)" ||
- return 1
- jq -e '
- .metadata.generation as $generation |
- any(
- .status.conditions[]?;
- .type == "Ready" and .status == "True" and
- (.observedGeneration // -1) == $generation
- ) | not
- ' <<<"$resource" >/dev/null
-}
-
-fixture_runtime_and_ready_absent() {
- fixture_runtime_absent && mcpserver_current_ready_absent "$MCP_NAME"
-}
-
-context_policies_absent() {
- resource_absent networkpolicy "$CONTEXT_POLICY" "$MCP_NS" &&
- resource_absent networkpolicy "$CONTEXT_EGRESS_POLICY" "$HOST_NS" &&
- resource_absent networkpolicy "$RPC_EGRESS_POLICY" "$RPC_PROXY_NS"
-}
-
-context_policies_converged() {
- local context_id=${1:-$CONTEXT_ID}
- local context_name=${2:-$CONTEXT_NAME}
- local server=${3:-$MCP_NAME}
- local ingress_policy="ctx-${context_id}-${server}"
- local host_egress_policy="${ingress_policy}-egress"
- local rpc_egress_policy="rpc-egress-${context_id}-${server}"
- kctl get networkpolicy "$ingress_policy" -n "$MCP_NS" -o json |
- jq -e \
- --arg contextId "$context_id" \
- --arg contextName "$context_name" \
- --arg server "$server" \
- --arg hostNs "$HOST_NS" \
- --arg rpcNs "$RPC_PROXY_NS" \
- --arg mcpNs "$MCP_NS" \
- --arg managedByLabel "$MANAGED_BY_LABEL" \
- --arg managedByValue "$MANAGED_BY_VALUE" \
- --arg contextLabel "$CONTEXT_LABEL" \
- --arg serverLabel "$MCP_SERVER_LABEL" \
- --arg policyTypeLabel "$POLICY_TYPE_LABEL" '
- .metadata.labels[$managedByLabel] == $managedByValue and
- .metadata.labels[$policyTypeLabel] == "context-allow" and
- .metadata.labels[$contextLabel] == $contextId and
- .metadata.labels[$serverLabel] == $server and
- .spec.podSelector.matchLabels[$serverLabel] == $server and
- .spec.policyTypes == ["Ingress"] and
- (.spec.ingress | length) == 1 and
- .spec.ingress[0].ports == [{port: 3000, protocol: "TCP"}] and
- (.spec.ingress[0].from | length) == 3 and
- any(.spec.ingress[0].from[];
- .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $hostNs and
- .podSelector.matchLabels[$managedByLabel] == $managedByValue and
- .podSelector.matchLabels[$contextLabel] == $contextName
- ) and
- any(.spec.ingress[0].from[];
- .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $rpcNs and
- .podSelector.matchLabels.app == "rpc-proxy"
- ) and
- any(.spec.ingress[0].from[];
- .namespaceSelector.matchLabels["kubernetes.io/metadata.name"] == $mcpNs and
- .podSelector.matchLabels.app == "mcp-proxy"
- )
- ' >/dev/null || return 1
- kctl get networkpolicy "$host_egress_policy" -n "$HOST_NS" -o json |
- jq -e \
- --arg contextId "$context_id" \
- --arg contextName "$context_name" \
- --arg server "$server" \
- --arg mcpNs "$MCP_NS" \
- --arg managedByLabel "$MANAGED_BY_LABEL" \
- --arg managedByValue "$MANAGED_BY_VALUE" \
- --arg contextLabel "$CONTEXT_LABEL" \
- --arg serverLabel "$MCP_SERVER_LABEL" \
- --arg policyTypeLabel "$POLICY_TYPE_LABEL" '
- .metadata.labels[$managedByLabel] == $managedByValue and
- .metadata.labels[$policyTypeLabel] == "context-allow" and
- .metadata.labels[$contextLabel] == $contextId and
- .metadata.labels[$serverLabel] == $server and
- .spec.podSelector.matchLabels[$managedByLabel] == $managedByValue and
- .spec.podSelector.matchLabels[$contextLabel] == $contextName and
- .spec.policyTypes == ["Egress"] and
- (.spec.egress | length) == 1 and
- .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and
- (.spec.egress[0].to | length) == 1 and
- .spec.egress[0].to[0].namespaceSelector.matchLabels[
- "kubernetes.io/metadata.name"
- ] == $mcpNs and
- .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server
- ' >/dev/null || return 1
- kctl get networkpolicy "$rpc_egress_policy" -n "$RPC_PROXY_NS" -o json |
- jq -e \
- --arg contextId "$context_id" \
- --arg server "$server" \
- --arg mcpNs "$MCP_NS" \
- --arg managedByLabel "$MANAGED_BY_LABEL" \
- --arg managedByValue "$MANAGED_BY_VALUE" \
- --arg contextLabel "$CONTEXT_LABEL" \
- --arg serverLabel "$MCP_SERVER_LABEL" \
- --arg policyTypeLabel "$POLICY_TYPE_LABEL" '
- .metadata.labels[$managedByLabel] == $managedByValue and
- .metadata.labels[$policyTypeLabel] == "rpc-proxy-egress" and
- .metadata.labels[$contextLabel] == $contextId and
- .metadata.labels[$serverLabel] == $server and
- .spec.podSelector.matchLabels.app == "rpc-proxy" and
- .spec.policyTypes == ["Egress"] and
- (.spec.egress | length) == 1 and
- .spec.egress[0].ports == [{port: 3000, protocol: "TCP"}] and
- (.spec.egress[0].to | length) == 1 and
- .spec.egress[0].to[0].namespaceSelector.matchLabels[
- "kubernetes.io/metadata.name"
- ] == $mcpNs and
- .spec.egress[0].to[0].podSelector.matchLabels[$serverLabel] == $server
- ' >/dev/null
-}
-
-external_egress_policy_converged_with_protocol() {
- local protocol=$1
- kctl get networkpolicy -n "$MCP_NS" \
- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
- -o json |
- jq -e \
- --arg server "$MCP_NAME" \
- --arg cidr "$EXTERNAL_EGRESS_CIDR" \
- --arg managedByLabel "$MANAGED_BY_LABEL" \
- --arg managedByValue "$MANAGED_BY_VALUE" \
- --arg serverLabel "$MCP_SERVER_LABEL" \
- --arg policyTypeLabel "$POLICY_TYPE_LABEL" \
- --arg protocol "$protocol" \
- --argjson port "$EXTERNAL_EGRESS_PORT" '
- (.items | length) == 1 and
- .items[0].metadata.labels[$managedByLabel] == $managedByValue and
- .items[0].metadata.labels[$policyTypeLabel] == "external-egress" and
- .items[0].metadata.labels[$serverLabel] == $server and
- .items[0].spec.podSelector.matchLabels[$serverLabel] == $server and
- .items[0].spec.policyTypes == ["Egress"] and
- (.items[0].spec.egress | length) == 1 and
- .items[0].spec.egress[0].ports == [{port: $port, protocol: $protocol}] and
- (.items[0].spec.egress[0].to | length) == 1 and
- .items[0].spec.egress[0].to[0].ipBlock == {cidr: $cidr}
- ' >/dev/null
-}
-
-external_egress_policy_converged() {
- external_egress_policy_converged_with_protocol "$EXTERNAL_EGRESS_PROTOCOL"
-}
-
-external_egress_policy_absent() {
- local policies
- policies="$(
- kctl get networkpolicy -n "$MCP_NS" \
- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
- -o name 2>/dev/null
- )" || return 1
- [ -z "$policies" ]
-}
-
-clean_window_netpol_log_negatives_hold() {
- # Family 1, fail-closed snapshot: read ONE authoritative log snapshot so a
- # kubectl logs failure cannot make the negative assertions pass vacuously.
- local pod=$1 logs
- [ -n "$pod" ] || return 1
- logs="$(kctl logs "pod/${pod}" -n "$HCC_NS" \
- -c host-context-controller 2>/dev/null)" || return 1
- ! grep -Fq "$NETPOL_INITIAL_FAILED_MARKER" <<<"$logs" &&
- ! grep -Fq "$NETPOL_ADDITIVE_FAILED_MARKER" <<<"$logs" &&
- ! grep -Fq "$NETPOL_RETRY_SCHEDULED_MARKER" <<<"$logs" &&
- ! grep -Fq "${NETPOL_CONTEXT_FAILED_MARKER}${CONTEXT_NAME}:" <<<"$logs"
-}
-
-clean_window_runtime_and_ready_absent() {
- # Family 3, clean-window form. fixture_runtime_and_ready_absent requires the
- # McpServer object to EXIST (mcpserver_current_ready_absent fails on a missing
- # object); in clean windows the fixture may not exist at all, which is the
- # strongest form of absence. An API error still fails closed.
- fixture_runtime_absent || return 1
- if ! resource_absent mcpserver "$MCP_NAME" "$MCP_NS"; then
- mcpserver_current_ready_absent "$MCP_NAME" || return 1
- fi
- external_egress_policy_absent
-}
-
-fixture_runtime_converged_with_protocol() {
- local protocol=$1 desired available
- desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
- -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1
- available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
- -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1
- [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1
- kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1
- mcpserver_current_ready "$MCP_NAME" || return 1
- external_egress_policy_converged_with_protocol "$protocol"
-}
-
-fixture_resources_absent() {
- fixture_mcp_runtime_absent &&
- context_policies_absent &&
- peer_fleet_runtime_absent &&
- peer_fleet_policies_absent
-}
-
-fixture_inputs_absent() {
- resource_absent context "$CONTEXT_NAME" "$MCP_NS" &&
- resource_absent mcpserver "$MCP_NAME" "$MCP_NS" && peer_fleet_inputs_absent
-}
-
-peer_fleet_inputs_absent() {
- local index
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- resource_absent context "${CONTEXT_FLEET_NAMES[index]}" "$MCP_NS" || return 1
- resource_absent mcpserver "${MCP_FLEET_NAMES[index]}" "$MCP_NS" || return 1
- done
-}
-
-peer_fleet_runtime_absent() {
- local index server
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- server="${MCP_FLEET_NAMES[index]}"
- resource_absent deployment "$server" "$MCP_NS" || return 1
- resource_absent service "$server" "$MCP_NS" || return 1
- done
-}
-
-peer_fleet_policies_absent() {
- local index server context context_policy external_policies
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- server="${MCP_FLEET_NAMES[index]}"
- context="${CONTEXT_FLEET_NAMES[index]}"
- context_policy="ctx-${context}-${server}"
- resource_absent networkpolicy "$context_policy" "$MCP_NS" || return 1
- resource_absent networkpolicy "${context_policy}-egress" "$HOST_NS" || return 1
- resource_absent networkpolicy "rpc-egress-${context}-${server}" "$RPC_PROXY_NS" || return 1
- external_policies="$(
- kctl get networkpolicy -n "$MCP_NS" \
- -l "${MCP_SERVER_LABEL}=${server},${POLICY_TYPE_LABEL}=external-egress" \
- -o name 2>/dev/null
- )" || return 1
- [ -z "$external_policies" ] || return 1
- done
-}
-
-peer_fleet_converged() {
- local index server context desired available
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- server="${MCP_FLEET_NAMES[index]}"
- context="${CONTEXT_FLEET_NAMES[index]}"
- desired="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1
- available="$(kctl get deployment "$server" -n "$MCP_NS" -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1
- [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1
- kctl get service "$server" -n "$MCP_NS" >/dev/null 2>&1 || return 1
- mcpserver_current_ready "$server" || return 1
- context_policies_converged "$context" "$context" "$server" || return 1
- done
-}
-
-baseline_policy_snapshot() {
- kctl get \
- "networkpolicy/deny-all-${MCP_NS}" \
- "networkpolicy/allow-dns-egress-${MCP_NS}" \
- networkpolicy/allow-host-context-controller-api \
- -n "$MCP_NS" -o json 2>/dev/null |
- jq -ceS \
- --arg managedBy "$MANAGED_BY_VALUE" \
- --arg deny "deny-all-${MCP_NS}" \
- --arg dns "allow-dns-egress-${MCP_NS}" '
- [.items[] | {
- name: .metadata.name,
- managedBy: .metadata.labels["clerum.io/managed-by"],
- policyType: .metadata.labels["clerum.io/policy-type"],
- spec
- }]
- | sort_by(.name)
- | select(length == 3)
- | select(all(.[];
- .managedBy == $managedBy and
- ((.name == $deny and .policyType == "default-deny") or
- (.name == $dns and .policyType == "infrastructure") or
- (.name == "allow-host-context-controller-api" and .policyType == "allow-api"))))
- '
-}
-
-baseline_policies_unchanged() {
- local current
- [ -n "$BASELINE_POLICY_SNAPSHOT" ] || return 1
- current="$(baseline_policy_snapshot)" || return 1
- [ "$current" = "$BASELINE_POLICY_SNAPSHOT" ]
-}
-
-context_watch_has_latest_spec() {
- kctl get context "$CONTEXT_NAME" -n "$MCP_NS" -o json |
- jq -e --arg server "$MCP_NAME" '.spec.mcpServers == [$server]' >/dev/null
-}
-
-mcp_egress_binding_is_udp() {
- kctl get mcpserver "$MCP_NAME" -n "$MCP_NS" -o json |
- jq -e --arg dns "$TARGET_DNS" '
- (.spec.egressBindings | length) == 1 and
- .spec.egressBindings[0].egressClass == "exact-host" and
- .spec.egressBindings[0].dns == $dns and
- .spec.egressBindings[0].port == 443 and
- .spec.egressBindings[0].protocol == "UDP"
- ' >/dev/null
-}
-
-create_peer_fleet() {
- local index server context
- # Arm cleanup before the first write so a partial apply cannot strand input
- # CRDs or leave HCC deliberately scaled down during failure recovery.
- PEER_FLEET_CREATED=1
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- server="${MCP_FLEET_NAMES[index]}"
- context="${CONTEXT_FLEET_NAMES[index]}"
- kctl apply -f - >/dev/null <<EOF
-apiVersion: clerum.io/v1alpha1
-kind: Context
-metadata:
- name: ${context}
- namespace: ${MCP_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- contextId: ${context}
- description: Gate-owned peer Context for HCC readiness fleet coverage.
- mcpServers: [${server}]
----
-apiVersion: clerum.io/v1alpha1
-kind: McpServer
-metadata:
- name: ${server}
- namespace: ${MCP_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- contextRef: ${context}
- description: Gate-owned peer McpServer for bounded HCC fleet coverage.
- image: clerum/mock-mcp-server:test
- imagePullPolicy: IfNotPresent
- transport:
- type: streamableHttp
- url: http://${server}.${MCP_NS}.svc.cluster.local:3000/mcp
- port: 3000
-EOF
- done
-}
-
-fixture_converged() {
- local desired available
- desired="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
- -o jsonpath='{.spec.replicas}' 2>/dev/null)" || return 1
- available="$(kctl get deployment "$MCP_NAME" -n "$MCP_NS" \
- -o jsonpath='{.status.availableReplicas}' 2>/dev/null)" || return 1
- [ "${desired:-0}" -ge 1 ] && [ "${available:-0}" -ge 1 ] || return 1
- kctl get service "$MCP_NAME" -n "$MCP_NS" >/dev/null 2>&1 || return 1
- mcpserver_current_ready "$MCP_NAME" || return 1
- external_egress_policy_converged || return 1
- context_policies_converged
-}
-
-probe_hcc_final_context() {
- local probe_script
- probe_script="$(cat <<'NODE'
-const http = require('http');
-const context = process.argv[1];
-const expected = process.argv[2];
-function get(path) {
- return new Promise((resolve, reject) => {
- const request = http.get({
- host: '127.0.0.1',
- port: Number(process.env.HCC_E2E_PORT),
- path
- }, response => {
- let body = '';
- response.setEncoding('utf8');
- response.on('data', chunk => { body += chunk; });
- response.on('end', () => resolve({status: response.statusCode, body}));
- });
- request.setTimeout(5000, () => request.destroy(new Error('timeout')));
- request.once('error', reject);
- });
-}
-(async () => {
- const ready = await get('/ready');
- const scoped = await get('/api/v1/mcpservers/context/' + encodeURIComponent(context));
- const body = JSON.parse(scoped.body);
- const fixture = body.servers?.find(server => server.name === expected);
- if (ready.status !== 200 || JSON.parse(ready.body).ready !== true) {
- throw new Error('readiness regressed: ' + ready.status + ' ' + ready.body);
- }
- if (scoped.status !== 200 || !fixture || fixture.status?.ready !== true) {
- throw new Error('latest Context did not converge in discovery: ' + scoped.status + ' ' + scoped.body);
- }
- console.log(JSON.stringify({
- readyStatus: ready.status,
- scopedDiscoveryStatus: scoped.status,
- contextRef: body.contextRef,
- fixture: fixture.name,
- fixtureReady: fixture.status.ready
- }));
-})().catch(error => { console.error(error.message); process.exit(1); });
-NODE
-)"
- kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \
- env "HCC_E2E_PORT=${HCC_PORT}" \
- node -e "$probe_script" "$CONTEXT_NAME" "$MCP_NAME"
-}
-
-probe_hcc_ready_pod() {
- local pod
- pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1
- kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \
- env "HCC_E2E_PORT=${HCC_PORT}" \
- node -e '
- const http = require("node:http");
- const request = http.get(
- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
- response => {
- let body = "";
- response.setEncoding("utf8");
- response.on("data", chunk => { body += chunk; });
- response.on("end", () => {
- let parsed;
- try { parsed = JSON.parse(body); } catch { process.exit(2); }
- process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);
- });
- }
- );
- request.setTimeout(5000, () => request.destroy(new Error("timeout")));
- request.once("error", () => process.exit(4));
- ' >/dev/null 2>&1
-}
-
-probe_new_hcc_ready_endpoint() {
- local output first_line
- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=pod-not-selected"
- [ -n "$NEW_HCC_POD" ] || return 1
- if output="$(
- kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \
- env "HCC_E2E_PORT=${HCC_PORT}" \
- node -e '
- const http = require("node:http");
- let finished = false;
- const finish = (code, category, detail = "") => {
- if (finished) return;
- finished = true;
- if (category) {
- console.error(`HCC_READY_PROBE category=${category}${detail}`);
- }
- process.exitCode = code;
- };
- const request = http.get(
- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
- response => {
- let body = "";
- response.setEncoding("utf8");
- response.on("data", chunk => { body += chunk; });
- response.on("end", () => {
- if (response.statusCode !== 200) {
- finish(3, "http-status", ` status=${response.statusCode ?? "unknown"}`);
- return;
- }
- let parsed;
- try {
- parsed = JSON.parse(body);
- } catch {
- finish(2, "json-invalid");
- return;
- }
- if (parsed?.ready !== true) {
- finish(3, "json-ready-false");
- return;
- }
- finish(0);
- });
- }
- );
- request.setTimeout(2000, () => {
- finish(4, "timeout");
- request.destroy();
- });
- request.once("error", () => finish(5, "transport"));
- ' 2>&1
- )"; then
- HCC_READY_PROBE_DIAGNOSTIC=""
- return 0
- fi
- first_line="${output%%$'\n'*}"
- case "$first_line" in
- "HCC_READY_PROBE category="*)
- printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s' "$first_line"
- ;;
- *)
- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=kubectl-exec"
- ;;
- esac
- return 1
-}
-
-# Third, independent signal of the safety contract (M6): the authoritative pass
-# must actually have MEASURED and REVOKED. The unit tests assert the readiness
-# predicate over vi.fn() mocks, so a metric rename would leave them green; these
-# family-name literals live in the gate, so a rename turns the REAL gate red.
-# Pure adjudication over a /metrics scrape — unit-testable without a cluster.
-safety_pass_metrics_are_certified() {
- local metrics=$1
- grep -Fq -- 'clerum_hcc_networkpolicy_safety_pass_duration_seconds' <<<"$metrics" || return 1
- # At least one policy revoked by an authoritative pass ($NF is the sample
- # value; tolerant of extra/reordered labels).
- awk '$0 ~ /^clerum_hcc_networkpolicy_safety_pass_policies_total\{/ && /operation="revoked"/ { if ($NF + 0 >= 1) found = 1 } END { exit(found ? 0 : 1) }' <<<"$metrics"
-}
-
-# Scrapes the running replacement pod's /metrics over the same in-pod node/http
-# channel the /ready probes use, then adjudicates. The sub-0.5s window B4 leaves
-# unobservable on /ready is covered here: the counter is monotonic.
-safety_pass_metrics_certify() {
- local metrics
- metrics="$(kctl exec "pod/${NEW_HCC_POD}" -n "$HCC_NS" -c host-context-controller -- \
- env "HCC_E2E_PORT=${HCC_PORT}" \
- node -e '
- const http = require("node:http");
- const request = http.get(
- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/metrics"},
- response => {
- let body = "";
- response.setEncoding("utf8");
- response.on("data", chunk => { body += chunk; });
- response.on("end", () => {
- process.stdout.write(body);
- process.exit(response.statusCode === 200 ? 0 : 1);
- });
- }
- );
- request.setTimeout(2000, () => { request.destroy(); process.exit(1); });
- request.on("error", () => process.exit(1));
- ')" || return 1
- safety_pass_metrics_are_certified "$metrics"
-}
-
-hcc_ready_after_stale_policy_revoked() {
- # A single poll that sees 200 WHILE the stale policy is still present is a
- # safety-contract violation and must be FATAL, never retried: retrying lets a
- # later poll (policy already revoked) turn the violation into a false PASS.
- # rc=2 signals that fatal ordering violation to wait_until_fast; rc=1 is a
- # benign "not ready yet, keep polling". The sub-0.5s window where revocation
- # races the probe inside one poll stays unobservable on THIS signal — the
- # third signal (M6, /metrics) covers it.
- local policy_absent_before=0
- external_egress_policy_absent && policy_absent_before=1
- probe_new_hcc_ready_endpoint || return 1
- if external_egress_policy_absent; then
- [ "$policy_absent_before" -eq 1 ] && return 0
- # Revocation raced the probe inside this poll: the final state is clean but
- # this poll cannot adjudicate the order. Retry to adjudicate cleanly.
- return 1
- fi
- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=ready-200-with-stale-policy-present"
- return 2
-}
-
-# Canonical, order-stable projection of the single external-egress policy for the
-# fixture server: name, ownership labels, and full spec. Used to prove a
-# same-intent restart RETAINS the identity-stable DNS allow byte-for-byte (B3
-# retain-identity-stable contract), the inverse of the divergent revoke path.
-external_egress_policy_snapshot() {
- kctl get networkpolicy -n "$MCP_NS" \
- -l "${MCP_SERVER_LABEL}=${MCP_NAME},${POLICY_TYPE_LABEL}=external-egress" \
- -o json 2>/dev/null |
- jq -ceS \
- --arg managedByLabel "$MANAGED_BY_LABEL" \
- --arg managedByValue "$MANAGED_BY_VALUE" \
- --arg policyTypeLabel "$POLICY_TYPE_LABEL" \
- --arg serverLabel "$MCP_SERVER_LABEL" \
- --arg server "$MCP_NAME" '
- [.items[] | {
- name: .metadata.name,
- managedBy: .metadata.labels[$managedByLabel],
- policyType: .metadata.labels[$policyTypeLabel],
- serverLabel: .metadata.labels[$serverLabel],
- spec
- }]
- | select(length == 1)
- | .[0]
- | select(.managedBy == $managedByValue
- and .policyType == "external-egress"
- and .serverLabel == $server)
- '
-}
-
-# True when the external-egress policy is present and identical (spec +
-# ownership) to the snapshot captured before the same-intent restart. An API
-# error or a missing/renamed policy fails closed.
-identity_stable_policy_retained() {
- local current
- [ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] || return 1
- current="$(external_egress_policy_snapshot)" || return 1
- [ -n "$current" ] || return 1
- [ "$current" = "$SAME_INTENT_EGRESS_SNAPSHOT" ]
-}
-
-hcc_ready_with_identity_stable_policy_retained() {
- # INVERSE of the pre-B3 revoke-before-ready contract. B3 retains an
- # identity-stable DNS allow across a same-intent restart with NO deny window and
- # certifies /ready=200 WHILE that allow is still present and unchanged. A single
- # poll that reaches /ready=200 with the identity-stable allow ABSENT from the
- # start is the retain regression (a deny window) and must be FATAL, never
- # retried: retrying would let a later poll — after the held-refresh timeout
- # legitimately fail-closed deletes the allow — mask the regression as a PASS.
- # rc=2 signals that fatal violation to wait_until_fast; rc=1 is a benign
- # "not ready yet, keep polling".
- local retained_before=0
- identity_stable_policy_retained && retained_before=1
- probe_new_hcc_ready_endpoint || return 1
- if identity_stable_policy_retained; then
- [ "$retained_before" -eq 1 ] && return 0
- # The allow reappeared within this poll: adjudicate cleanly on the next poll.
- return 1
- fi
- if [ "$retained_before" -eq 1 ]; then
- # The allow was present immediately before this probe and is gone now: a
- # legitimate fail-closed refresh timeout can race the probe inside one poll.
- # Retry to adjudicate — a real retain regression never presents the allow at
- # readiness, so a later poll still returns rc=2.
- return 1
- fi
- HCC_READY_PROBE_DIAGNOSTIC="HCC_READY_PROBE category=identity-stable-policy-revoked"
- return 2
-}
-
-hcc_restore_is_verified() {
- local deployment
- deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1
- jq -e \
- --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \
- --argjson replicas "${ORIGINAL_REPLICAS:-1}" '
- .spec.replicas == $replicas and
- (.status.readyReplicas // 0) == $replicas and
- (.status.availableReplicas // 0) == $replicas and
- (.spec.template.spec.dnsPolicy // "ClusterFirst") == $dnsPolicy and
- (.spec.template.spec.dnsConfig // null) == null
- ' <<<"$deployment" >/dev/null || return 1
- probe_hcc_ready_pod
-}
-
-release_dns_blocker() {
- local release_script
- release_script="$(cat <<'NODE'
-const http = require('http');
-const request = http.request(
- {host: '127.0.0.1', port: 8090, path: '/release', method: 'POST'},
- response => {
- response.resume();
- response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));
- }
-);
-request.setTimeout(5000, () => request.destroy(new Error('timeout')));
-request.once('error', error => { console.error(error.message); process.exit(3); });
-request.end();
-NODE
-)"
- kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \
- node -e "$release_script"
-}
-
-hold_dns_blocker() {
- local hold_script
- DNS_HOLD_COUNT_AT_ARM="$(dns_hold_count)" ||
- return 1
- hold_script="$(cat <<'NODE'
-const http = require('http');
-const request = http.request(
- {host: '127.0.0.1', port: 8090, path: '/hold', method: 'POST'},
- response => {
- response.resume();
- response.on('end', () => process.exit(response.statusCode === 200 ? 0 : 2));
- }
-);
-request.setTimeout(5000, () => request.destroy(new Error('timeout')));
-request.once('error', error => { console.error(error.message); process.exit(3); });
-request.end();
-NODE
-)"
- kctl exec "deployment/${DNS_BLOCKER_NAME}" -n "$HCC_NS" -c dns-blocker -- \
- node -e "$hold_script"
-}
-
-delete_fixture_runtime_leaving_dns_policy_for_safety() {
- local failed=0
- kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete service "$MCP_NAME" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- [ "$failed" = 0 ] && fixture_runtime_absent
-}
-
-delete_dns_blocker_fixture() {
- local failed=0
- kctl delete networkpolicy "$HCC_DNS_POLICY" "$DNS_BLOCKER_POLICY" -n "$HCC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- [ "$failed" = 0 ]
-}
-
-delete_residual_fixture_runtime() {
- local failed=0
- kctl delete deployment "$MCP_NAME" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete service "$MCP_NAME" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete configmap "${MCP_NAME}-nginx-conf" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy "$CONTEXT_POLICY" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy "$CONTEXT_EGRESS_POLICY" -n "$HOST_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy "$RPC_EGRESS_POLICY" -n "$RPC_PROXY_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${MCP_NAME}" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- delete_peer_fleet_runtime || failed=1
- [ "$failed" = 0 ]
-}
-
-delete_peer_fleet_runtime() {
- local index server context context_policy failed=0
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- server="${MCP_FLEET_NAMES[index]}"
- context="${CONTEXT_FLEET_NAMES[index]}"
- context_policy="ctx-${context}-${server}"
- kctl delete deployment "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \
- >/dev/null 2>&1 || failed=1
- kctl delete service "$server" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \
- >/dev/null 2>&1 || failed=1
- kctl delete configmap "${server}-nginx-conf" -n "$MCP_NS" --ignore-not-found --wait=true --timeout=60s \
- >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy -n "$MCP_NS" -l "${MCP_SERVER_LABEL}=${server}" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy "${context_policy}-egress" -n "$HOST_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete networkpolicy "rpc-egress-${context}-${server}" -n "$RPC_PROXY_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- done
- [ "$failed" = 0 ]
-}
-
-delete_peer_fleet_inputs() {
- local index failed=0
- for ((index = 1; index < ${#MCP_FLEET_NAMES[@]}; index++)); do
- kctl delete context "${CONTEXT_FLEET_NAMES[index]}" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete mcpserver "${MCP_FLEET_NAMES[index]}" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- done
- [ "$failed" = 0 ]
-}
-
-suite_resources_absent() {
- local namespaced cluster_crds
- namespaced="$(kctl get deployment,service,networkpolicy -A \
- -l "e2e.clerum.io/suite=${SUITE_NAME}" -o name 2>/dev/null)" || return 1
- cluster_crds="$(
- kctl get mcpserver,context -A -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- -o name 2>/dev/null
- )" || return 1
- [ -z "$namespaced$cluster_crds" ]
-}
-
-print_repair_instructions() {
- cat >&2 <<EOF
-HCC MCP readiness gate cleanup could not restore a verified clean state.
-Context: ${E2E_KUBECONTEXT}
-HCC: ${HCC_NS}/${HCC_DEPLOY}
-DNS blocker: ${HCC_NS}/${DNS_BLOCKER_NAME}
-Fixtures: ${MCP_NS}/${MCP_NAME}, ${MCP_NS}/${CONTEXT_NAME}
-
-Inspect before changing anything:
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
- kubectl --context=${E2E_KUBECONTEXT} get deployment,service,networkpolicy -A -l e2e.clerum.io/suite=${SUITE_NAME}
- kubectl --context=${E2E_KUBECONTEXT} get mcpserver,context -A -l e2e.clerum.io/suite=${SUITE_NAME}
-
-Do not remove the HCC gate lock until HCC health and fixture absence are verified.
-
-Restore the original HCC replica count before verifying rollout and /ready:
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=240s
-EOF
-}
-
-cleanup() {
- local status=$? cleanup_failed=0 restore_ok=1 fixture_inputs_removed=1 restore_patch
- local clean_window_pod
- trap - EXIT
- set +e
-
- if [ "$HCC_MUTATED" = 1 ]; then
- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1 ||
- restore_ok=0
- if [ "$restore_ok" = 1 ]; then
- wait_until 120 "HCC pods to stop before fixture cleanup" \
- hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
- fi
- fi
-
- if [ "$CONTEXT_CREATED" = 1 ]; then
- kctl delete context "$CONTEXT_NAME" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||
- {
- cleanup_failed=1
- fixture_inputs_removed=0
- }
- fi
- if [ "$MCP_CREATED" = 1 ]; then
- kctl delete mcpserver "$MCP_NAME" -n "$MCP_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 ||
- {
- cleanup_failed=1
- fixture_inputs_removed=0
- }
- fi
- if [ "$PEER_FLEET_CREATED" = 1 ]; then
- delete_peer_fleet_inputs || {
- cleanup_failed=1
- fixture_inputs_removed=0
- }
- fi
- if ! wait_until 60 "gate-owned Context and McpServer inputs to disappear" \
- fixture_inputs_absent >/dev/null 2>&1; then
- cleanup_failed=1
- fixture_inputs_removed=0
- fi
-
- if [ "$HCC_MUTATED" = 1 ]; then
- if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then
- restore_patch="$(jq -cn --arg dnsPolicy "$ORIGINAL_DNS_POLICY" \
- '{spec:{template:{spec:{dnsPolicy:$dnsPolicy,dnsConfig:null}}}}')"
- kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \
- -p "$restore_patch" >/dev/null 2>&1 || restore_ok=0
- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
- kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- --timeout=240s >/dev/null 2>&1 || restore_ok=0
- if [ "$restore_ok" = 1 ]; then
- wait_until 60 "restored HCC DNS, replicas, Ready pod, and /ready endpoint" \
- hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0
- fi
- else
- restore_ok=0
- fi
- fi
-
- if [ "$restore_ok" = 1 ]; then
- if ! wait_until 180 "HCC to remove all MCP/Context fixture runtime resources" \
- fixture_resources_absent >/dev/null 2>&1; then
- cleanup_failed=1
- delete_residual_fixture_runtime || cleanup_failed=1
- fi
- if [ "$DNS_BLOCKER_CREATED" = 1 ]; then
- delete_dns_blocker_fixture || cleanup_failed=1
- fi
- wait_until 60 "all HCC MCP readiness suite resources to disappear" \
- suite_resources_absent >/dev/null 2>&1 || cleanup_failed=1
-
- # Clean window B: DNS restored and fixture inputs gone. The restored pod is
- # new (scale-0 preceded the restore), so its log is scoped to this window.
- clean_window_pod="$(running_hcc_pod)"
- if [ -n "$clean_window_pod" ] &&
- clean_window_netpol_log_negatives_hold "$clean_window_pod"; then
- clean_window_checks_run=$((clean_window_checks_run + 1))
- else
- fail "post-restore clean-window NetworkPolicy log negatives did not hold"
- cleanup_failed=1
- fi
- if fixture_mcp_runtime_absent; then
- clean_window_checks_run=$((clean_window_checks_run + 1))
- else
- fail "post-restore fixture runtime or external-egress policy survived"
- cleanup_failed=1
- fi
- if clean_window_runtime_and_ready_absent; then
- clean_window_checks_run=$((clean_window_checks_run + 1))
- else
- fail "post-restore McpServer Ready status or external-egress policy survived"
- cleanup_failed=1
- fi
- else
- cleanup_failed=1
- fi
-
- if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then
- cleanup_failed=1
- fi
-
- if [ "$status" = 0 ]; then
- [ "${clean_window_checks_run:-0}" -gt 0 ] ||
- die "no clean-window negative checks ran (clean_window_checks_run=0)"
- fi
-
- if [ "$cleanup_failed" -ne 0 ]; then
- echo "HCC MCP readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2
- [ "$restore_ok" = 1 ] || print_repair_instructions
- [ "$status" = 0 ] && status=1
- fi
- if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
- header "HCC McpServer/Context readiness gate passed"
- echo "$final_probe"
- fi
- exit "$status"
-}
-trap cleanup EXIT
-
-header "HCC readiness during initial McpServer and Context convergence"
-
-require_branch_owned_hcc_gate "$HCC_NS"
-kctl get nodes -o json |
- jq -e --arg context "$E2E_KUBECONTEXT" \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
- die "target context is not a minikube cluster"
-suite_resources_absent ||
- die "stale ${SUITE_NAME} resources exist; inspect them before starting a new fault injection"
-acquire_hcc_watch_gate_lock ||
- die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
-ok "branch helper, profile, exact HEAD/fingerprint/gate, and exclusive HCC lock verified"
-
-# Clean window A: the DNS fault injection is not applied yet. The branch-deployed
-# HCC must show a clean NetworkPolicy lane and zero fixture residue before the
-# gate mutates anything.
-clean_window_pod="$(running_hcc_pod)" && [ -n "$clean_window_pod" ] ||
- die "no Running HCC pod before fault injection"
-clean_window_netpol_log_negatives_hold "$clean_window_pod" ||
- die "pre-injection HCC logs already contain NetworkPolicy-lane failure or retry markers"
-clean_window_checks_run=$((clean_window_checks_run + 1))
-fixture_mcp_runtime_absent ||
- die "pre-injection fixture runtime or external-egress policy already exists"
-clean_window_checks_run=$((clean_window_checks_run + 1))
-clean_window_runtime_and_ready_absent ||
- die "pre-injection McpServer Ready status or external-egress policy already exists"
-clean_window_checks_run=$((clean_window_checks_run + 1))
-ok "all three clean-window negative families hold before DNS fault injection"
-
-ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- -o jsonpath='{.spec.replicas}')"
-[ "$ORIGINAL_REPLICAS" = 1 ] ||
- die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
-ORIGINAL_DNS_POLICY="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- -o jsonpath='{.spec.template.spec.dnsPolicy}')"
-ORIGINAL_DNS_POLICY="${ORIGINAL_DNS_POLICY:-ClusterFirst}"
-[ "$ORIGINAL_DNS_POLICY" = ClusterFirst ] ||
- die "HCC already has non-default dnsPolicy ${ORIGINAL_DNS_POLICY}; refusing a non-restorable fixture"
-ORIGINAL_DNS_CONFIG="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json |
- jq -c '.spec.template.spec.dnsConfig // null')"
-[ "$ORIGINAL_DNS_CONFIG" = null ] ||
- die "HCC already has dnsConfig ${ORIGINAL_DNS_CONFIG}; refusing a non-restorable fixture"
-HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
-[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"
-HCC_PORT="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json | jq -r '
- .spec.template.spec.containers[]? |
- select(.name == "host-context-controller") as $container |
- (
- [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //
- [$container.ports[]? | select(.name == "http") | .containerPort][0] //
- empty
- )
-')"
-[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||
- die "could not resolve a valid HCC HTTP port from the Deployment"
-UPSTREAM_DNS_IP="$(kctl get service kube-dns -n kube-system -o jsonpath='{.spec.clusterIP}')"
-[ -n "$UPSTREAM_DNS_IP" ] || die "could not resolve kube-system/kube-dns ClusterIP"
-running_dns_pods="$(kctl get pods -n kube-system -l k8s-app=kube-dns \
- --field-selector=status.phase=Running -o name)"
-[ -n "$running_dns_pods" ] ||
- die "no Running kube-dns pod is available"
-
-DNS_BLOCKER_CREATED=1
-kctl apply -f - >/dev/null <<EOF
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: ${DNS_BLOCKER_NAME}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: ${DNS_BLOCKER_NAME}
- template:
- metadata:
- labels:
- app: ${DNS_BLOCKER_NAME}
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
- spec:
- automountServiceAccountToken: false
- terminationGracePeriodSeconds: 1
- containers:
- - name: dns-blocker
- image: ${HCC_IMAGE}
- imagePullPolicy: IfNotPresent
- command: [node, -e]
- args:
- - >-
- const dgram=require('node:dgram'),http=require('node:http'),
- net=require('node:net');
- const target=process.env.TARGET_DNS,upstream=process.env.UPSTREAM_DNS;
- const udp=dgram.createSocket('udp4');let released=process.env.START_RELEASED==='1';const held=[];
- function question(message){let offset=12,labels=[];while(offset<message.length){
- const length=message[offset++];if(length===0)break;if(offset+length>message.length)
- return null;labels.push(message.subarray(offset,offset+length).toString('ascii'));
- offset+=length;}if(offset+4>message.length)return null;return
- {name:labels.join('.'),type:message.readUInt16BE(offset),end:offset+4};}
- function answer(message,parsed){const header=Buffer.from(message.subarray(0,12));
- header.writeUInt16BE(0x8180,2);header.writeUInt16BE(1,6);
- header.writeUInt16BE(0,8);header.writeUInt16BE(0,10);
- const record=Buffer.from([0xc0,0x0c,0x00,0x01,0x00,0x01,0x00,0x00,
- 0x00,0x1e,0x00,0x04,93,184,216,34]);
- return Buffer.concat([header,message.subarray(12,parsed.end),record]);}
- function tcpFrame(message){const frame=Buffer.alloc(message.length+2);
- frame.writeUInt16BE(message.length,0);message.copy(frame,2);return frame;}
- function respond(entry){const response=answer(entry.message,entry.parsed);
- if(entry.transport==='udp'){udp.send(response,entry.rinfo.port,
- entry.rinfo.address);console.log('released DNS A '+target+' via UDP');return;}
- if(!entry.socket.destroyed){entry.socket.write(tcpFrame(response));
- console.log('released DNS A '+target+' via TCP');}}
- function forwardUdp(message,rinfo){const relay=dgram.createSocket('udp4');
- const timer=setTimeout(()=>relay.close(),5000);relay.once('message',response=>{
- clearTimeout(timer);udp.send(response,rinfo.port,rinfo.address);relay.close();});
- relay.once('error',()=>{clearTimeout(timer);relay.close();});
- relay.send(message,53,upstream);}
- function forwardTcp(message,socket){const relay=net.createConnection(
- {host:upstream,port:53},()=>relay.write(tcpFrame(message)));
- let pending=Buffer.alloc(0);relay.setTimeout(5000,()=>relay.destroy());
- relay.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);
- if(pending.length<2)return;const length=pending.readUInt16BE(0);
- if(pending.length<length+2)return;if(!socket.destroyed)
- socket.write(pending.subarray(0,length+2));relay.destroy();});
- relay.once('error',()=>relay.destroy());}
- function handle(message,entry){const parsed=question(message);
- if(parsed&&parsed.name===target&&parsed.type===1){entry.message=message;
- entry.parsed=parsed;if(released)respond(entry);else{held.push(entry);
- console.log('holding DNS A '+target+' via '+entry.transport.toUpperCase());}
- return;}if(entry.transport==='udp')forwardUdp(message,entry.rinfo);
- else forwardTcp(message,entry.socket);}
- let listeners=0;function listening(){listeners+=1;if(listeners===2)
- console.log('dns hold proxy ready for '+target+' over UDP and TCP');}
- udp.on('message',(message,rinfo)=>handle(message,{transport:'udp',rinfo}));
- udp.bind(8053,'0.0.0.0',listening);
- const tcp=net.createServer(socket=>{let pending=Buffer.alloc(0);
- socket.on('data',chunk=>{pending=Buffer.concat([pending,chunk]);
- while(pending.length>=2){const length=pending.readUInt16BE(0);
- if(pending.length<length+2)return;const message=pending.subarray(2,length+2);
- pending=pending.subarray(length+2);handle(message,{transport:'tcp',socket});}});
- });tcp.listen(8053,'0.0.0.0',listening);
- http.createServer((request,response)=>{if(request.method==='POST'&&
- request.url==='/release'){released=true;while(held.length)respond(held.shift());
- response.writeHead(200);response.end('released');return;}if(request.method==='POST'&&
- request.url==='/hold'){released=false;response.writeHead(200);response.end('holding');return;}
- response.writeHead(404);response.end();}).listen(8090,'127.0.0.1');
- env:
- - name: TARGET_DNS
- value: ${TARGET_DNS}
- - name: START_RELEASED
- value: "1"
- - name: UPSTREAM_DNS
- value: ${UPSTREAM_DNS_IP}
- resources:
- requests: {cpu: 5m, memory: 20Mi}
- limits: {cpu: 75m, memory: 64Mi}
- securityContext:
- allowPrivilegeEscalation: false
- capabilities: {drop: [ALL]}
- readOnlyRootFilesystem: true
- runAsNonRoot: true
- runAsUser: 1000
- runAsGroup: 1000
- seccompProfile: {type: RuntimeDefault}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: ${DNS_BLOCKER_NAME}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- selector:
- app: ${DNS_BLOCKER_NAME}
- ports:
- - name: dns-udp
- protocol: UDP
- port: 53
- targetPort: 8053
- - name: dns-tcp
- protocol: TCP
- port: 53
- targetPort: 8053
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: ${DNS_BLOCKER_POLICY}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- podSelector:
- matchLabels:
- app: ${DNS_BLOCKER_NAME}
- policyTypes: [Ingress, Egress]
- ingress:
- - from:
- - podSelector:
- matchLabels:
- app: ${HCC_DEPLOY}
- ports:
- - {protocol: UDP, port: 8053}
- - {protocol: UDP, port: 53}
- - {protocol: TCP, port: 8053}
- - {protocol: TCP, port: 53}
- egress:
- - to:
- - namespaceSelector:
- matchLabels:
- kubernetes.io/metadata.name: kube-system
- podSelector:
- matchLabels:
- k8s-app: kube-dns
- ports:
- - {protocol: UDP, port: 53}
- - {protocol: TCP, port: 53}
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: ${HCC_DNS_POLICY}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- podSelector:
- matchLabels:
- app: ${HCC_DEPLOY}
- policyTypes: [Egress]
- egress:
- - to:
- - podSelector:
- matchLabels:
- app: ${DNS_BLOCKER_NAME}
- ports:
- - {protocol: UDP, port: 53}
- - {protocol: UDP, port: 8053}
- - {protocol: TCP, port: 53}
- - {protocol: TCP, port: 8053}
-EOF
-kctl rollout status deployment "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
- --timeout=90s >/dev/null || die "DNS blocker did not become ready"
-wait_until 30 "DNS blocker process to listen" \
- dns_log_contains "dns hold proxy ready for ${TARGET_DNS}" ||
- die "DNS blocker process did not start"
-DNS_BLOCKER_IP="$(kctl get service "$DNS_BLOCKER_NAME" -n "$HCC_NS" \
- -o jsonpath='{.spec.clusterIP}')"
-[ -n "$DNS_BLOCKER_IP" ] || die "DNS blocker Service has no ClusterIP"
-ok "isolated DNS hold proxy is ready and forwards non-fixture UDP/TCP queries to CoreDNS"
-
-[ "${clean_window_checks_run:-0}" -ge 3 ] ||
- die "pre-injection clean-window negatives did not run before the DNS mutation"
-
-HCC_MUTATED=1
-dns_patch="$(jq -cn --arg nameserver "$DNS_BLOCKER_IP" '
- {spec:{template:{spec:{
- dnsPolicy:"None",
- dnsConfig:{
- nameservers:[$nameserver],
- options:[
- {name:"ndots",value:"1"},
- {name:"timeout",value:"30"},
- {name:"attempts",value:"1"}
- ]
- }
- }}}}
-')"
-kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic \
- -p "$dns_patch" >/dev/null
-kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null ||
- die "HCC did not become Ready with the deterministic DNS proxy"
-wait_until 60 "HCC /ready endpoint with the released DNS proxy" probe_hcc_ready_pod ||
- die "HCC readiness was unavailable before the stale-policy fixture"
-
-CONTEXT_CREATED=1
-kctl apply -f - >/dev/null <<EOF
-apiVersion: clerum.io/v1alpha1
-kind: Context
-metadata:
- name: ${CONTEXT_NAME}
- namespace: ${MCP_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- contextId: ${CONTEXT_ID}
- description: E2E HCC readiness Context updated while initial MCP convergence is blocked.
- mcpServers: []
-EOF
-
-MCP_CREATED=1
-kctl apply -f - >/dev/null <<EOF
-apiVersion: clerum.io/v1alpha1
-kind: McpServer
-metadata:
- name: ${MCP_NAME}
- namespace: ${MCP_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_LABEL}"
-spec:
- contextRef: ${CONTEXT_NAME}
- description: E2E MCP fixture for initial fleet readiness.
- image: clerum/mock-mcp-server:test
- imagePullPolicy: IfNotPresent
- transport:
- type: streamableHttp
- url: http://${MCP_NAME}.${MCP_NS}.svc.cluster.local:3000/mcp
- port: 3000
- egressBindings:
- - dns: ${TARGET_DNS}
- port: 443
- protocol: ${INITIAL_EXTERNAL_EGRESS_PROTOCOL}
-EOF
-
-wait_until 180 "HCC to create the real initial TCP external-egress policy" \
- external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
- die "HCC never created the initial external-egress policy for the stale-policy fixture"
-ok "HCC created a real initial TCP external-egress policy"
-BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)" ||
- die "could not capture canonical baseline NetworkPolicy ownership and specs"
-
-# Same-intent safety phase: restart with the exact same desired binding. The
-# safety pass must RETAIN the identity-stable DNS-derived policy without resolving
-# DNS (B3 retain-identity-stable contract), certify global readiness with that
-# allow still present — no deny window — and leave the affected runtime blocked
-# for the additive lane. Retention-at-readiness is adjudicated promptly: a
-# held-refresh timeout later fail-closed deletes the retained allow, so the proof
-# must land before any refresh timeout.
-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
-wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||
- die "existing HCC pod did not stop"
-delete_fixture_runtime_leaving_dns_policy_for_safety ||
- die "could not remove the initial MCP runtime while retaining its external-egress policy"
-external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
- die "the initial TCP policy disappeared before the same-intent safety pass"
-SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)" ||
- die "could not capture the identity-stable external-egress policy snapshot"
-[ -n "$SAME_INTENT_EGRESS_SNAPSHOT" ] ||
- die "the identity-stable external-egress policy snapshot was empty before the same-intent restart"
-create_peer_fleet || die "could not create the gate-owned MCP/Context peer fleet"
-ok "created ${#MCP_FLEET_NAMES[@]} gate-owned MCP/Context fixtures before fleet bootstrap"
-
-hold_dns_blocker || die "could not hold the next fixture DNS query"
-DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||
- die "could not capture the DNS release baseline before the same-intent restart"
-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
-wait_until 120 "same-intent HCC pod to enter Running" capture_hcc ||
- die "same-intent HCC pod did not start"
-identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \
- -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"
-read -r HCC_UID HCC_RESTARTS <<<"$identity"
-[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||
- die "could not capture same-intent HCC pod identity"
-# Adjudicate retention-at-readiness FIRST — at the earliest possible moment —
-# because the certifying safety pass retains WITHOUT resolving DNS, so /ready=200
-# and the retained allow flip together. The separate additive refresh reaches the
-# DNS hold and, on its timeout, fail-closed deletes the retained allow; latching
-# the retained+Ready observation here beats that timeout.
-wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained" \
- hcc_ready_with_identity_stable_policy_retained ||
- die "HCC did not certify readiness with the same-intent identity-stable DNS policy retained (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
-# Retention-at-readiness is latched. Corroborate promptly — present, unchanged,
-# under a still-held DNS, with no restart and the baseline intact — before any
-# held-refresh timeout can fail-closed delete the retained allow.
-dns_has_not_released_since_hold ||
- die "the DNS blocker released before the same-intent retention assertion"
-identity_stable_policy_retained ||
- die "the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)"
-fixture_runtime_and_ready_absent ||
- die "the affected runtime or Ready status escaped the held same-intent DNS boundary"
-hcc_identity_is_stable ||
- die "HCC restarted while the same-intent identity-stable DNS policy was retained"
-baseline_policies_unchanged ||
- die "baseline safety NetworkPolicy ownership or specs changed during same-intent retention"
-wait_until 30 "Kubernetes to corroborate same-intent HCC readiness" \
- hcc_kubernetes_readiness_is_exact ||
- die "Kubernetes did not corroborate same-intent HCC readiness"
-wait_until 120 "the same-intent external-egress refresh to reach the DNS hold" \
- dns_hold_observed_since_arm ||
- die "the same-intent external-egress refresh never reached the DNS hold"
-ok "same-intent safety retained the identity-stable DNS policy while HCC stayed Ready and runtime stayed blocked"
-
-release_dns_blocker || die "could not release the same-intent DNS query"
-wait_until 30 "DNS blocker to answer the same-intent query" \
- dns_released_since_hold ||
- die "DNS blocker did not answer the same-intent query"
-wait_until 120 "current TCP external-egress policy after same-intent DNS release" \
- external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
- die "the current TCP external-egress policy did not converge after DNS release"
-wait_until 180 "same-intent runtime convergence after DNS release" \
- fixture_runtime_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
- die "same-intent runtime did not converge after DNS release"
-
-# Security phase: a protocol change invalidates the current TCP policy. The old
-# TCP allow must be revoked even when DNS prevents constructing the UDP policy.
-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
-wait_until 120 "same-intent HCC pod to stop" hcc_pods_absent ||
- die "same-intent HCC pod did not stop"
-delete_fixture_runtime_leaving_dns_policy_for_safety ||
- die "could not remove the same-intent runtime before the divergent revision"
-external_egress_policy_converged_with_protocol "$INITIAL_EXTERNAL_EGRESS_PROTOCOL" ||
- die "the current TCP policy disappeared before the divergent safety pass"
-hold_dns_blocker || die "could not hold the divergent fixture DNS query"
-DNS_RELEASE_COUNT_AT_HOLD="$(dns_release_count)" ||
- die "could not capture the DNS release baseline before the divergent revision"
-egress_patch="$(jq -cn --arg dns "$TARGET_DNS" \
- '{spec:{egressBindings:[{dns:$dns,port:443,protocol:"UDP"}]}}')"
-kctl patch mcpserver "$MCP_NAME" -n "$MCP_NS" --type=merge \
- -p "$egress_patch" >/dev/null
-wait_until 60 "McpServer API to expose the revised UDP external-egress binding" \
- mcp_egress_binding_is_udp ||
- die "McpServer external-egress revision did not persist"
-
-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
-wait_until 120 "replacement HCC pod to enter Running" capture_hcc ||
- die "replacement HCC pod did not start"
-identity="$(kctl get pod "$NEW_HCC_POD" -n "$HCC_NS" \
- -o jsonpath='{.metadata.uid}{" "}{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}')"
-read -r HCC_UID HCC_RESTARTS <<<"$identity"
-[ -n "$HCC_UID" ] && [ -n "$HCC_RESTARTS" ] ||
- die "could not capture replacement HCC pod identity"
-
-wait_until 120 "fixture external-egress DNS query to reach the hold proxy" \
- dns_hold_observed_since_arm ||
- die "NetworkPolicy safety convergence never reached the divergent DNS hold"
-wait_until_fast 120 "HCC readiness after revoking the divergent stale policy" \
- hcc_ready_after_stale_policy_revoked ||
- die "HCC did not become globally ready after revoking the divergent stale policy (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
-wait_until 30 "Kubernetes to corroborate HCC readiness during divergent DNS hold" \
- hcc_kubernetes_readiness_is_exact ||
- die "Kubernetes did not corroborate HCC readiness during divergent DNS hold"
-dns_log_contains "holding DNS A ${TARGET_DNS}" ||
- die "DNS hold ended before the fail-closed revocation assertion"
-if ! dns_has_not_released_since_hold; then
- die "DNS blocker released before the fail-closed revocation assertion"
-fi
-fixture_runtime_and_ready_absent ||
- die "fixture runtime or Ready status appeared while divergent DNS remained held"
-external_egress_policy_absent ||
- die "a divergent external-egress policy survived the fail-closed revocation"
-context_policies_absent ||
- die "Context policy existed before the Context watch update"
-baseline_policies_unchanged ||
- die "baseline safety NetworkPolicy ownership or specs changed during safety convergence"
-hcc_identity_is_stable ||
- die "HCC restarted while the MCP external-egress query was held"
-wait_until_fast 30 "authoritative safety pass metrics to certify the revocation" \
- safety_pass_metrics_certify ||
- die "safety_pass_* metrics did not corroborate the stale-policy revocation (families renamed or pass never measured?)"
-ok "/ready is 200 while the divergent stale policy and affected runtime remain absent"
-ok "safety_pass_* metrics corroborate an authoritative pass revoked the stale policy"
-ok "baseline NetworkPolicies remain present throughout fail-closed safety convergence"
-
-initial_empty_context_log="[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []"
-wait_until 120 "initial NetworkPolicy pass to reconcile the fixture's empty Context snapshot" \
- hcc_log_contains "$initial_empty_context_log" ||
- die "initial NetworkPolicy pass never reconciled the fixture's empty Context"
-wait_until 120 "a real divergent external-egress retry to be scheduled" \
- external_egress_retry_progress_is_observed ||
- die "the held divergent refresh did not schedule a real retry"
-DNS_HOLD_COUNT_AT_RETRY_SCHEDULE="$(dns_hold_count)" ||
- die "could not snapshot DNS hold evidence after the divergent retry was scheduled"
-wait_until 180 "the scheduled divergent external-egress retry to execute" \
- dns_retry_query_observed_since_schedule ||
- die "the scheduled divergent external-egress retry did not execute"
-context_policies_absent ||
- die "Context policy appeared before the isolated MODIFIED event"
-
-context_patch="$(jq -cn --arg server "$MCP_NAME" '{spec:{mcpServers:[$server]}}')"
-kctl patch context "$CONTEXT_NAME" -n "$MCP_NS" --type=merge \
- -p "$context_patch" >/dev/null
-wait_until 60 "Context API to expose the latest server membership" context_watch_has_latest_spec ||
- die "Context update did not persist"
-wait_until 60 "HCC Context watch to observe the update during the blocked fleet pass" \
- hcc_log_contains "Context watch event: MODIFIED for ${CONTEXT_NAME}" ||
- die "HCC did not observe the real Context update while initial MCP convergence was blocked"
-wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold" \
- context_policies_converged ||
- die "Context policies remained blocked behind the held McpServer convergence"
-fixture_runtime_and_ready_absent ||
- die "McpServer runtime or Ready status escaped the deterministic DNS hold boundary"
-external_egress_policy_absent ||
- die "a divergent external-egress policy reappeared while DNS remained held"
-dns_log_contains "holding DNS A ${TARGET_DNS}" ||
- die "DNS hold evidence disappeared before Context policy convergence"
-if ! dns_has_not_released_since_hold; then
- die "DNS blocker released the McpServer query before the Context isolation assertion"
-fi
-probe_new_hcc_ready_endpoint ||
- die "HCC lost global readiness while the affected runtime remained safely blocked (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
-hcc_kubernetes_readiness_is_exact ||
- die "Kubernetes lost HCC readiness during isolated Context convergence"
-hcc_identity_is_stable ||
- die "HCC restarted while Context policies converged around the held MCP entity"
-ok "the MODIFIED handler, not the initial empty-Context pass, converged all three policies"
-ok "global readiness and Context progress continued while the affected runtime stayed blocked"
-
-release_dns_blocker ||
- die "could not release the fixture DNS query"
-wait_until 30 "DNS blocker to answer the held query" \
- dns_released_since_hold ||
- die "DNS blocker did not answer the held query"
-wait_until 120 "current UDP external-egress policy after DNS release" \
- external_egress_policy_converged_with_protocol UDP ||
- die "the current UDP external-egress policy did not converge after DNS release"
-probe_new_hcc_ready_endpoint ||
- die "HCC lost readiness while the released UDP policy converged (${HCC_READY_PROBE_DIAGNOSTIC:-HCC_READY_PROBE category=unknown})"
-wait_until 300 "McpServer runtime, status, and latest Context NetworkPolicies to converge" \
- fixture_converged ||
- die "McpServer/Context/NetworkPolicy convergence did not complete after DNS release"
-wait_until 300 "all peer MCP/Context runtimes to converge after releasing the held primary egress" \
- peer_fleet_converged ||
- die "peer MCP/Context fleet did not converge after the held primary egress was released"
-hcc_identity_is_stable ||
- die "HCC restarted instead of completing the released background convergence"
-final_probe="$(probe_hcc_final_context)" ||
- die "HCC final Context-scoped discovery did not expose the converged fixture"
-echo "$final_probe" | jq -e \
- --arg fixture "$MCP_NAME" --arg context "$CONTEXT_NAME" '
- .readyStatus == 200 and .scopedDiscoveryStatus == 200 and
- .fixture == $fixture and .fixtureReady == true and .contextRef == $context
- ' >/dev/null ||
- die "HCC final probe returned an invalid result: ${final_probe}"
-baseline_policies_unchanged ||
- die "baseline safety NetworkPolicy ownership or specs changed before final convergence"
-ok "DNS release created the current UDP policy and affected runtime without gating HCC readiness"
-ok "the full worker-width-crossing peer fleet converged after releasing the held primary egress"
-ok "Context ingress, mcp-host egress, and rpc-proxy egress policies reflect the latest watch state"
-ok "HCC stayed Ready through divergent revocation while the affected runtime remained fail-closed"
-
-header "HCC McpServer/Context assertions complete; restoring branch-owned runtime"
diff --git a/scripts/e2e/e2e-hcc-readiness-bootstrap.sh b/scripts/e2e/e2e-hcc-readiness-bootstrap.sh
deleted file mode 100755
index eb485a590..000000000
--- a/scripts/e2e/e2e-hcc-readiness-bootstrap.sh
+++ /dev/null
@@ -1,785 +0,0 @@
-#!/usr/bin/env bash
-# Proves that HCC exposes its real readiness and discovery endpoints while the
-# initial Host fleet reconciliation is still demonstrably in progress.
-#
-# The gate temporarily routes HCC's control-api token issuance to an isolated
-# in-cluster HTTP blocker, creates a gate-owned fleet while HCC is stopped, and
-# restarts HCC. There are more fixture Hosts than two complete bounded worker
-# waves, so the initial pass remains deterministically active even though each
-# individual held token request has its own client timeout. No real Host or
-# runtime-token Secret is modified.
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=scripts/e2e/e2e-lib.sh
-source "${SCRIPT_DIR}/e2e-lib.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
-
-[ -n "$E2E_KUBECONTEXT" ] || {
- echo "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." >&2
- exit 1
-}
-is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
- echo "Refusing HCC readiness fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
- exit 1
-}
-require_safe_kube_context
-[ "${E2E_HCC_READINESS_FAULT_INJECTION:-0}" = 1 ] || {
- echo "Set E2E_HCC_READINESS_FAULT_INJECTION=1 to acknowledge temporary HCC fault injection." >&2
- exit 1
-}
-command -v jq >/dev/null 2>&1 || {
- echo "jq is required" >&2
- exit 1
-}
-
-HCC_NS="${HCC_NS:-control-plane}"
-HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
-HCC_GATEWAY_DEPLOY="${HCC_GATEWAY_DEPLOY:-host-context-controller-api-gateway}"
-HOST_NS="${MCP_HOST_NS:-mcp-host}"
-MCP_NS="${MCP_SERVER_NS:-mcp-server}"
-TOKEN_REQUEST_PATH="/api/v1/auth/mcp-host/${HOST_NS}/standalone/tokens"
-TOKEN_REQUEST_LOG_PREFIX="holding-token POST ${TOKEN_REQUEST_PATH} host="
-RUN_ID="$(date +%s)-$$"
-BLOCKER_NAME="$(truncate_rfc1123 "e2e-hcc-readiness-blocker-${RUN_ID}")"
-BLOCKER_EGRESS_POLICY_NAME="$(truncate_rfc1123 "${BLOCKER_NAME}-egress")"
-SUITE_NAME="hcc-readiness-bootstrap"
-FIXTURE_HOST_PREFIX="$(truncate_rfc1123 "e2e-hcc-ready-${RUN_ID}")"
-FIXTURE_CONTEXT="$(truncate_rfc1123 "e2e-hcc-ready-context-${RUN_ID}")"
-FIXTURE_SECRET="$(truncate_rfc1123 "e2e-hcc-ready-llm-${RUN_ID}")"
-START_MARKER='Starting initial Host background convergence'
-COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'
-FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'
-ORIGINAL_CONTROL_API_BASE_URL=""
-ORIGINAL_REPLICAS=""
-HCC_PORT=""
-HOST_RECONCILE_CONCURRENCY=""
-FIXTURE_HOST_COUNT=0
-FIXTURE_HOST_NAMES=()
-HCC_MUTATED=0
-BLOCKER_CREATED=0
-FIXTURES_CREATED=0
-# Read and mutated by the sourced compare-and-swap lock helper.
-# shellcheck disable=SC2034
-HCC_GATE_LOCK_ACQUIRED=0
-# shellcheck disable=SC2034
-HCC_GATE_LOCK_NAME=""
-# shellcheck disable=SC2034
-HCC_GATE_LOCK_UID=""
-# shellcheck disable=SC2034
-HCC_GATE_FINALIZATION_FAILURE=""
-
-die() {
- if [ -n "${new_hcc_pod:-}" ]; then
- echo "Recent HCC logs from ${new_hcc_pod}:" >&2
- kctl logs "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller --tail=300 >&2 || true
- fi
- fail "$*"
- exit 1
-}
-
-wait_until() {
- local timeout=$1 description=$2
- shift 2
- local deadline now
- deadline=$(( $(date +%s) + timeout ))
- while :; do
- "$@" && return 0
- now="$(date +%s)"
- [ "$now" -lt "$deadline" ] || break
- sleep 1
- done
- echo "Timed out after ${timeout}s waiting for ${description}" >&2
- return 1
-}
-
-running_hcc_pod() {
- local rows
- rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
- --field-selector=status.phase=Running \
- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
- 2>/dev/null)" || return 1
- awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"
-}
-
-hcc_pods_absent() {
- local pods
- pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" ||
- return 1
- [ -z "$pods" ]
-}
-
-hcc_logs() {
- local pod=$1
- kctl logs "pod/${pod}" -n "$HCC_NS" -c host-context-controller 2>/dev/null
-}
-
-hcc_log_contains() {
- local pod=$1 marker=$2 logs
- logs="$(hcc_logs "$pod")" || return 1
- hcc_log_snapshot_contains "$logs" "$marker"
-}
-
-blocker_holds_token_request() {
- local logs
- logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1
- hcc_log_snapshot_contains "$logs" \
- "${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-"
-}
-
-blocker_fixture_request_count() {
- local logs
- logs="$(kctl logs "deployment/${BLOCKER_NAME}" -n "$HCC_NS" 2>/dev/null)" || return 1
- awk -v prefix="${TOKEN_REQUEST_LOG_PREFIX}${FIXTURE_HOST_PREFIX}-" '
- index($0, prefix) {
- host = substr($0, index($0, "host=") + length("host="))
- sub(/[[:space:]].*$/, "", host)
- if (!(host in seen)) {
- seen[host] = 1
- count += 1
- }
- }
- END { print count + 0 }
- ' <<<"$logs"
-}
-
-blocker_observed_more_than_one_worker_wave() {
- local count
- count="$(blocker_fixture_request_count)" || return 1
- [ "$count" -gt "$HOST_RECONCILE_CONCURRENCY" ]
-}
-
-initial_host_pass_is_active() {
- local pod=$1 logs
- logs="$(hcc_logs "$pod")" || return 1
- hcc_initial_pass_snapshot_is_active \
- "$logs" "$START_MARKER" "$COMPLETE_MARKER" "$FAIL_MARKER"
-}
-
-control_api_service_excludes_blocker() {
- kctl get endpointslice -n "$HCC_NS" -l 'kubernetes.io/service-name=control-api' -o json |
- jq -e --arg blocker "$BLOCKER_NAME" \
- 'all(.items[]?.endpoints[]?; .targetRef.name != $blocker)' >/dev/null
-}
-
-readiness_blocker_suite_absent() {
- local resources
- resources="$(
- kctl get deployment,service,networkpolicy -n "$HCC_NS" \
- -l 'e2e.clerum.io/hcc-readiness' -o name 2>/dev/null
- )" || return 1
- [ -z "$resources" ]
-}
-
-probe_hcc_ready_pod() {
- local pod
- pod="$(ready_pod_name "$HCC_NS" "app=${HCC_DEPLOY}")" || return 1
- kctl exec "pod/${pod}" -n "$HCC_NS" -c host-context-controller -- \
- env "HCC_E2E_PORT=${HCC_PORT}" \
- node -e '
- const http = require("node:http");
- const request = http.get(
- {host: "127.0.0.1", port: Number(process.env.HCC_E2E_PORT), path: "/ready"},
- response => {
- let body = "";
- response.setEncoding("utf8");
- response.on("data", chunk => { body += chunk; });
- response.on("end", () => {
- let parsed;
- try { parsed = JSON.parse(body); } catch { process.exit(2); }
- process.exit(response.statusCode === 200 && parsed?.ready === true ? 0 : 3);
- });
- }
- );
- request.setTimeout(5000, () => request.destroy(new Error("timeout")));
- request.once("error", () => process.exit(4));
- ' >/dev/null 2>&1
-}
-
-running_hcc_gateway_pod() {
- ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"
-}
-
-hcc_gateway_deployment_ready() {
- kctl get deployment "$HCC_GATEWAY_DEPLOY" -n "$HCC_NS" -o json |
- jq -e '
- .spec.replicas == 1 and
- .status.observedGeneration == .metadata.generation and
- (.status.updatedReplicas // 0) == 1 and
- (.status.readyReplicas // 0) == 1 and
- (.status.availableReplicas // 0) == 1 and
- (.status.unavailableReplicas // 0) == 0
- ' >/dev/null
-}
-
-hcc_gateway_local_health_ok() {
- local pod
- pod="$(running_hcc_gateway_pod)" || return 1
- kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
- wget -qO- http://127.0.0.1:8081/health |
- jq -e '.status == "ok"' >/dev/null
-}
-
-hcc_gateway_ready_proxy_unavailable() {
- local pod output status
- pod="$(running_hcc_gateway_pod)" || return 1
- output="$(
- kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
- wget -T 70 -t 1 -O /dev/null -S http://127.0.0.1:8081/ready 2>&1
- )" && return 1
- status="$(
- awk '
- /^[[:space:]]*HTTP\/[0-9.]+[[:space:]]+[0-9][0-9][0-9]([[:space:]]|$)/ {
- code = $2
- }
- END {
- if (code != "") print code
- }
- ' <<<"$output"
- )"
- case "$status" in
- 502|503|504) return 0 ;;
- *) return 1 ;;
- esac
-}
-
-hcc_gateway_remains_ready_without_hcc() {
- local duration=$1 deadline
- deadline=$(( $(date +%s) + duration ))
- while [ "$(date +%s)" -lt "$deadline" ]; do
- hcc_gateway_deployment_ready &&
- hcc_gateway_local_health_ok &&
- hcc_gateway_ready_proxy_unavailable &&
- hcc_gateway_deployment_ready &&
- hcc_gateway_local_health_ok ||
- return 1
- sleep 1
- done
-}
-
-hcc_gateway_ready_proxy_recovers() {
- local pod
- pod="$(running_hcc_gateway_pod)" || return 1
- kctl exec "pod/${pod}" -n "$HCC_NS" -c nginx -- \
- wget -qO- http://127.0.0.1:8081/ready |
- jq -e '.ready == true and .status == "ready"' >/dev/null
-}
-
-fixture_inputs_absent() {
- local hosts contexts secrets
- hosts="$(kctl get host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- -o name 2>/dev/null)" || return 1
- contexts="$(kctl get context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- -o name 2>/dev/null)" || return 1
- secrets="$(kctl get secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- -o name 2>/dev/null)" || return 1
- [ -z "${hosts}${contexts}${secrets}" ]
-}
-
-fixture_runtime_absent() {
- local host resources
- [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0
- for host in "${FIXTURE_HOST_NAMES[@]}"; do
- resources="$(
- kctl get deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \
- -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \
- -o name 2>/dev/null
- )" || return 1
- [ -z "$resources" ] || return 1
- done
-}
-
-fixture_resources_absent() {
- fixture_inputs_absent && fixture_runtime_absent
-}
-
-hcc_restore_is_verified() {
- local deployment
- deployment="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" || return 1
- jq -e \
- --arg controlApiBaseUrl "$ORIGINAL_CONTROL_API_BASE_URL" \
- --argjson replicas "${ORIGINAL_REPLICAS:-1}" '
- .spec.replicas == $replicas and
- .status.observedGeneration == .metadata.generation and
- (.status.updatedReplicas // 0) == $replicas and
- (.status.readyReplicas // 0) == $replicas and
- (.status.availableReplicas // 0) == $replicas and
- (.status.unavailableReplicas // 0) == 0 and
- any(.spec.template.spec.containers[]?;
- .name == "host-context-controller" and
- any(.env[]?;
- .name == "CONTROL_API_BASE_URL" and .value == $controlApiBaseUrl
- )
- )
- ' <<<"$deployment" >/dev/null || return 1
- probe_hcc_ready_pod && fixture_resources_absent
-}
-
-delete_blocker_fixture() {
- local failed=0
- kctl delete networkpolicy "$BLOCKER_EGRESS_POLICY_NAME" "$BLOCKER_NAME" -n "$HCC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete service "$BLOCKER_NAME" -n "$HCC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete deployment "$BLOCKER_NAME" -n "$HCC_NS" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- [ "$failed" = 0 ]
-}
-
-create_host_fixtures() {
- local index host
- FIXTURES_CREATED=1
-
- kctl apply -f - >/dev/null <<EOF || return 1
-apiVersion: v1
-kind: Secret
-metadata:
- name: ${FIXTURE_SECRET}
- namespace: ${HOST_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_ID}"
-type: Opaque
-stringData:
- # Test-only placeholder: Host secret validation checks existence, while token
- # issuance is intentionally held before any mcp-host Deployment is created.
- OPENAI_API_KEY: "e2e-readiness-not-a-real-key"
----
-apiVersion: clerum.io/v1alpha1
-kind: Context
-metadata:
- name: ${FIXTURE_CONTEXT}
- namespace: ${MCP_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_ID}"
-spec:
- contextId: ${FIXTURE_CONTEXT}
- description: Gate-owned Context for HCC Host bootstrap readiness.
- mcpServers: []
-EOF
-
- for ((index = 1; index <= FIXTURE_HOST_COUNT; index++)); do
- host="$(printf '%s-%02d' "$FIXTURE_HOST_PREFIX" "$index")"
- FIXTURE_HOST_NAMES+=("$host")
- kctl apply -f - >/dev/null <<EOF || return 1
-apiVersion: clerum.io/v1alpha1
-kind: Host
-metadata:
- name: ${host}
- namespace: ${HOST_NS}
- labels:
- e2e.clerum.io/suite: ${SUITE_NAME}
- e2e.clerum.io/run: "${RUN_ID}"
-spec:
- host: ${host}
- contextRef: ${FIXTURE_CONTEXT}
- secretRef: ${FIXTURE_SECRET}
- model:
- provider: openai
- name: e2e-readiness
-EOF
- done
-}
-
-delete_host_fixtures() {
- local failed=0 host
- kctl delete host -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
- kctl delete context -n "$MCP_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- kctl delete secret -n "$HOST_NS" -l "e2e.clerum.io/suite=${SUITE_NAME}" \
- --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
- [ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"
- for host in "${FIXTURE_HOST_NAMES[@]}"; do
- kctl delete \
- deployment,service,serviceaccount,role,rolebinding,secret,persistentvolumeclaim,networkpolicy \
- -A -l "clerum.io/managed-by=host-context-controller,clerum.io/host=${host}" \
- --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
- done
- [ "$failed" = 0 ] || return 1
- wait_until 60 "all gate-owned Host inputs and managed runtime resources to disappear" \
- fixture_resources_absent
-}
-
-print_repair_instructions() {
- cat >&2 <<EOF
-HCC readiness bootstrap cleanup could not restore a verified clean state.
-Context: ${E2E_KUBECONTEXT}
-HCC: ${HCC_NS}/${HCC_DEPLOY}
-Fixture Hosts: ${HOST_NS}/${FIXTURE_HOST_PREFIX}-*
-Fixture Context: ${MCP_NS}/${FIXTURE_CONTEXT}
-Fixture LLM Secret: ${HOST_NS}/${FIXTURE_SECRET}
-Retained blocker: ${HCC_NS}/${BLOCKER_NAME}
-
-Inspect before changing anything:
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
- kubectl --context=${E2E_KUBECONTEXT} -n ${HOST_NS} get host,secret -l e2e.clerum.io/suite=${SUITE_NAME}
- kubectl --context=${E2E_KUBECONTEXT} -n ${MCP_NS} get context -l e2e.clerum.io/suite=${SUITE_NAME}
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/hcc-readiness
-
-Do not remove the blocker or HCC gate lock until HCC configuration, rollout, Pod readiness,
-/ready, and fixture absence are verified.
-
-Restore the original HCC replica count before verifying rollout and /ready:
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
- kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s
-EOF
-}
-
-cleanup() {
- local status=$? cleanup_failed=0 restore_ok=1 hcc_stopped=1
- trap - EXIT
- set +e
-
- # Remove every gate-owned input while HCC is stopped. Restarting HCC with a
- # partially deleted fixture fleet would let the real control-api path mutate
- # those leftovers and make cleanup non-deterministic.
- if [ "$HCC_MUTATED" = 1 ]; then
- hcc_stopped=0
- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 \
- >/dev/null 2>&1 || restore_ok=0
- if [ "$restore_ok" = 1 ]; then
- wait_until 120 "HCC pods to stop before fixture deletion" \
- hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
- fi
- [ "$restore_ok" = 1 ] && hcc_stopped=1
- fi
-
- if [ "$FIXTURES_CREATED" = 1 ]; then
- if [ "$hcc_stopped" = 1 ]; then
- delete_host_fixtures >/dev/null 2>&1 || restore_ok=0
- else
- restore_ok=0
- fi
- fi
-
- if [ "$HCC_MUTATED" = 1 ] && [ "$restore_ok" = 1 ] &&
- [ -n "$ORIGINAL_CONTROL_API_BASE_URL" ]; then
- kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
- "CONTROL_API_BASE_URL=${ORIGINAL_CONTROL_API_BASE_URL}" >/dev/null 2>&1 ||
- restore_ok=0
- kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 ||
- restore_ok=0
- kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s \
- >/dev/null 2>&1 || restore_ok=0
- if [ "$restore_ok" = 1 ]; then
- wait_until 240 \
- "restored HCC config, rollout, Ready pod, /ready, and fixture absence" \
- hcc_restore_is_verified >/dev/null 2>&1 || restore_ok=0
- fi
- fi
-
- if [ "$restore_ok" = 1 ]; then
- if [ "$BLOCKER_CREATED" = 1 ]; then
- delete_blocker_fixture || cleanup_failed=1
- fi
- wait_until 60 "all HCC readiness blocker resources to disappear" \
- readiness_blocker_suite_absent >/dev/null 2>&1 || cleanup_failed=1
- else
- cleanup_failed=1
- fi
-
- if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"; then
- cleanup_failed=1
- fi
-
- if [ "$cleanup_failed" -ne 0 ]; then
- echo "HCC readiness gate cleanup failed on context ${E2E_KUBECONTEXT}." >&2
- [ "$restore_ok" = 1 ] || print_repair_instructions
- [ "$status" = 0 ] && status=1
- fi
- if [ "$status" = 0 ] && [ "$cleanup_failed" = 0 ] && [ "$restore_ok" = 1 ]; then
- header "HCC readiness bootstrap gate passed"
- fi
- exit "$status"
-}
-trap cleanup EXIT
-
-header "HCC readiness during initial Host fleet reconciliation"
-
-require_branch_owned_hcc_gate "$HCC_NS"
-ok "branch helper, profile, exact HEAD/fingerprint/gate verified"
-
-kctl get nodes -o json |
- jq -e --arg context "$E2E_KUBECONTEXT" \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
- die "target context is not a minikube cluster"
-readiness_blocker_suite_absent ||
- die "stale HCC readiness blocker resources exist; inspect them before fault injection"
-fixture_inputs_absent ||
- die "stale ${SUITE_NAME} Host, Context, or Secret fixtures exist; inspect them before fault injection"
-acquire_hcc_watch_gate_lock ||
- die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
-ok "exclusive HCC fault-injection lock acquired"
-
-HCC_DEPLOYMENT_JSON="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o json)" ||
- die "could not read HCC Deployment"
-ORIGINAL_REPLICAS="$(jq -r '.spec.replicas // empty' <<<"$HCC_DEPLOYMENT_JSON")"
-[ "$ORIGINAL_REPLICAS" = 1 ] ||
- die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
-ORIGINAL_CONTROL_API_BASE_URL="$(jq -r '
- first(
- .spec.template.spec.containers[]? |
- select(.name == "host-context-controller") |
- .env[]? |
- select(.name == "CONTROL_API_BASE_URL") |
- .value
- ) // empty
-' <<<"$HCC_DEPLOYMENT_JSON")"
-[ -n "$ORIGINAL_CONTROL_API_BASE_URL" ] ||
- die "HCC CONTROL_API_BASE_URL is missing"
-HCC_PORT="$(jq -r '
- .spec.template.spec.containers[]? |
- select(.name == "host-context-controller") as $container |
- (
- [$container.env[]? | select(.name == "CONTEXT_MAPPER_PORT") | .value][0] //
- [$container.ports[]? | select(.name == "http") | .containerPort][0] //
- empty
- )
-' <<<"$HCC_DEPLOYMENT_JSON")"
-[[ "$HCC_PORT" =~ ^[0-9]+$ ]] && [ "$HCC_PORT" -ge 1 ] && [ "$HCC_PORT" -le 65535 ] ||
- die "could not resolve a valid HCC HTTP port from the Deployment"
-HOST_RECONCILE_CONCURRENCY="$(jq -r '
- first(
- .spec.template.spec.containers[]? |
- select(.name == "host-context-controller") |
- .env[]? |
- select(.name == "HCC_HOST_FULL_RECONCILE_CONCURRENCY") |
- .value
- ) // "2"
-' <<<"$HCC_DEPLOYMENT_JSON")"
-[[ "$HOST_RECONCILE_CONCURRENCY" =~ ^[1-8]$ ]] ||
- die "HCC_HOST_FULL_RECONCILE_CONCURRENCY must be 1..8, found ${HOST_RECONCILE_CONCURRENCY}"
-FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))
-HCC_IMAGE="$(jq -r '
- first(
- .spec.template.spec.containers[]? |
- select(.name == "host-context-controller") |
- .image
- ) // empty
-' <<<"$HCC_DEPLOYMENT_JSON")"
-[ -n "$HCC_IMAGE" ] || die "could not resolve the HCC image"
-ok "planned ${FIXTURE_HOST_COUNT} gate-owned Hosts for concurrency ${HOST_RECONCILE_CONCURRENCY}"
-
-BLOCKER_CREATED=1
-kctl apply -f - >/dev/null <<EOF
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: ${BLOCKER_NAME}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
-spec:
- replicas: 1
- selector:
- matchLabels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
- template:
- metadata:
- labels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
- spec:
- automountServiceAccountToken: false
- terminationGracePeriodSeconds: 1
- containers:
- - name: blocker
- image: ${HCC_IMAGE}
- imagePullPolicy: IfNotPresent
- env:
- - name: EXPECTED_TOKEN_PATH
- value: ${TOKEN_REQUEST_PATH}
- command: [node, -e]
- args:
- - >-
- require('http').createServer((request,response) => {
- if(request.method!=='POST'||request.url!==process.env.EXPECTED_TOKEN_PATH){
- console.log('rejecting '+request.method+' '+request.url);
- request.resume();
- request.on('end',()=>{response.statusCode=404;response.end()});
- return;
- }
- let body='';
- request.setEncoding('utf8');
- request.on('data',chunk=>{if(body.length<65536)body+=chunk;});
- request.on('end',()=>{
- let host='unknown';
- try{const parsed=JSON.parse(body);if(typeof parsed.host==='string')
- host=parsed.host;}catch{}
- console.log('holding-token '+request.method+' '+request.url+' host='+host);
- });
- }).listen(8090, '0.0.0.0');
- readinessProbe:
- tcpSocket: {port: 8090}
- periodSeconds: 1
- resources:
- requests: {cpu: 5m, memory: 16Mi}
- limits: {cpu: 50m, memory: 48Mi}
- securityContext:
- allowPrivilegeEscalation: false
- capabilities: {drop: [ALL]}
- readOnlyRootFilesystem: true
- runAsNonRoot: true
- runAsUser: 1000
- runAsGroup: 1000
- seccompProfile: {type: RuntimeDefault}
----
-apiVersion: v1
-kind: Service
-metadata:
- name: ${BLOCKER_NAME}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
-spec:
- selector:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
- ports:
- - name: http
- port: 8090
- targetPort: 8090
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: ${BLOCKER_EGRESS_POLICY_NAME}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
-spec:
- podSelector:
- matchLabels:
- app: ${HCC_DEPLOY}
- policyTypes: [Egress]
- egress:
- - to:
- - podSelector:
- matchLabels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
- ports:
- - protocol: TCP
- port: 8090
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: ${BLOCKER_NAME}
- namespace: ${HCC_NS}
- labels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
-spec:
- podSelector:
- matchLabels:
- e2e.clerum.io/hcc-readiness: "${RUN_ID}"
- policyTypes: [Ingress]
- ingress:
- - from:
- - podSelector:
- matchLabels:
- app: ${HCC_DEPLOY}
- ports:
- - protocol: TCP
- port: 8090
-EOF
-kctl rollout status deployment "$BLOCKER_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||
- die "isolated token-issuance blocker did not become ready"
-control_api_service_excludes_blocker ||
- die "isolated blocker was selected by the real control-api Service"
-ok "isolated in-cluster blocker is ready"
-
-HCC_MUTATED=1
-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
-wait_until 120 "existing HCC pods to stop" hcc_pods_absent ||
- die "existing HCC pod did not stop"
-hcc_gateway_remains_ready_without_hcc 20 ||
- die "HCC API gateway did not remain Ready on local /health while proxied /ready was unavailable"
-ok "API gateway remained Ready for a complete probe-failure window while /ready reflected HCC unavailability"
-create_host_fixtures ||
- die "could not create the gate-owned Host fleet while HCC was stopped"
-ok "created ${FIXTURE_HOST_COUNT} gate-owned Hosts without modifying any real Host or Secret"
-kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
- "CONTROL_API_BASE_URL=http://${BLOCKER_NAME}.${HCC_NS}.svc.cluster.local:8090" >/dev/null
-kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
-
-new_hcc_pod=""
-capture_running_hcc() {
- new_hcc_pod="$(running_hcc_pod)"
- [ -n "$new_hcc_pod" ]
-}
-wait_until 120 "replacement HCC pod to enter Running" capture_running_hcc ||
- die "replacement HCC pod did not start"
-wait_until 60 "initial Host reconciliation to start" \
- hcc_log_contains "$new_hcc_pod" "$START_MARKER" ||
- die "HCC never started the initial Host reconciliation"
-wait_until 120 "a gate-owned Host token request to reach the blocker" blocker_holds_token_request ||
- die "initial Host reconciliation never reached the deterministic blocker"
-wait_until 180 "more than one bounded Host worker wave to reach the blocker" \
- blocker_observed_more_than_one_worker_wave ||
- die "fixture fleet did not prove progress beyond the first bounded Host worker wave"
-initial_host_pass_is_active "$new_hcc_pod" ||
- die "initial Host reconciliation failed or completed before the readiness assertion"
-ok "initial Host reconciliation is active beyond one bounded worker wave"
-
-probe_script="$(cat <<'NODE'
-const http = require('http');
-function get(path) {
- return new Promise((resolve, reject) => {
- const request = http.get(
- {host: '127.0.0.1', port: Number(process.env.HCC_E2E_PORT), path},
- response => {
- let body = '';
- response.setEncoding('utf8');
- response.on('data', chunk => { body += chunk; });
- response.on('end', () => resolve({status: response.statusCode, body}));
- }
- );
- request.setTimeout(5000, () => request.destroy(new Error('timeout')));
- request.once('error', reject);
- });
-}
-(async () => {
- const ready = await get('/ready');
- const readyBody = JSON.parse(ready.body);
- if (ready.status !== 200 || readyBody.status !== 'ready' || readyBody.ready !== true) {
- throw new Error('unexpected readiness response: ' + ready.status + ' ' + ready.body);
- }
- const discovery = await get('/api/v1/mcpservers');
- if (discovery.status !== 200) {
- throw new Error('unexpected discovery response: ' + discovery.status);
- }
- const discoveryBody = JSON.parse(discovery.body);
- if (!Array.isArray(discoveryBody.servers) || discoveryBody.contextRef !== '*') {
- throw new Error('discovery response does not match the live API contract');
- }
- console.log(JSON.stringify({readyStatus: ready.status, discoveryStatus: discovery.status,
- discoveredServers: discoveryBody.servers.length}));
-})().catch(error => { console.error(error.message); process.exit(1); });
-NODE
-)"
-probe_result="$(kctl exec "pod/${new_hcc_pod}" -n "$HCC_NS" -c host-context-controller -- \
- env "HCC_E2E_PORT=${HCC_PORT}" node -e "$probe_script")" ||
- die "real HCC readiness/discovery HTTP probe failed while the fleet pass was active"
-echo "$probe_result" | jq -e \
- '.readyStatus == 200 and .discoveryStatus == 200 and (.discoveredServers | type == "number")' \
- >/dev/null ||
- die "HCC HTTP probe returned an invalid result: ${probe_result}"
-hcc_gateway_ready_proxy_recovers ||
- die "HCC API gateway /ready did not recover after HCC became ready"
-initial_host_pass_is_active "$new_hcc_pod" ||
- die "initial Host reconciliation failed or completed before the HTTP assertions finished"
-ok "/ready returned 200 while the initial Host fleet pass remained active"
-ok "API gateway /ready recovered without changing its local readiness"
-ok "/api/v1/mcpservers returned a valid live discovery response during that same pass"
-
-header "HCC readiness assertions passed; restoring branch-owned runtime"
-echo "$probe_result"
diff --git a/scripts/e2e/e2e-host-storm-gate.sh b/scripts/e2e/e2e-host-storm-gate.sh
index 1015dd276..095bb8c93 100755
--- a/scripts/e2e/e2e-host-storm-gate.sh
+++ b/scripts/e2e/e2e-host-storm-gate.sh
@@ -26,14 +26,16 @@
# construction, so A1 is primarily a RECORDING of that evidence — it
# can only fail if the harness itself degenerates into sequential
# phases; the load-bearing falsifiable claims are A2-A8.
-# A2 urgent queue-wait coverage has two independent same-pod windows:
-# (a) the supported wake window immediately before the HCC kill requires
-# every completed urgent sample <= 5s, after proving the wake reconcile
-# finished and the urgent lane returned idle; and
-# (b) ten dedicated Host ADDED events issued only after the replacement
-# HCC completed LIST→WATCH recovery use a D - C1 p95 delta. A slow wake
-# can therefore never be diluted by later fast post-recovery probes, and
-# neither window crosses the intentional process restart.
+# A2 urgent queue-wait p95 <= 5s DURING the storm window, computed from
+# the clerum_hcc_host_reconcile_queue_wait_seconds{lane="urgent"}
+# histogram buckets (§14.1) across two counter windows: pre-kill pod
+# delta (scrape B - scrape A) plus post-kill pod absolute (scrape C1,
+# taken IMMEDIATELY after created-Host convergence so post-storm quiet
+# samples cannot dilute the p95 — a fresh process starts at zero, and
+# the C1 window includes replacement-pod recovery traffic). Samples in
+# flight between scrape B and process death are unrecoverable (counter
+# reset) and are recorded as a documented measurement gap, never
+# silently backfilled.
# A3 no starvation: EVERY created Host's Deployment materializes within
# STORM_MATERIALIZATION_BUDGET_S (20s, §14.2 "New Host
# materialization") of that Host's OWN metadata.creationTimestamp,
@@ -57,16 +59,13 @@
# All-zero is legitimate (deletes fully handled by the direct
# watch-DELETE path, which by design does not touch this counter) —
# A5 remains the falsifiable cleanup proof.
-# A7 recovered fleet convergence is bounded, from
-# clerum_hcc_host_fleet_requests_total (scrape C2) plus the replacement
-# pod's explicit initial-fleet COMPLETED log: failed == 0 in both windows,
-# the exact replacement pod completes its startup fleet pass, and
-# trailing <= coalesced. The pod identity, completion marker, and A4's
-# successful recovery histogram form the causal fence; started is not
-# capped because an independent request may legitimately begin after the
-# recovery pass settles. Requiring a coalesced request here would invent
-# a second overlapping signal; the concurrent-request coalescing contract
-# is covered deterministically by hostFleetScheduler.test.ts.
+# A7 fleet coalescing held (one active + one trailing slot), from
+# clerum_hcc_host_fleet_requests_total (scrape C2): failed == 0 in
+# both windows; on the post-kill pod started >= 1 (the recovery
+# convergence pass ran), coalesced >= 1 (the recovery-triggered
+# request landed against the active initial pass), and trailing <=
+# coalesced (every trailing drain corresponds to at least one request
+# that was queued behind the single pending slot).
# A8 wake ingredient integrity: the woken Host leaves state=suspended and
# reaches a Ready pod within STORM_WAKE_READY_BUDGET_S of the wake
# POST. This is an ingredient-integrity bound (a silently no-op wake
@@ -98,26 +97,23 @@
#
# ----------------------------------------------------------------------
# INPUTS (environment; fail-loud if missing/invalid)
-# KUBECONTEXT / E2E_K8S_CONTEXT
-# Exact generated branch-owned minikube context.
-# MINIKUBE_PROFILE
-# Must equal that context.
-# E2E_EXPECTED_PRE_GATE_GATE
-# Exact gate recorded by the branch-owned pre-gate sync.
-# E2E_HCC_HOST_STORM_FAULT_INJECTION=1
-# Explicit acknowledgement of the disruptive HCC pod restart.
+# CONTEXT kubectl context. REFUSED unless it matches
+# ^clerum-(test|codex-|detached-|claude-) — a local minikube
+# profile. Any GKE / *prod* context is rejected outright.
#
# OPTIONAL INPUTS
# E2E_STORM_WAKE_HOST_REF suspended-wake target (default chatllm-stateless,
# the seeded stateless Host — it must exist with
# spec.lifecycle.stateless=true and be associated to the E2E
# user; seed via scripts/e2e/seed-stateless-host.sh).
+# EXTERNAL_REST_API_BASE_URL default http://127.0.0.1:8091
+# RPC_PROXY_BASE_URL default http://127.0.0.1:8094
# E2E_DEV_LOGIN_EMAIL default [email protected]
-# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-changeme123!}
+# E2E_USER_PASSWORD default ${ADMIN_PASSWORD:-admin123!}
# STORM_CONTEXT_REF Context CR fixtures reference (default context1)
# STORM_MODEL_PROVIDER / STORM_MODEL_NAME fixture model (default zai/glm-5.1)
-# CLERUM_PROFILE_PORTS_ENV optional explicit path to the selected branch
-# profile's generated ports.env.
+# HCC_METRICS_LOCAL_PORT ephemeral scrape port (default 18082)
+# KEEP_FIXTURES set to 1 to skip teardown (diagnosis only)
#
# Prereqs (each a HARD FAIL with the concrete reason):
# - branch HCC image deployed (clerum_hcc_host_delete_cleanup_total
@@ -126,10 +122,7 @@
# - external-rest-api + rpc-proxy reachable (make minikube-pf-all)
#
# Usage:
-# MINIKUBE_PROFILE=<profile> KUBECONTEXT=<profile> \
-# E2E_EXPECTED_PRE_GATE_GATE=<gate> \
-# E2E_HCC_HOST_STORM_FAULT_INJECTION=1 \
-# bash scripts/e2e/e2e-host-storm-gate.sh
+# CONTEXT=clerum-test bash scripts/e2e/e2e-host-storm-gate.sh
#
# EXIT CODES
# 0 every storm assertion passed and teardown left zero fixtures
@@ -138,17 +131,6 @@
# ======================================================================
set -euo pipefail
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=scripts/e2e/e2e-lib.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/e2e-lib.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
-# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-lock.sh
-# shellcheck disable=SC1091
-source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
-
# ─── Constants mirrored from host-context-controller/src/{constants,config}.ts ──
readonly MANAGED_BY_LABEL='clerum.io/managed-by'
readonly MANAGED_BY_VALUE='host-context-controller'
@@ -162,10 +144,6 @@ readonly CONTROL_NS='control-plane'
readonly HCC_DEPLOY='host-context-controller'
readonly HCC_PORT='8081' # containerPort http (deploy/base/control-plane)
readonly HCC_METRICS_PATH='/metrics' # server.ts GET /metrics
-readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'
-readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'
-# Shared HCC ownership/lock helpers use HCC_NS.
-HCC_NS="$CONTROL_NS"
# Owned-child kinds + namespaces of a per-Host bundle (mirrors
# measure-host-bundle-reconcile.sh — includes per-Host NetworkPolicies).
@@ -178,7 +156,6 @@ readonly BUNDLE_NAMESPACES="${MCP_HOST_NS} ${CHANNELS_NS} ${RPC_PROXY_NS}"
# (`QUEUE_WAIT_METRIC="$QUEUE_WAIT_METRIC" python3 …`) would fail as a readonly
# reassignment under `set -e` — export once here and drop the inline prefixes.
readonly QUEUE_WAIT_METRIC='clerum_hcc_host_reconcile_queue_wait_seconds'
-readonly IN_FLIGHT_METRIC='clerum_hcc_host_reconcile_in_flight'
readonly WATCH_RECOVERY_METRIC='clerum_hcc_host_watch_recovery_seconds'
readonly FLEET_METRIC='clerum_hcc_host_fleet_requests_total'
readonly DELETE_CLEANUP_METRIC='clerum_hcc_host_delete_cleanup_total'
@@ -187,20 +164,11 @@ export QUEUE_WAIT_METRIC WATCH_RECOVERY_METRIC FLEET_METRIC DELETE_CLEANUP_METRI
# ─── Storm shape + budgets (each constant states its contract) ────────
# Addendum 5.1: ">=10" created fixture Hosts, mixed stateful/stateless.
readonly STORM_CREATE_COUNT=10
-# A2's direct-watch probe is deliberately separate from the restart window:
-# a cold-start LIST legitimately converges objects on the fleet lane. These
-# ten ADDED events are emitted only once the replacement's LIST→WATCH recovery
-# has succeeded, so their queue histogram is an unambiguous urgent-lane SLO.
-readonly STORM_URGENT_PROBE_COUNT=10
# Addendum 5.1: ">=3" pre-created fixture Hosts deleted mid-storm.
readonly STORM_DELETE_COUNT=3
# §14.2 "urgent new/wake queue wait": local hard gate <= 5s (histogram
# bucket boundary le="5" exists in RECONCILE_LATENCY_BUCKETS).
readonly STORM_URGENT_P95_BUDGET_S=5
-# The queue value itself is adjudicated against the 5s hard gate. This larger
-# deadline bounds collection of the completed reconcile evidence and produces
-# a loud operational failure if the wake never settles or telemetry is absent.
-readonly STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S=60
# §14.2 "New Host materialization": Host CR -> Deployment <= 20s.
readonly STORM_MATERIALIZATION_BUDGET_S=20
# §14.2 "Watch recovery": recovery requested -> WATCH installed <= 15s
@@ -272,44 +240,26 @@ print_results() {
log "fairness and convergence under realistic concurrency, not API degradation."
}
+# ─── kubectl wrapper — EVERY call carries --context (CLAUDE.md mandate) ──
+kctl() { kubectl --context "$CONTEXT" "$@"; }
+
# ─── Portable time helper (python3; mirrors measure script) ───────────
now_ms() { python3 -c 'import time;print(int(time.time()*1000))'; }
# ─── Strict local-profile context guard (fail-loud) ───────────────────
require_storm_context() {
- [[ -n "${E2E_KUBECONTEXT:-}" ]] ||
- die "KUBECONTEXT or E2E_K8S_CONTEXT must select an explicit branch-scoped minikube context." 2
- is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" ||
- die "Refusing Host storm fault injection on non-branch context '${E2E_KUBECONTEXT}'." 2
- require_safe_kube_context ||
- die "context '${E2E_KUBECONTEXT}' is not an allowed local E2E context." 2
+ [[ -n "${CONTEXT:-}" ]] || die "CONTEXT is required (a local minikube profile context)." 2
+ case "$CONTEXT" in
+ *gke_*|*prod*)
+ die "Refusing: CONTEXT='${CONTEXT}' looks like a GKE/production context. Local profiles only." 2 ;;
+ esac
+ if [[ ! "$CONTEXT" =~ ^clerum-(test|codex-|detached-|claude-) ]]; then
+ die "Refusing: CONTEXT='${CONTEXT}' is not an allowed local profile. Must match ^clerum-(test|codex-|detached-|claude-)." 2
+ fi
if ! kctl get nodes -o name >/dev/null 2>&1; then
- die "context '${E2E_KUBECONTEXT}' is not reachable (kubectl get nodes failed)." 1
+ die "CONTEXT='${CONTEXT}' is not reachable (kubectl get nodes failed). Start the profile and its port-forwards." 1
fi
- kctl get nodes -o json | jq -e --arg context "$E2E_KUBECONTEXT" \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' >/dev/null ||
- die "target context is not the exact selected minikube profile '${E2E_KUBECONTEXT}'." 2
- log "branch-owned minikube context OK: ${E2E_KUBECONTEXT}"
-}
-
-load_branch_profile_endpoints() {
- local ports_env external_url rpc_url
- ports_env="${CLERUM_PROFILE_PORTS_ENV:-${HOME}/.cache/clerum/minikube-profiles/${E2E_KUBECONTEXT}/ports.env}"
- [[ -r "$ports_env" ]] ||
- die "branch profile ports evidence is missing at ${ports_env}; refresh the branch-profile helper." 2
- external_url="$(profile_env_value EXTERNAL_REST_API_BASE_URL "$ports_env")"
- [[ -n "$external_url" ]] ||
- external_url="$(profile_env_value EXTERNAL_REST_API_URL "$ports_env")"
- rpc_url="$(profile_env_value RPC_PROXY_BASE_URL "$ports_env")"
- [[ -n "$rpc_url" ]] ||
- rpc_url="$(profile_env_value RPC_PROXY_URL "$ports_env")"
- [[ "$external_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||
- die "branch ports.env has no valid loopback External REST URL." 2
- [[ "$rpc_url" =~ ^http://127\.0\.0\.1:[0-9]{2,5}$ ]] ||
- die "branch ports.env has no valid loopback RPC proxy URL." 2
- EXT_BASE="$external_url"
- RPC_BASE="$rpc_url"
- log "using branch-profile endpoints from ${ports_env}"
+ log "context OK: ${CONTEXT}"
}
require_deps() {
@@ -345,12 +295,11 @@ RUN_ID=''; WORK_DIR=''
FIXTURE_SECRET=''
STORM_CREATED_HOSTS=()
STORM_DELETE_HOSTS=()
-STORM_URGENT_PROBE_HOSTS=()
WAKE_HOST_REF="${E2E_STORM_WAKE_HOST_REF:-chatllm-stateless}"
-EXT_BASE=''
-RPC_BASE=''
+EXT_BASE="${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}"
+RPC_BASE="${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}"
DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:[email protected]}"
-DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"
+DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-admin123!}}"
STORM_CTX_REF="${STORM_CONTEXT_REF:-context1}"
STORM_PROVIDER="${STORM_MODEL_PROVIDER:-zai}"
STORM_MODEL="${STORM_MODEL_NAME:-glm-5.1}"
@@ -362,14 +311,6 @@ T_CREATE_FIRST_MS=''; T_DELETE_MS=''; T_WAKE_MS=''; T_KILL_MS=''; T_CREATE_LAST_
PF_PID=''
HCC_ENV_SAVED=''
TEARDOWN_DONE=0
-TEARDOWN_CLEANUP_FAILED=0
-HCC_RESTORE_OK=1
-STORM_FIXTURES_CREATED=0
-# Read and mutated by the sourced compare-and-swap lock helper.
-HCC_GATE_LOCK_ACQUIRED=0
-HCC_GATE_LOCK_NAME=''
-HCC_GATE_LOCK_UID=''
-HCC_GATE_FINALIZATION_FAILURE=''
cleanup_portforward() {
if [[ -n "$PF_PID" ]] && kill -0 "$PF_PID" 2>/dev/null; then
@@ -399,16 +340,10 @@ restore_hcc_env() {
_hcc_restore_args
[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }
log "Restoring HCC cadence env on deployment/${HCC_DEPLOY}"
- if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then
- warn "failed to restore HCC env (manual check required)"
- HCC_RESTORE_OK=0
- return 1
- fi
- if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1 || \
+ warn "failed to restore HCC env (manual check advised)"
+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1 || \
warn "HCC rollout did not settle after env restore"
- HCC_RESTORE_OK=0
- return 1
- fi
HCC_ENV_SAVED=''
}
@@ -423,12 +358,10 @@ restore_hcc_env_strict() {
[[ ${#args[@]} -gt 0 ]] || { HCC_ENV_SAVED=''; return 0; }
log "Restoring HCC cadence env on deployment/${HCC_DEPLOY} (strict)"
if ! kctl set env "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" "${args[@]}" >/dev/null 2>&1; then
- HCC_RESTORE_OK=0
fail "cadence restore FAILED — the profile is left with test cadences on deployment/${HCC_DEPLOY} (the EXIT trap retries best-effort; verify manually)"
return 0
fi
if ! kctl rollout status "deployment/${HCC_DEPLOY}" -n "$CONTROL_NS" --timeout=180s >/dev/null 2>&1; then
- HCC_RESTORE_OK=0
fail "HCC rollout did not settle after the cadence restore — verify deployment/${HCC_DEPLOY} manually"
return 0
fi
@@ -438,11 +371,10 @@ restore_hcc_env_strict() {
# shellcheck disable=SC2329 # invoked via the EXIT trap below
cleanup_on_exit() {
- local status=$? cleanup_failed=0
+ local status=$?
set +e
cleanup_portforward
- if [[ "$TEARDOWN_DONE" != '1' && "$STORM_FIXTURES_CREATED" == '1' && -n "$RUN_ID" ]]; then
- cleanup_failed=1
+ if [[ "$TEARDOWN_DONE" != '1' && -n "$RUN_ID" && "${KEEP_FIXTURES:-}" != '1' ]]; then
local sel="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"
warn "abnormal exit — best-effort deletion of storm fixtures (${sel})"
kctl -n "$MCP_HOST_NS" delete host -l "$sel" --wait=false >/dev/null 2>&1 || \
@@ -450,10 +382,7 @@ cleanup_on_exit() {
kctl -n "$MCP_HOST_NS" delete secret -l "$sel" --wait=false >/dev/null 2>&1 || \
warn "best-effort fixture Secret deletion failed (manual check advised)"
fi
- restore_hcc_env || HCC_RESTORE_OK=0
- if ! finalize_hcc_watch_gate_lock "$cleanup_failed" "$HCC_RESTORE_OK"; then
- status=1
- fi
+ restore_hcc_env
exit "$status"
}
trap cleanup_on_exit EXIT
@@ -476,9 +405,6 @@ init_run() {
for i in $(seq 1 "$STORM_DELETE_COUNT"); do
STORM_DELETE_HOSTS+=("$(printf 's791-%s-d%d' "$RUN_ID" "$i")")
done
- for i in $(seq 1 "$STORM_URGENT_PROBE_COUNT"); do
- STORM_URGENT_PROBE_HOSTS+=("$(printf 's791-%s-u%02d' "$RUN_ID" "$i")")
- done
log "run-id ${RUN_ID}; work dir ${WORK_DIR}"
log "create fixtures: ${STORM_CREATED_HOSTS[*]}"
log "delete fixtures: ${STORM_DELETE_HOSTS[*]}"
@@ -605,30 +531,14 @@ storm_diagnostics() {
# different pod (e.g. a terminating one during a rollout) than the one the
# identity guard checked; cross-pod counter deltas would false-PASS A4.
scrape_hcc_metrics() {
- local out="$1" pod="$2" lport='' url pf_log
- pf_log="${WORK_DIR}/pf-${pod}.log"
- # Let kubectl allocate a process-owned loopback port. A fixed port is
- # machine-global and can silently collide with another branch/profile gate.
- kctl -n "$CONTROL_NS" port-forward --address=127.0.0.1 \
- "pod/${pod}" ":${HCC_PORT}" >"$pf_log" 2>&1 &
+ local out="$1" pod="$2" lport="${HCC_METRICS_LOCAL_PORT:-18082}"
+ local url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"
+ kctl -n "$CONTROL_NS" port-forward "pod/${pod}" "${lport}:${HCC_PORT}" >"${WORK_DIR}/pf.log" 2>&1 &
PF_PID=$!
# shellcheck disable=SC2329 # invoked by name via wait_until
- _pf_ready_probe() {
- kill -0 "$PF_PID" 2>/dev/null || return 1
- lport="$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> .*/\1/p' "$pf_log" | head -n1)"
- [[ "$lport" =~ ^[0-9]+$ ]]
- }
- # shellcheck disable=SC2329 # invoked by name via wait_until
- _pf_diag() { { echo '----- port-forward log -----'; cat "$pf_log" 2>/dev/null; } >&2; }
- if ! wait_until 30 1 "random HCC metrics port-forward" _pf_ready_probe _pf_diag; then
- cleanup_portforward
- return 1
- fi
- url="http://127.0.0.1:${lport}${HCC_METRICS_PATH}"
- # shellcheck disable=SC2329 # invoked by name via wait_until
_scrape_probe() { curl -fsS --max-time 3 "$url" -o "$out" 2>/dev/null; }
# shellcheck disable=SC2329 # invoked by name via wait_until
- _scrape_diag() { _pf_diag; }
+ _scrape_diag() { { echo '----- port-forward log -----'; cat "${WORK_DIR}/pf.log" 2>/dev/null; } >&2; }
if ! wait_until 30 2 "HCC metrics scrape -> ${out}" _scrape_probe _scrape_diag; then
cleanup_portforward
return 1
@@ -641,23 +551,6 @@ wake_lifecycle_state() {
kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true
}
-# shellcheck disable=SC2329 # invoked from the wait_until callback below
-wake_handled_generation() {
- kctl -n "$MCP_HOST_NS" get host "$WAKE_HOST_REF" \
- -o jsonpath='{.status.lifecycle.wakeHandledGeneration}' 2>/dev/null || true
-}
-
-# Correlate the public wake request with HCC's durable lifecycle status. This
-# predicate is deliberately pure so the static gate test can exercise the
-# same ordering contract used by the cluster E2E.
-# shellcheck disable=SC2329 # invoked from the wait_until callback below
-wake_generation_is_handled() {
- local requested="$1" handled="$2"
- [[ "$requested" =~ ^[1-9][0-9]*$ ]] || return 1
- [[ "$handled" =~ ^[0-9]+$ ]] || return 1
- (( handled >= requested ))
-}
-
# shellcheck disable=SC2329 # invoked from _wake_converged_probe (runs via wait_until)
wake_ready_pod_exists() {
kctl -n "$MCP_HOST_NS" get pods -l "app=${WAKE_HOST_REF}" \
@@ -696,31 +589,19 @@ mint_rpc_token() {
# The wake route is the user-facing rpc-proxy endpoint Desktop uses;
# rpc-proxy owns the control-api hop. Never kubectl scale.
-WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''
+WAKE_STATUS=''; WAKE_BODY=''
post_wake() {
- local resp response_status response_generation
- WAKE_STATUS=''; WAKE_BODY=''; WAKE_GENERATION=''
+ local resp
mint_rpc_token || return 1
resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST \
"${RPC_BASE}/api/v1/rpc/hosts/${WAKE_HOST_REF}/wake" \
-H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json') || {
echo "wake POST failed at ${RPC_BASE}" >&2; return 1; }
WAKE_STATUS="$(echo "$resp" | tail -n1)"; WAKE_BODY="$(echo "$resp" | sed '$d')"
- [[ "$WAKE_STATUS" == '202' ]] || {
- echo "wake POST returned HTTP ${WAKE_STATUS}, expected 202: ${WAKE_BODY}" >&2
- return 1
- }
- response_status="$(jq -r '.status // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"
- response_generation="$(jq -r '.wakeGeneration // empty' <<<"$WAKE_BODY" 2>/dev/null || true)"
- [[ "$response_status" == 'wake-requested' ]] || {
- echo "wake POST returned unexpected status '${response_status:-<missing>}': ${WAKE_BODY}" >&2
- return 1
- }
- [[ "$response_generation" =~ ^[1-9][0-9]*$ ]] || {
- echo "wake POST returned invalid wakeGeneration '${response_generation:-<missing>}': ${WAKE_BODY}" >&2
- return 1
- }
- WAKE_GENERATION="$response_generation"
+ case "$WAKE_STATUS" in
+ 200|202|204) return 0 ;;
+ *) echo "wake POST returned HTTP ${WAKE_STATUS}: ${WAKE_BODY}" >&2; return 1 ;;
+ esac
}
# ─── HCC cadence shortening (only when the wake target must suspend) ──
@@ -773,15 +654,8 @@ _wake_target_diag() {
require_deps
require_storm_context
init_run
-[[ "${KEEP_FIXTURES:-0}" != '1' ]] ||
- die "KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result; refusing before mutation." 2
-[[ "${E2E_HCC_HOST_STORM_FAULT_INJECTION:-0}" == '1' ]] ||
- die "Set E2E_HCC_HOST_STORM_FAULT_INJECTION=1 to acknowledge the temporary HCC pod restart." 2
-require_branch_owned_hcc_gate "$CONTROL_NS"
-load_branch_profile_endpoints
header "Prerequisites"
-ok "branch helper, profile, exact HEAD/fingerprint/gate, and minikube node label verified"
if kctl get deployment "$HCC_DEPLOY" -n "$CONTROL_NS" >/dev/null 2>&1; then
ok "HCC deployment present in ${CONTROL_NS}"
else
@@ -821,18 +695,12 @@ else
fail "cannot mint RPC token for hostRef=${WAKE_HOST_REF} -- is ${DEV_EMAIL} associated to it? Seed: bash scripts/e2e/seed-stateless-host.sh"
print_results; exit 1
fi
-if acquire_hcc_watch_gate_lock; then
- ok "exclusive shared HCC fault-injection lock acquired"
-else
- die "another disruptive HCC gate owns context ${E2E_KUBECONTEXT}"
-fi
# ======================================================================
# Prep 1 — storm fixtures that will be DELETED mid-storm (bundles must
# exist first: deleting a bundle-less Host proves nothing)
# ======================================================================
header "Prep — pre-create ${STORM_DELETE_COUNT} delete-target fixture Hosts"
-STORM_FIXTURES_CREATED=1
ensure_fixture_secret
for h in "${STORM_DELETE_HOSTS[@]}"; do
apply_storm_host "$h" false
@@ -880,9 +748,6 @@ SCRAPE_A="${WORK_DIR}/metrics-A.prom"
SCRAPE_B="${WORK_DIR}/metrics-B.prom"
SCRAPE_C1="${WORK_DIR}/metrics-C1.prom"
SCRAPE_C2="${WORK_DIR}/metrics-C2.prom"
-SCRAPE_D="${WORK_DIR}/metrics-D.prom"
-SCRAPE_WAKE_PRE="${WORK_DIR}/metrics-wake-pre.prom"
-SCRAPE_WAKE_POST="${WORK_DIR}/metrics-wake-post.prom"
HCC_POD_A="$(hcc_pod_name)"
if [[ -z "$HCC_POD_A" ]]; then
fail "no Running HCC pod found for baseline identity"; storm_diagnostics; print_results; exit 1
@@ -916,89 +781,22 @@ for h in "${STORM_DELETE_HOSTS[@]}"; do
done
log "storm: ${STORM_DELETE_COUNT} mid-storm Host deletions issued"
-# Isolate the supported wake on the pre-kill pod. Samples from the first half
-# of Host creation may still settle inside this window; they are valid urgent
-# traffic and the hard-gate assertion below requires every one of them, not
-# only a percentile, to remain within budget.
-HCC_POD_WAKE="$(hcc_pod_name)"
-if [[ -z "$HCC_POD_WAKE" || "$HCC_POD_WAKE" != "$HCC_POD_A" ]]; then
- fail "storm: HCC pod changed before the wake metric baseline (${HCC_POD_A} -> ${HCC_POD_WAKE:-<none>})"
- storm_diagnostics; print_results; exit 1
-fi
-if ! scrape_hcc_metrics "$SCRAPE_WAKE_PRE" "$HCC_POD_WAKE"; then
- fail "storm: could not capture the same-pod wake metric baseline"
- print_results; exit 1
-fi
-
T_WAKE_MS="$(now_ms)"
if post_wake; then
- log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS}, generation ${WAKE_GENERATION})"
+ log "storm: wake POST accepted for ${WAKE_HOST_REF} (HTTP ${WAKE_STATUS})"
else
fail "storm: mid-storm wake POST failed for ${WAKE_HOST_REF}"; print_results; exit 1
fi
-# Prove that this specific supported wake reached HCC: its response generation
-# must be durably reflected in lifecycle status. Then require a new urgent
-# queue sample and an idle urgent lane on the same pod. Prometheus collectors
-# are read independently, so this deliberately uses a two-scrape fence instead
-# of assuming cross-metric atomicity: once the first scrape observes the urgent
-# lane idle, HCC's observe-before-in-flight-decrement ordering guarantees the
-# wake observation is durable and therefore visible to the second scrape.
-# shellcheck disable=SC2329 # invoked by name via wait_until
-_wake_urgent_evidence_recorded() {
- local current_pod lifecycle_state handled_generation
- local fence_in_flight pre_count post_count in_flight
- current_pod="$(hcc_pod_name)"
- [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_WAKE" ]] || return 1
- lifecycle_state="$(wake_lifecycle_state)"
- [[ -n "$lifecycle_state" && "$lifecycle_state" != 'suspended' ]] || return 1
- handled_generation="$(wake_handled_generation)"
- wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation" || return 1
- scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1
- fence_in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '
- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }
- END { print total + 0 }
- ' "$SCRAPE_WAKE_POST")"
- awk -v active="$fence_in_flight" 'BEGIN { exit !(active == 0) }' || return 1
-
- # The wake histogram observation precedes the decrement seen by the fence.
- # Re-scrape so the adjudicated histogram cannot predate that observation.
- scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE" || return 1
- pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
- END { print total + 0 }
- ' "$SCRAPE_WAKE_PRE")"
- post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
- END { print total + 0 }
- ' "$SCRAPE_WAKE_POST")"
- in_flight="$(awk -v metric="${IN_FLIGHT_METRIC}" '
- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ { total += $NF }
- END { print total + 0 }
- ' "$SCRAPE_WAKE_POST")"
- awk -v before="$pre_count" -v after="$post_count" -v active="$in_flight" \
- 'BEGIN { exit !((after - before) >= 1 && active == 0) }'
-}
-if wait_until "$STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S" 1 \
- "supported wake urgent reconcile + same-pod queue evidence" \
- _wake_urgent_evidence_recorded _wake_target_diag; then
- ok "A2 wake input: supported wake reconcile completed and published same-pod urgent queue evidence"
-else
- fail "A2 wake input: wake did not complete with durable same-pod urgent queue evidence within ${STORM_WAKE_QUEUE_EVIDENCE_DEADLINE_S}s"
- print_results; exit 1
-fi
-
-# Capture A7's fleet-failure baseline immediately before the kill. The wake
-# scrape remains the A2 wake-window oracle; a separate fresh scrape prevents
-# fleet failures after wake adjudication from falling outside A7's delta.
+# Scrape B: last look at the pre-kill pod's counters, immediately before
+# the kill (samples still in flight at process death are a recorded gap).
HCC_POD_B="$(hcc_pod_name)"
if [[ "$HCC_POD_B" != "$HCC_POD_A" ]]; then
fail "storm: HCC pod changed unexpectedly before the kill (${HCC_POD_A} -> ${HCC_POD_B:-<none>}) — counter deltas would be invalid"
storm_diagnostics; print_results; exit 1
fi
if ! scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"; then
- fail "storm: could not capture the fresh pre-kill metric baseline"
- print_results; exit 1
+ fail "storm: could not scrape HCC metrics immediately before the kill"; print_results; exit 1
fi
log "storm: pre-kill scrape B captured (pod ${HCC_POD_B})"
@@ -1050,41 +848,6 @@ else
print_results; exit 1
fi
-# A7's causal business signal is the replacement pod's own completion log for
-# its startup fleet reconciliation. A scheduled/started counter is insufficient:
-# it can increment before the pass fails, stalls, or is superseded.
-FLEET_COMPLETION_PROVEN=0
-replacement_fleet_state=''
-# shellcheck disable=SC2329 # invoked by name via wait_until
-_replacement_fleet_completed_probe() {
- local logs current_pod
- current_pod="$(hcc_pod_name)"
- [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1
- logs="$(kctl -n "$CONTROL_NS" logs "pod/${HCC_POD_C}" \
- -c host-context-controller 2>/dev/null)" || return 1
- if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then
- replacement_fleet_state='failed'
- return 0
- fi
- if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then
- replacement_fleet_state='completed'
- return 0
- fi
- return 1
-}
-if ! wait_until "$STORM_CONVERGENCE_DEADLINE_S" 2 \
- "replacement HCC initial Host fleet reconciliation completion" \
- _replacement_fleet_completed_probe storm_diagnostics; then
- fail "A7: replacement pod did not complete its initial Host fleet reconciliation within ${STORM_CONVERGENCE_DEADLINE_S}s"
- print_results; exit 1
-fi
-if [[ "$replacement_fleet_state" != 'completed' ]]; then
- fail "A7: replacement pod's initial Host fleet reconciliation failed"
- storm_diagnostics; print_results; exit 1
-fi
-FLEET_COMPLETION_PROVEN=1
-ok "A7 input: replacement pod logged completed initial Host fleet reconciliation"
-
# shellcheck disable=SC2329 # invoked by name via wait_until
_all_created_deploys_probe() {
local h
@@ -1117,68 +880,6 @@ if ! scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"; then
fi
log "scrape C1 captured (replacement pod ${HCC_POD_C})"
-# ─── A2 input: direct-watch urgent probe after replacement LIST→WATCH ──
-# The storm's second half is intentionally issued immediately after pod
-# deletion, when a cold-start LIST may legitimately own those Hosts on the
-# fleet lane. C1 proves the replacement has completed LIST→WATCH recovery;
-# issue a fresh, independent set now so every observed sample is a direct
-# watch ADDED admission on the urgent lane.
-header "A2 input — urgent direct-watch probe (${STORM_URGENT_PROBE_COUNT} Hosts)"
-for i in $(seq 1 "${#STORM_URGENT_PROBE_HOSTS[@]}"); do
- h="${STORM_URGENT_PROBE_HOSTS[$((i - 1))]}"
- if (( i % 2 == 0 )); then apply_storm_host "$h" true; else apply_storm_host "$h" false; fi
-done
-log "urgent probe: ${STORM_URGENT_PROBE_COUNT} Host ADDED events issued after replacement LIST→WATCH"
-
-# shellcheck disable=SC2329 # invoked by name via wait_until
-_urgent_probe_deploys_probe() {
- local h
- for h in "${STORM_URGENT_PROBE_HOSTS[@]}"; do
- kctl -n "$MCP_HOST_NS" get deployment "$h" >/dev/null 2>&1 || return 1
- done
-}
-if wait_until "$STORM_CONVERGENCE_DEADLINE_S" 3 \
- "all ${STORM_URGENT_PROBE_COUNT} urgent-probe Hosts' Deployments to exist" \
- _urgent_probe_deploys_probe storm_diagnostics; then
- ok "A2 input: every direct-watch urgent-probe Host materialized"
-else
- fail "A2 input: urgent-probe Host materialization incomplete after ${STORM_CONVERGENCE_DEADLINE_S}s"
- print_results; exit 1
-fi
-HCC_POD_C_NOW="$(hcc_pod_name)"
-if [[ -z "$HCC_POD_C_NOW" || "$HCC_POD_C_NOW" != "$HCC_POD_C" ]]; then
- fail "replacement HCC pod changed before urgent-probe scrape D (${HCC_POD_C} -> ${HCC_POD_C_NOW:-<none>}) — urgent counter window is invalid"
- storm_diagnostics; print_results; exit 1
-fi
-
-# Deployment existence can precede the end of reconcileCore, where the queue
-# histogram is observed. Poll the metric itself instead of sleeping, otherwise
-# a fast API server can make this gate race its own telemetry.
-# shellcheck disable=SC2329 # invoked by name via wait_until
-_urgent_probe_metrics_recorded() {
- local current_pod pre_count post_count
- current_pod="$(hcc_pod_name)"
- [[ -n "$current_pod" && "$current_pod" == "$HCC_POD_C" ]] || return 1
- scrape_hcc_metrics "$SCRAPE_D" "$HCC_POD_C" || return 1
- pre_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
- END { print total + 0 }
- ' "$SCRAPE_C1")"
- post_count="$(awk -v metric="${QUEUE_WAIT_METRIC}_bucket" '
- index($0, metric "{") == 1 && $0 ~ /lane="urgent"/ && $0 ~ /le="\+Inf"/ { total += $NF }
- END { print total + 0 }
- ' "$SCRAPE_D")"
- awk -v before="$pre_count" -v after="$post_count" -v required="$STORM_URGENT_PROBE_COUNT" \
- 'BEGIN { exit !((after - before) >= required) }'
-}
-if ! wait_until 60 2 \
- "at least ${STORM_URGENT_PROBE_COUNT} direct-watch urgent metric samples" \
- _urgent_probe_metrics_recorded storm_diagnostics; then
- fail "urgent-probe reconciles did not publish ${STORM_URGENT_PROBE_COUNT} queue samples within 60s"
- print_results; exit 1
-fi
-log "scrape D captured (replacement pod ${HCC_POD_C}; direct-watch urgent window C1→D)"
-
# ─── A3: per-host materialization <= 20s from ITS OWN creation ───────
header "A3 — per-host materialization (budget ${STORM_MATERIALIZATION_BUDGET_S}s, §14.2)"
MAT_ROWS="${WORK_DIR}/materialization.rows"
@@ -1290,14 +991,13 @@ if ! scrape_hcc_metrics "$SCRAPE_C2" "$HCC_POD_C"; then
fi
log "scrape C2 captured (replacement pod ${HCC_POD_C}; A6/A7 counters settled post-A5)"
-# One python pass performs the five metric assertions and emits one
+# One python pass performs the four metric assertions and emits one
# VERDICT|<name>|PASS/FAIL|<detail> line each. python exits nonzero only
# when it cannot parse its inputs (a hard failure of the gate itself).
METRIC_VERDICTS="${WORK_DIR}/metric-verdicts.txt"
-if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" SCRAPE_D="$SCRAPE_D" \
- SCRAPE_WAKE_PRE="$SCRAPE_WAKE_PRE" SCRAPE_WAKE_POST="$SCRAPE_WAKE_POST" \
+if ! SCRAPE_A="$SCRAPE_A" SCRAPE_B="$SCRAPE_B" SCRAPE_C1="$SCRAPE_C1" SCRAPE_C2="$SCRAPE_C2" \
P95_BUDGET_S="$STORM_URGENT_P95_BUDGET_S" RECOVERY_BUDGET_S="$STORM_WATCH_RECOVERY_BUDGET_S" \
- MIN_URGENT_SAMPLES="$STORM_URGENT_PROBE_COUNT" FLEET_COMPLETION_PROVEN="$FLEET_COMPLETION_PROVEN" \
+ MIN_URGENT_SAMPLES="$STORM_CREATE_COUNT" \
python3 >"$METRIC_VERDICTS" <<'PY'
import os, re, sys
from collections import defaultdict
@@ -1333,9 +1033,6 @@ a, _ = load(os.environ["SCRAPE_A"])
b, _ = load(os.environ["SCRAPE_B"])
c1, _ = load(os.environ["SCRAPE_C1"])
c2, c2_types = load(os.environ["SCRAPE_C2"])
-d, _ = load(os.environ["SCRAPE_D"])
-wake_pre, _ = load(os.environ["SCRAPE_WAKE_PRE"])
-wake_post, _ = load(os.environ["SCRAPE_WAKE_POST"])
def buckets(samples, metric, match):
"""Cumulative histogram buckets {le: count} summed over matching series."""
@@ -1360,60 +1057,30 @@ def le_key(le):
verdicts = []
-# ── A2a: every completed urgent reconcile in the supported wake window ─
+# ── A2: urgent queue-wait p95 <= budget across (B-A) + C1 ─────────────
qm = os.environ["QUEUE_WAIT_METRIC"]
budget = float(os.environ["P95_BUDGET_S"])
-wake_before = buckets(wake_pre, qm, {"lane": "urgent"})
-wake_after = buckets(wake_post, qm, {"lane": "urgent"})
-wake_delta = defaultdict(float)
-wake_reset = False
-for le in set(wake_before) | set(wake_after):
- delta = wake_after.get(le, 0.0) - wake_before.get(le, 0.0)
- if delta < 0:
- wake_reset = True
- wake_delta[le] = delta
-if wake_reset:
- verdicts.append(("A2-wake-urgent-hard-gate", False,
- "negative bucket delta in the same-pod wake window"))
-else:
- wake_total = wake_delta.get("+Inf", 0.0)
- wake_within = wake_delta.get(str(int(budget)), wake_delta.get(str(budget), 0.0))
- wake_dist = " ".join(
- f"le{le}={wake_delta[le]:.0f}" for le in sorted(wake_delta, key=le_key)
- )
- if wake_total < 1:
- verdicts.append(("A2-wake-urgent-hard-gate", False,
- "no completed urgent queue sample in the supported wake window"))
- elif wake_within != wake_total:
- verdicts.append(("A2-wake-urgent-hard-gate", False,
- f"{wake_total - wake_within:.0f} of {wake_total:.0f} urgent sample(s) exceeded "
- f"{budget:.0f}s in the supported wake window ({wake_dist})"))
- else:
- verdicts.append(("A2-wake-urgent-hard-gate", True,
- f"all {wake_total:.0f} urgent sample(s) <= {budget:.0f}s in the supported "
- f"same-pod wake window ({wake_dist})"))
-
-# ── A2b: urgent queue-wait p95 across direct-watch window D-C1 ────────
-window_note = "window: replacement-pod direct-watch probe delta (D-C1) after LIST→WATCH recovery"
-pre = buckets(c1, qm, {"lane": "urgent"})
-post = buckets(d, qm, {"lane": "urgent"})
+window_note = "windows: pre-kill delta (B-A) + replacement pod C1, which includes replacement-pod recovery traffic"
+pre_a = buckets(a, qm, {"lane": "urgent"})
+pre_b = buckets(b, qm, {"lane": "urgent"})
+post = buckets(c1, qm, {"lane": "urgent"})
combined = defaultdict(float)
reset = False
-for le in set(pre) | set(post):
- delta = post.get(le, 0.0) - pre.get(le, 0.0)
+for le in set(pre_a) | set(pre_b) | set(post):
+ delta = pre_b.get(le, 0.0) - pre_a.get(le, 0.0)
if delta < 0:
reset = True
- combined[le] = delta
+ combined[le] = delta + post.get(le, 0.0)
if reset:
- verdicts.append(("A2-direct-watch-urgent-p95", False,
- "negative bucket delta between scrapes C1 and D (unexpected counter reset)"))
+ verdicts.append(("A2-urgent-queue-wait-p95", False,
+ "negative bucket delta between scrapes A and B (unexpected counter reset)"))
else:
total = combined.get("+Inf", 0.0)
min_samples = float(os.environ["MIN_URGENT_SAMPLES"])
if total < min_samples:
- verdicts.append(("A2-direct-watch-urgent-p95", False,
- f"only {total:.0f} direct-watch urgent samples (need >= {min_samples:.0f} — "
- "the urgent probe did not exercise its declared lane)"))
+ verdicts.append(("A2-urgent-queue-wait-p95", False,
+ f"only {total:.0f} urgent samples in the storm window (need >= {min_samples:.0f} — "
+ "the storm did not exercise the urgent lane, cannot claim a p95)"))
else:
p95_ub = None
for le in sorted(combined, key=le_key):
@@ -1422,10 +1089,11 @@ else:
break
dist = " ".join(f"le{le}={combined[le]:.0f}" for le in sorted(combined, key=le_key))
if p95_ub is not None and le_key(p95_ub) <= budget:
- verdicts.append(("A2-direct-watch-urgent-p95", True,
- f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note})"))
+ verdicts.append(("A2-urgent-queue-wait-p95", True,
+ f"p95 <= {p95_ub}s over {total:.0f} samples (budget {budget:.0f}s; {dist}; {window_note}; "
+ "gap: samples in flight between scrape B and process death are unrecoverable)"))
else:
- verdicts.append(("A2-direct-watch-urgent-p95", False,
+ verdicts.append(("A2-urgent-queue-wait-p95", False,
f"p95 bucket upper bound {p95_ub} exceeds {budget:.0f}s over {total:.0f} samples ({dist}; {window_note})"))
# ── A4: watch recovery <= budget on the replacement pod (C1) ──────────
@@ -1449,36 +1117,28 @@ else:
verdicts.append(("A4-watch-recovery", True,
f"{succ_total:.0f} recovery sample(s), all <= {bound}s (budget {rbudget}s)"))
-# ── A7: recovered fleet convergence remains bounded (settled C2) ─────
+# ── A7: fleet coalescing held (settled window C2) ─────────────────────
fm = os.environ["FLEET_METRIC"]
-# Presence check (A6 pattern): counter() returns 0 for an absent family, so
-# without this a renamed/unregistered FLEET_METRIC makes every count 0 and A7
-# passes vacuously. The causal leg (FLEET_COMPLETION_PROVEN) alone cannot catch
-# a metric rename — the metric itself must be registered on the running image.
-if fm not in c2_types:
- verdicts.append(("A7-fleet-recovery-bounded", False,
- f"{fm} not registered on the running image — is the branch HCC image deployed?"))
+pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")
+post_started = counter(c2, fm, "result", "started")
+post_coalesced = counter(c2, fm, "result", "coalesced")
+post_trailing = counter(c2, fm, "result", "trailing")
+post_failed = counter(c2, fm, "result", "failed")
+problems = []
+if pre_failed > 0 or post_failed > 0:
+ problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")
+if post_started < 1:
+ problems.append("no fleet pass started on the replacement pod (recovery convergence missing)")
+if post_coalesced < 1:
+ problems.append("no coalesced fleet request on the replacement pod (recovery request should land against the active initial pass)")
+if post_trailing > post_coalesced:
+ problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")
+detail = (f"started={post_started:.0f} coalesced={post_coalesced:.0f} "
+ f"trailing={post_trailing:.0f} failed={post_failed:.0f} (pre-kill failed delta={pre_failed:.0f})")
+if problems:
+ verdicts.append(("A7-fleet-coalescing", False, "; ".join(problems) + f" [{detail}]"))
else:
- pre_failed = counter(b, fm, "result", "failed") - counter(a, fm, "result", "failed")
- post_started = counter(c2, fm, "result", "started")
- post_coalesced = counter(c2, fm, "result", "coalesced")
- post_trailing = counter(c2, fm, "result", "trailing")
- post_failed = counter(c2, fm, "result", "failed")
- problems = []
- if pre_failed != 0 or post_failed > 0:
- problems.append(f"fleet pass failures (pre={pre_failed:.0f} post={post_failed:.0f})")
- if os.environ.get("FLEET_COMPLETION_PROVEN") != "1":
- problems.append("replacement-pod initial fleet completion was not causally proven")
- if post_trailing > post_coalesced:
- problems.append(f"trailing={post_trailing:.0f} > coalesced={post_coalesced:.0f} (violates the single-pending-slot structure)")
- detail = (f"completion-proven={os.environ.get('FLEET_COMPLETION_PROVEN', 'missing')} "
- f"started={post_started:.0f} coalesced={post_coalesced:.0f} "
- f"trailing={post_trailing:.0f} failed={post_failed:.0f} "
- f"(replacement pod; A4 recovery-success fenced; pre-kill failed delta={pre_failed:.0f})")
- if problems:
- verdicts.append(("A7-fleet-recovery-bounded", False, "; ".join(problems) + f" [{detail}]"))
- else:
- verdicts.append(("A7-fleet-recovery-bounded", True, detail))
+ verdicts.append(("A7-fleet-coalescing", True, detail))
# ── A6: delete-cleanup counters consistent (settled window C2) ────────
dm = os.environ["DELETE_CLEANUP_METRIC"]
@@ -1518,8 +1178,8 @@ while IFS='|' read -r tag name verdict detail; do
fail "${name}: ${detail}"
fi
done <"$METRIC_VERDICTS"
-if [[ "$VERDICT_COUNT" -ne 5 ]]; then
- fail "metric adjudication emitted ${VERDICT_COUNT}/5 verdicts — truncated output is a failure, not a pass"
+if [[ "$VERDICT_COUNT" -ne 4 ]]; then
+ fail "metric adjudication emitted ${VERDICT_COUNT}/4 verdicts — truncated output is a failure, not a pass"
fi
# ======================================================================
@@ -1527,7 +1187,11 @@ fi
# The profile and seeded data (including the wake target) stay.
# ======================================================================
header "Teardown — storm fixtures (${FIXTURE_RUN_KEY}=${RUN_ID})"
-STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"
+if [[ "${KEEP_FIXTURES:-}" == '1' ]]; then
+ warn "KEEP_FIXTURES=1 — leaving fixtures in place for diagnosis (run-id ${RUN_ID})"
+ TEARDOWN_DONE=1
+else
+ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID}"
kctl -n "$MCP_HOST_NS" delete host -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \
warn "teardown: label-selector Host deletion returned nonzero (leftover check below is authoritative)"
kctl -n "$MCP_HOST_NS" delete secret -l "$STORM_SEL" --wait=false >/dev/null 2>&1 || \
@@ -1550,7 +1214,6 @@ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID
ok "teardown: all storm fixture Hosts deleted via the supported CR path"
else
fail "teardown: storm fixture Hosts still present after ${STORM_TEARDOWN_DEADLINE_S}s (run-id ${RUN_ID})"
- TEARDOWN_CLEANUP_FAILED=1
fi
# FAIL-CLOSED (B2): an unverifiable read counts as a leftover — the teardown
@@ -1564,7 +1227,7 @@ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID
teardown_converge_deadline=$((SECONDS + STORM_TEARDOWN_CONVERGE_S))
while :; do
TEARDOWN_LEFTOVER=0; teardown_orphan=0; teardown_unverifiable=0; teardown_report=''
- for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_URGENT_PROBE_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; do
+ for h in "${STORM_CREATED_HOSTS[@]}" "${STORM_DELETE_HOSTS[@]}"; do
if ! leftovers="$(collect_children "$h")"; then
teardown_report+=" ${h}: kubectl read FAILED (unverifiable, fail-closed)"$'\n'
TEARDOWN_LEFTOVER=$((TEARDOWN_LEFTOVER + 1)); teardown_unverifiable=1
@@ -1617,19 +1280,14 @@ STORM_SEL="${FIXTURE_LABEL_KEY}=${FIXTURE_LABEL_VAL},${FIXTURE_RUN_KEY}=${RUN_ID
ok "teardown: zero storm-fixture leftovers across ${BUNDLE_NAMESPACES} (profile and seeded data untouched)"
else
fail "teardown: ${TEARDOWN_LEFTOVER} storm fixture subject(s) left resources behind (run-id ${RUN_ID}) — investigate before trusting results"
- TEARDOWN_CLEANUP_FAILED=1
fi
-TEARDOWN_DONE=1
+ TEARDOWN_DONE=1
+fi
# M3: on the success path a failed cadence restore must FAIL the gate
# (restore_hcc_env_strict registers fail() and leaves HCC_ENV_SAVED for
# the trap's best-effort retry); the EXIT trap keeps the lenient restore.
restore_hcc_env_strict
-if ! finalize_hcc_watch_gate_lock "$TEARDOWN_CLEANUP_FAILED" "$HCC_RESTORE_OK"; then
- fail "shared HCC fault-injection lock was retained fail-closed; follow the printed repair instructions"
-else
- ok "exclusive shared HCC fault-injection lock finalized"
-fi
print_results
if [[ "$g_fail" -gt 0 ]]; then
diff --git a/scripts/e2e/e2e-wrc-hcc-contracts.sh b/scripts/e2e/e2e-wrc-hcc-contracts.sh
index 570f0f923..e0b4c5029 100755
--- a/scripts/e2e/e2e-wrc-hcc-contracts.sh
+++ b/scripts/e2e/e2e-wrc-hcc-contracts.sh
@@ -271,8 +271,6 @@ spec:
agent:
provider: zai
model: glm-4.7
- triggers:
- onDemand: {}
steps:
- id: tool-step
instruction: "Synthetic status-contract step for toolsCalled args validation."
@@ -298,7 +296,7 @@ else
fi
rm -f "$running_response_file"
-if wait_for_running_workflowrecipe_status >/dev/null; then
+if running_status_json="$(wait_for_running_workflowrecipe_status)"; then
ok "WorkflowRecipe becomes active when workflow execution is running"
else
fail "WorkflowRecipe did not become active on running workflow execution"
@@ -429,7 +427,7 @@ else
fi
rm -f "$pending_response_file"
-if wait_for_pending_preserves_non_terminal_recipe_status >/dev/null; then
+if pending_status_json="$(wait_for_pending_preserves_non_terminal_recipe_status)"; then
ok "WorkflowRecipe stays non-terminal when workflow execution is pending"
else
fail "WorkflowRecipe did not stay non-terminal on pending workflow execution"
diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
index f72cbca00..35a61e913 100755
--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
@@ -18,8 +18,6 @@ BACKEND_ID="${E2E_BACKEND_ID:-be-${RUN_ID}}"
DENIED_POD="${E2E_DENIED_POD:-deny-${RUN_ID}}"
BACKEND_PORT="${E2E_BACKEND_PORT:-8080}"
CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"
-SOURCE_DEPLOYMENT="$SOURCE_ID"
-BACKEND_DEPLOYMENT="$BACKEND_ID"
CREATED=0
cleanup() {
@@ -30,9 +28,9 @@ cleanup() {
--ignore-not-found --wait=false >/dev/null 2>&1 || status=1
wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \
>/dev/null 2>&1 || status=1
- kctl delete deployment "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \
+ kctl delete deployment "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \
--ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- kctl delete service "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \
+ kctl delete service "$SOURCE_ID" "$BACKEND_ID" -n "$SANDBOX_NS" \
--ignore-not-found >/dev/null 2>&1 || status=1
kctl delete networkpolicy -n "$SANDBOX_NS" \
-l "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \
@@ -87,21 +85,6 @@ wait_internal_ready() {
return 1
}
-wait_for_workload_instance() {
- local workload_id=$1 timeout=${2:-$TIMEOUT_POD} elapsed=0 instance
- while [ "$elapsed" -lt "$timeout" ]; do
- instance="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
- -o "jsonpath={.status.workloadInstances.${workload_id}}" 2>/dev/null || true)"
- if [ -n "$instance" ]; then
- printf '%s\n' "$instance"
- return 0
- fi
- sleep "$POLL_INTERVAL"
- elapsed=$((elapsed + POLL_INTERVAL))
- done
- return 1
-}
-
policy_refs() {
kctl get networkpolicy -A -l "$1" \
-o go-template='{{range .items}}{{.metadata.namespace}}/{{.metadata.name}}{{"\n"}}{{end}}' \
@@ -231,26 +214,18 @@ fi
ok "Fixture has no egressBindings shortcut"
header "Phase 2 - WRC reconciliation"
-SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID" "$TIMEOUT_POD")" || {
- fail "Source workload instance was not assigned"
- exit 1
-}
-BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID" "$TIMEOUT_POD")" || {
- fail "Backend workload instance was not assigned"
- exit 1
-}
-wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD" && ok "Source deployment ready" || {
+wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID" "$TIMEOUT_POD" && ok "Source deployment ready" || {
fail "Source deployment not ready"
exit 1
}
-wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD" && ok "Backend deployment ready" || {
+wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID" "$TIMEOUT_POD" && ok "Backend deployment ready" || {
fail "Backend deployment not ready"
exit 1
}
wait_internal_ready "$TIMEOUT_POD"
-resolved_target="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"
-expected_target="http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"
+resolved_target="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"
+expected_target="http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"
[ "$resolved_target" = "$expected_target" ] && ok "TARGET_URL resolved to ${resolved_target}" || {
fail "TARGET_URL resolved to '${resolved_target}', expected '${expected_target}'"
exit 1
@@ -265,7 +240,7 @@ assert_policy "$egress_ref" "egress" "$SOURCE_ID" "$BACKEND_ID"
assert_policy "$ingress_ref" "ingress" "$BACKEND_ID" "$SOURCE_ID"
header "Phase 4 - Positive packet flow"
-positive_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \
+positive_output="$(kctl exec "deploy/${SOURCE_ID}" -n "$SANDBOX_NS" -- \
sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"
printf "%s" "$positive_output" | grep -Fq "issue485-ok" && \
ok "WRC source reached backend through inferred internal dependency" || {
@@ -307,7 +282,7 @@ wait_for_pod "$SANDBOX_NS" "run=${DENIED_POD}" 60 && ok "Unlabeled negative pod
}
if kctl exec "$DENIED_POD" -n "$SANDBOX_NS" -- wget -qO- \
--timeout="$CONNECT_TIMEOUT" --tries=1 \
- "http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then
+ "http://${BACKEND_ID}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then
fail "Unlabeled pod reached backend; policy is too broad or NetworkPolicy is not enforced"
exit 1
fi
diff --git a/scripts/e2e/measure-host-bundle-reconcile.sh b/scripts/e2e/measure-host-bundle-reconcile.sh
index c1f7ee5a7..1eb66d287 100755
--- a/scripts/e2e/measure-host-bundle-reconcile.sh
+++ b/scripts/e2e/measure-host-bundle-reconcile.sh
@@ -151,10 +151,9 @@ readonly DELPASS_RESURRECTION_SETTLE_S=30 # quiet window before re-asserting
# termination grace. A child with NO deletionTimestamp never gets this
# extension (the delete was never issued → immediate fail).
readonly MEASURE_TEARDOWN_CONVERGE_S=120
-# The startup marker is emitted immediately before the cold-start Host
-# inventory/watch recovery. COMPLETE/FAIL below retain the precise
-# 'initial Host reconciliation' reason supplied to the fleet pass.
-readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'
+# Exact log lines emitted by host-context-controller/src/k8sClient.ts for
+# the startup full fleet pass (reason string 'initial Host reconciliation').
+readonly HCC_PASS_STARTED_MARKER='Running initial Host reconciliation'
readonly HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'
readonly HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'
@@ -965,9 +964,9 @@ phase_delete_during_pass() {
_delpass_pass_active_probe() {
local logs
logs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"
- if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi
- if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs"; then pass_state='completed'; return 0; fi
- if grep -Fq -- "$HCC_PASS_STARTED_MARKER" <<<"$logs"; then pass_state='active'; return 0; fi
+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi
+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then pass_state='completed'; return 0; fi
+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_STARTED_MARKER"; then pass_state='active'; return 0; fi
return 1
}
# shellcheck disable=SC2329 # invoked by name via wait_until
@@ -1016,7 +1015,7 @@ phase_delete_during_pass() {
fail "delete-during-pass: could not re-read HCC logs immediately after the delete — overlap cannot be verified (fail-closed)"
return 1
fi
- if grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$post_delete_logs"; then
+ if printf '%s' "$post_delete_logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER"; then
fail "delete-during-pass: pass logged COMPLETED within the probe->delete window — overlap NOT established; re-run"
return 1
fi
@@ -1028,8 +1027,8 @@ phase_delete_during_pass() {
_delpass_pass_completed_probe() {
local logs
logs="$(kctl -n "$CONTROL_NS" logs "pod/${new_pod}" 2>/dev/null || true)"
- if grep -Fq -- "$HCC_PASS_FAILED_MARKER" <<<"$logs"; then pass_state='failed'; return 0; fi
- grep -Fq -- "$HCC_PASS_COMPLETED_MARKER" <<<"$logs" && pass_state='completed'
+ if printf '%s' "$logs" | grep -Fq "$HCC_PASS_FAILED_MARKER"; then pass_state='failed'; return 0; fi
+ printf '%s' "$logs" | grep -Fq "$HCC_PASS_COMPLETED_MARKER" && pass_state='completed'
}
if ! wait_until "$DELPASS_PASS_COMPLETE_DEADLINE_S" 2 "initial full pass completion after the delete" \
_delpass_pass_completed_probe _delpass_logs_diag; then
diff --git a/scripts/minikube/full-setup.sh b/scripts/minikube/full-setup.sh
index 0186ed584..0f9247a86 100755
--- a/scripts/minikube/full-setup.sh
+++ b/scripts/minikube/full-setup.sh
@@ -1012,7 +1012,7 @@ if [ "$RESET_DB" = true ]; then
$KC -n control-plane rollout status deployment/trace-maintenance-worker --timeout=180s >/dev/null
fi
if [ "$RESET_HCC_REPLICAS" -gt 0 ]; then
- $KC -n control-plane rollout status deployment/host-context-controller --timeout=900s >/dev/null
+ $KC -n control-plane rollout status deployment/host-context-controller --timeout=180s >/dev/null
fi
ok "Control-api database and GFS roles converged after reset"
else
diff --git a/scripts/tests/test-converge-control-db-after-reset.sh b/scripts/tests/test-converge-control-db-after-reset.sh
index 590c749c0..71f036828 100755
--- a/scripts/tests/test-converge-control-db-after-reset.sh
+++ b/scripts/tests/test-converge-control-db-after-reset.sh
@@ -51,7 +51,7 @@ case "$*" in
*'rollout status deployment/workflow-recipes --timeout=180s') exit 0 ;;
*'rollout status deployment/trace-maintenance-worker --timeout=180s') exit 0 ;;
*'scale deployment/host-context-controller --replicas=4') exit 0 ;;
- *'rollout status deployment/host-context-controller --timeout=900s') exit 0 ;;
+ *'rollout status deployment/host-context-controller --timeout=180s') exit 0 ;;
*'get pods -l app=workflow-recipes -o name') printf 'pod/workflow-old\n' ;;
*'get pods -l app=trace-maintenance-worker -o name') printf 'pod/trace-old\n' ;;
*'wait --for=delete pod -l app=workflow-recipes --timeout=180s') exit 0 ;;
diff --git a/scripts/tests/test-hcc-api-gateway-readiness.sh b/scripts/tests/test-hcc-api-gateway-readiness.sh
deleted file mode 100755
index ce2930e86..000000000
--- a/scripts/tests/test-hcc-api-gateway-readiness.sh
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/usr/bin/env bash
-# Functions under test are evaluated from the E2E source, so ShellCheck cannot
-# see their use of the mock environment variables declared by each harness.
-# shellcheck disable=SC2034
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-DEPLOYMENT="${ROOT}/deploy/base/control-plane/host-context-controller-api-gateway.yaml"
-CONFIGMAP="${ROOT}/deploy/base/control-plane/configmaps.yaml"
-BOOTSTRAP_E2E="${ROOT}/scripts/e2e/e2e-hcc-readiness-bootstrap.sh"
-
-fail() {
- echo "FAIL: $1" >&2
- exit 1
-}
-
-readiness_probe="$(
- awk '
- /^[[:space:]]*readinessProbe:/ { in_probe = 1; next }
- in_probe && /^[[:space:]]*resources:/ { exit }
- in_probe { print }
- ' "${DEPLOYMENT}"
-)"
-
-if ! grep -Eq '^[[:space:]]*path:[[:space:]]*/health[[:space:]]*$' <<<"${readiness_probe}"; then
- fail "HCC API gateway readiness probe must use the gateway-local /health endpoint"
-fi
-echo "PASS: HCC API gateway readiness probe uses the gateway-local /health endpoint"
-
-hcc_config="$(
- awk '
- BEGIN { RS = "---" }
- /name:[[:space:]]*host-context-controller-api-gateway/ { print; exit }
- ' "${CONFIGMAP}"
-)"
-
-if [[ -z "${hcc_config}" ]]; then
- fail "HCC API gateway nginx ConfigMap is missing"
-fi
-
-health_location="$(
- awk '
- /^[[:space:]]*location = \/health \{/ { in_location = 1 }
- in_location { print }
- in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }
- ' <<<"${hcc_config}"
-)"
-
-if [[ "${health_location}" != *'return 200'* ]] ||
- [[ "${health_location}" == *'proxy_pass'* ]]; then
- fail "HCC API gateway /health must remain a local nginx health response"
-fi
-echo "PASS: HCC API gateway /health remains local to nginx"
-
-ready_location="$(
- awk '
- /^[[:space:]]*location = \/ready \{/ { in_location = 1 }
- in_location { print }
- in_location && /^[[:space:]]*\}[[:space:]]*$/ { exit }
- ' <<<"${hcc_config}"
-)"
-
-if [[ "${ready_location}" != *'proxy_pass http://host_context_controller_upstream;'* ]]; then
- fail "HCC API gateway /ready must continue proxying the HCC readiness contract"
-fi
-echo "PASS: HCC API gateway /ready continues proxying HCC readiness"
-
-runtime_hold_function="$(
- sed -n '/^hcc_gateway_remains_ready_without_hcc() {$/,/^}$/p' "${BOOTSTRAP_E2E}"
-)"
-unavailable_probe_function="$(
- sed -n '/^hcc_gateway_ready_proxy_unavailable() {$/,/^}$/p' "${BOOTSTRAP_E2E}"
-)"
-gateway_pod_function="$(
- sed -n '/^running_hcc_gateway_pod() {$/,/^}$/p' "${BOOTSTRAP_E2E}"
-)"
-if [[ "${gateway_pod_function}" != *'ready_pod_name "$HCC_NS" "app=${HCC_GATEWAY_DEPLOY}"'* ]] ||
- [[ "${gateway_pod_function}" == *'running_pod_name'* ]]; then
- fail "gateway runtime proof must select a Ready pod through the shared E2E helper"
-fi
-echo "PASS: gateway runtime proof selects a Ready pod through the shared helper"
-
-if [[ "${runtime_hold_function}" != *'hcc_gateway_deployment_ready'* ]] ||
- [[ "${runtime_hold_function}" != *'hcc_gateway_local_health_ok'* ]] ||
- [[ "${runtime_hold_function}" != *'hcc_gateway_ready_proxy_unavailable'* ]] ||
- ! grep -Fq 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}"; then
- fail "bootstrap E2E must hold HCC unavailable across a full gateway probe-failure window"
-fi
-echo "PASS: bootstrap E2E proves gateway readiness remains local while HCC is unavailable"
-
-# Exercise the full conjunction with a bounded synthetic clock. Each required
-# observation must independently fail the hold; this kills a permissive
-# && -> || mutation without requiring a cluster or a real sleep.
-gateway_hold_case() (
- local failed_observation=$1 clock_file
- clock_file="$(mktemp "${TMPDIR:-/tmp}/hcc-gateway-clock.XXXXXX")"
- printf '%s' 0 >"$clock_file"
- trap 'rm -f "$clock_file"' EXIT
- HCC_NS=control-plane
- date() {
- local calls
- calls="$(<"$clock_file")"
- calls=$((calls + 1))
- printf '%s' "$calls" >"$clock_file"
- [ "$calls" -le 2 ] && printf '%s\n' 0 || printf '%s\n' 1
- }
- sleep() { :; }
- hcc_gateway_deployment_ready() {
- [ "$failed_observation" != deployment ]
- }
- hcc_gateway_local_health_ok() {
- [ "$failed_observation" != health ]
- }
- hcc_gateway_ready_proxy_unavailable() {
- [ "$failed_observation" != proxy ]
- }
- eval "${runtime_hold_function}"
- hcc_gateway_remains_ready_without_hcc 1
-)
-
-if gateway_hold_case none &&
- ! gateway_hold_case deployment &&
- ! gateway_hold_case health &&
- ! gateway_hold_case proxy; then
- echo "PASS: gateway hold requires every readiness, health, and proxy-unavailable observation"
-else
- fail "gateway hold can accept a failed readiness, health, or proxy observation"
-fi
-
-for unavailable_status in 502 503 504; do
- if (
- HCC_NS=control-plane
- running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }
- kctl() {
- printf ' HTTP/1.1 %s upstream unavailable\n' "${unavailable_status}"
- return 8
- }
- eval "${unavailable_probe_function}"
- hcc_gateway_ready_proxy_unavailable
- ); then
- echo "PASS: gateway hold accepts explicit upstream-unavailable HTTP ${unavailable_status}"
- else
- fail "gateway hold rejects expected upstream-unavailable HTTP ${unavailable_status}"
- fi
-done
-
-if (
- HCC_NS=control-plane
- running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }
- kctl() {
- printf '%s\n' 'error: unable to upgrade connection: container not found'
- return 1
- }
- eval "${unavailable_probe_function}"
- ! hcc_gateway_ready_proxy_unavailable
-); then
- echo "PASS: gateway hold rejects kubectl or container transport failure"
-else
- fail "gateway hold can mistake kubectl or container transport failure for upstream unavailability"
-fi
-
-if (
- HCC_NS=control-plane
- running_hcc_gateway_pod() { printf '%s\n' gateway-pod; }
- kctl() {
- printf '%s\n' 'healthy upstream response'
- return 0
- }
- eval "${unavailable_probe_function}"
- ! hcc_gateway_ready_proxy_unavailable
-); then
- echo "PASS: gateway hold rejects a successful proxied /ready response"
-else
- fail "gateway hold can accept a successful proxied /ready response as unavailable"
-fi
-
-hold_line="$(grep -nF 'hcc_gateway_remains_ready_without_hcc 20' "${BOOTSTRAP_E2E}" | cut -d: -f1)"
-restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "${BOOTSTRAP_E2E}" | cut -d: -f1)"
-recovery_line="$(grep -nF 'hcc_gateway_ready_proxy_recovers ||' "${BOOTSTRAP_E2E}" | cut -d: -f1)"
-if [[ -z "${hold_line}" || -z "${restart_line}" || -z "${recovery_line}" ]] ||
- (( hold_line >= restart_line || restart_line >= recovery_line )); then
- fail "gateway unavailable/ready/recovery assertions must surround the HCC restart"
-fi
-echo "PASS: bootstrap E2E proves proxied /ready failure and recovery in causal order"
diff --git a/scripts/tests/test-hcc-watch-recovery-gate.sh b/scripts/tests/test-hcc-watch-recovery-gate.sh
index 379d362e6..50104032d 100755
--- a/scripts/tests/test-hcc-watch-recovery-gate.sh
+++ b/scripts/tests/test-hcc-watch-recovery-gate.sh
@@ -1,39 +1,21 @@
#!/usr/bin/env bash
-# Values in the eval-based fixture harness are consumed by extracted gate
-# functions, which static shell analysis cannot follow.
-# shellcheck disable=SC2034
set -u
FAIL=0
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SCRIPT_DIR="${ROOT}/scripts/e2e"
-WATCH_GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"
-READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-readiness-bootstrap.sh"
-MCP_READINESS_GATE="${SCRIPT_DIR}/e2e-hcc-mcp-context-readiness.sh"
-HOST_BUNDLE_MEASURE="${SCRIPT_DIR}/measure-host-bundle-reconcile.sh"
-HOST_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"
-GATES=("$WATCH_GATE" "$READINESS_GATE" "$MCP_READINESS_GATE")
+GATE="${SCRIPT_DIR}/e2e-hcc-communicationchannel-watch-recovery.sh"
LOCK_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
LOG_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"
-FIXTURE_HELPER="${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
-HCC_K8S_CLIENT="${ROOT}/host-context-controller/src/k8sClient.ts"
-HCC_EGRESS_COORDINATOR="$(
- printf '%s' "${ROOT}/host-context-controller/src/externalEgressConvergenceCoordinator.ts"
-)"
-HCC_NETWORK_POLICY_RECONCILER="$(
- printf '%s' "${ROOT}/host-context-controller/src/networkPolicyReconciler.ts"
-)"
-WATCH_GATE_TARGET="$(sed -n '/^test-e2e-hcc-communicationchannel-watch-recovery:/,/^$/p' "${ROOT}/Makefile")"
MOCK_STATE_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-lock-test.XXXXXX")"
MOCK_LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/hcc-log-test.XXXXXX")"
-MOCK_PROFILE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hcc-profile-test.XXXXXX")"
rm -f "$MOCK_STATE_FILE"
-trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"; rm -rf "$MOCK_PROFILE_ROOT"' EXIT
+trap 'rm -f "$MOCK_STATE_FILE" "$MOCK_LOG_FILE"' EXIT
pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }
-for script in "${GATES[@]}" "$LOCK_HELPER" "$LOG_HELPER" "$FIXTURE_HELPER"; do
+for script in "$GATE" "$LOCK_HELPER" "$LOG_HELPER"; do
if bash -n "$script"; then
pass "$(basename "$script") has valid bash syntax"
else
@@ -41,1282 +23,12 @@ for script in "${GATES[@]}" "$LOCK_HELPER" "$LOG_HELPER" "$FIXTURE_HELPER"; do
fi
done
-# A node carrying a minikube label from another profile must not satisfy the
-# destructive-gate boundary. Each gate binds the label value to the explicit
-# context instead of accepting any non-null label.
-matching_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"branch-profile"}}}]}'
-mismatched_nodes='{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"other-profile"}}}]}'
-if jq -e --arg context branch-profile \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \
- <<<"$matching_nodes" >/dev/null &&
- ! jq -e --arg context branch-profile \
- 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $context)' \
- <<<"$mismatched_nodes" >/dev/null &&
- [ "$(grep -Fl 'minikube.k8s.io/name"] == $context' "${GATES[@]}" | wc -l | tr -d ' ')" = 3 ] &&
- ! grep -Fq 'minikube.k8s.io/name"] != null' "${GATES[@]}"; then
- pass "all destructive HCC gates bind the minikube node label to the explicit profile"
-else
- fail "an HCC gate can accept a minikube node owned by another profile"
-fi
-
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if [[ "$WATCH_GATE_TARGET" == *'E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)"'* ]] &&
- [[ "$WATCH_GATE_TARGET" == *'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT)'* ]]; then
- pass "CommunicationChannel gate target propagates exact pre-gate and profile ownership"
-else
- fail "CommunicationChannel gate target omits exact pre-gate or profile ownership"
-fi
-
-proxy_rollout_function="$(
- sed -n '/^print_hcc_proxy_rollout_diagnostics() {$/,/^}$/p' "$WATCH_GATE"
-)"
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if [[ "$proxy_rollout_function" == *'kctl get pods'* ]] &&
- [[ "$proxy_rollout_function" == *'--previous --tail=120'* ]] &&
- [[ "$proxy_rollout_function" == *'status.containerStatuses'* ]] &&
- [[ "$proxy_rollout_function" == *'// "unknown"'* ]] &&
- [[ "$proxy_rollout_function" != *'\\"unknown\\"'* ]] &&
- grep -Fq 'if ! kctl rollout status deployment "$HCC_DEPLOY"' "$WATCH_GATE" &&
- [ "$(grep -Fc 'print_hcc_proxy_rollout_diagnostics' "$WATCH_GATE")" = 2 ]; then
- pass "proxy rollout failure emits bounded HCC pod and previous-container diagnostics"
-else
- fail "proxy rollout failure can discard the distinguishing HCC startup evidence"
-fi
-
-if (
- HCC_NS=control-plane
- HCC_DEPLOY='host-context-controller'
- kctl() {
- if [ "$1" = get ] && [ "$2" = pods ] && [[ "$*" == *"-o jsonpath="* ]]; then
- printf '%s\n' hcc-pod
- elif [ "$1" = get ] && [ "$2" = pods ]; then
- printf '%s\n' '{"items":[{"metadata":{"name":"hcc-pod"},"status":{"phase":"Pending","containerStatuses":[{"name":"host-context-controller","ready":false,"restartCount":2,"state":{"waiting":{"reason":"CrashLoopBackOff"}},"lastState":{}}]}}]}'
- elif [ "$1" = logs ]; then
- printf '%s\n' 'bounded diagnostic log'
- else
- return 1
- fi
- }
- eval "$proxy_rollout_function"
- output="$(print_hcc_proxy_rollout_diagnostics 2>&1)"
- [[ "$output" == *"pod=hcc-pod phase=Pending"* ]] &&
- [[ "$output" != *"pod=null"* ]] &&
- [[ "$output" != *"phase=null"* ]]
-); then
- pass "proxy rollout diagnostics retain the parent pod identity and phase"
-else
- fail "proxy rollout diagnostics lose pod metadata inside container status iteration"
-fi
-
-# The proxy changes only the HCC client's target hostname. The HCC's
-# allow-k8s-api-egress policies must retain the Kubernetes-injected API IP,
-# which is the explicit fail-closed configuration seam for CIDR policy data.
-# shellcheck disable=SC2016
-if grep -Fq 'K8S_API_SERVICE_HOST="$(kctl exec deployment/"$HCC_DEPLOY"' "$WATCH_GATE" &&
- grep -Fq 'K8S_API_CIDR="${K8S_API_SERVICE_HOST}/32"' "$WATCH_GATE" &&
- grep -Fq -- '--arg api_cidr "$K8S_API_CIDR"' "$WATCH_GATE" &&
- grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS",value:$api_cidr' "$WATCH_GATE" &&
- grep -Fq 'allow-k8s-api-egress-${MCP_NS}' "$WATCH_GATE" &&
- grep -Fq '[ "$proxy_policy_cidrs" = "$K8S_API_CIDR" ]' "$WATCH_GATE" &&
- grep -Fq 'CONTEXT_MAPPER_K8S_API_CIDRS-' "$FIXTURE_HELPER"; then
- pass "proxy fault injection preserves an explicit valid API egress CIDR and restores it"
-else
- fail "proxy fault injection can turn a DNS hostname into an invalid API egress CIDR"
-fi
-
-if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' "$READINESS_GATE" &&
- [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$READINESS_GATE")" = 1 ] &&
- ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$READINESS_GATE" &&
- ! grep -Fq 'cluster_fingerprint_file=' "$READINESS_GATE" &&
- ! grep -Fq 'profile_env=' "$READINESS_GATE"; then
- pass "readiness gate delegates ownership, HEAD, profile, fingerprint, and gate proof once"
-else
- fail "readiness gate duplicates, bypasses, or splits the shared branch-owned proof"
-fi
-
-bootstrap_repair_function="$(
- sed -n '/^print_repair_instructions() {$/,/^}$/p' "$READINESS_GATE"
-)"
-mcp_repair_function="$(
- sed -n '/^print_repair_instructions() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-# Literal source-code assertions: every fail-closed repair path must restore
-# the captured replica count before treating rollout health as repair proof.
-# shellcheck disable=SC2016
-if [[ "$bootstrap_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&
- [[ "$bootstrap_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=180s'* ]] &&
- [[ "$mcp_repair_function" == *'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}'* ]] &&
- [[ "$mcp_repair_function" == *'rollout status deployment/${HCC_DEPLOY} --timeout=240s'* ]] &&
- grep -Fq 'scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}' "$WATCH_GATE" &&
- grep -Fq 'rollout status deployment/${HCC_DEPLOY} --timeout=180s' "$WATCH_GATE"; then
- pass "all HCC repair instructions restore the original replica count and verify rollout"
-else
- fail "an HCC repair path can leave the controller scaled to zero"
-fi
-
-fixture_gate_function="$(sed -n '/^require_branch_owned_hcc_gate() {$/,/^}$/p' "$FIXTURE_HELPER")"
-marker_get_count="$(
- grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' <<<"$fixture_gate_function"
-)"
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if [ "$marker_get_count" = 1 ] &&
- [[ "$fixture_gate_function" == *'-o json'* ]] &&
- [[ "$fixture_gate_function" == *'.data.worktreeId'* ]] &&
- [[ "$fixture_gate_function" == *'.data.gitHead'* ]] &&
- [[ "$fixture_gate_function" == *'<<<"$HCC_BRANCH_GATE_SYNC_MARKER"'* ]]; then
- pass "shared branch gate derives ownership and HEAD from one exported ConfigMap JSON snapshot"
-else
- fail "shared branch gate can split ownership and HEAD across marker reads"
-fi
-
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if [[ "$fixture_gate_function" == *'expected_branch="$(git -C "$HCC_BRANCH_GATE_REPO_ROOT" branch --show-current)"'* ]] &&
- [[ "$fixture_gate_function" == *'[ -n "$expected_branch" ]'* ]] &&
- [[ "$fixture_gate_function" == *'[ "${MINIKUBE_PROFILE:-}" = "$E2E_KUBECONTEXT" ]'* ]] &&
- [[ "$fixture_gate_function" == *'profile_env="${E2E_BRANCH_PROFILE_ENV:-'* ]] &&
- [[ "$fixture_gate_function" == *'profile_env_value PROFILE'* ]] &&
- [[ "$fixture_gate_function" == *'profile_env_value REPO_DIR'* ]] &&
- [[ "$fixture_gate_function" == *'profile_env_value BRANCH'* ]] &&
- [[ "$fixture_gate_function" != *'profile_env_value SHA_SHORT'* ]] &&
- [[ "$fixture_gate_function" == *'profile_env_value DIRTY'* ]] &&
- [[ "$fixture_gate_function" == *'cluster_fingerprint_file='* ]] &&
- [[ "$fixture_gate_function" == *'.data.clusterFingerprint'* ]] &&
- [[ "$fixture_gate_function" == *'.data.gate'* ]] &&
- [[ "$fixture_gate_function" == *'E2E_EXPECTED_PRE_GATE_GATE'* ]]; then
- pass "shared branch gate owns the complete profile, fingerprint, and expected-gate proof"
-else
- fail "shared branch gate omits a required profile, fingerprint, or expected-gate boundary"
-fi
-
-for gate in "${GATES[@]}"; do
- marker_reads="$(grep -Fc 'kctl get configmap clerum-pre-gate-sync-state' "$gate")"
- if [ "$marker_reads" = 0 ] &&
- ! grep -Fq 'profile_env=' "$gate" &&
- ! grep -Fq 'cluster_fingerprint_file=' "$gate" &&
- grep -Fq 'require_branch_owned_hcc_gate' "$gate"; then
- pass "$(basename "$gate") delegates its branch-owned proof to the shared authority"
- else
- fail "$(basename "$gate") duplicates or bypasses the shared branch-owned proof"
- fi
-done
-
-run_branch_gate_case() (
- local mode=$1
- local repo_root="${MOCK_PROFILE_ROOT}/repo"
- local context="clerum-codex-profile-test-1234abcd"
- local head="1111111111111111111111111111111111111111"
- local short_head="11111111"
- local branch="codex/profile-test"
- local fingerprint="2222222222222222222222222222222222222222"
- local marker_fingerprint="$fingerprint"
- local expected_gate="full"
- local marker_gate="$expected_gate"
- local minikube_profile="$context"
- local profile="$context"
- local profile_repo="$repo_root"
- local profile_branch="$branch"
- local profile_sha="$short_head"
- local profile_dirty=false
- local mock_worktree_dirty=""
- local worktree_id
- worktree_id="$(printf '%s' "$repo_root" | shasum | awk '{print $1}')"
-
- case "$mode" in
- success) ;;
- detached) branch="" ;;
- minikube-profile-missing) minikube_profile="" ;;
- minikube-profile) minikube_profile="clerum-other-profile-1234abcd" ;;
- profile-name) profile="clerum-other-profile-1234abcd" ;;
- profile-repo) profile_repo="${repo_root}-other" ;;
- profile-branch) profile_branch="codex/other" ;;
- reused-profile) profile_sha="aaaaaaaa" ;;
- profile-dirty) profile_dirty=true ;;
- dirty-worktree) mock_worktree_dirty=" M changed.ts" ;;
- fingerprint) marker_fingerprint="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ;;
- gate) marker_gate="targeted" ;;
- *) exit 98 ;;
- esac
-
- mkdir -p "${MOCK_PROFILE_ROOT}/state/${worktree_id}"
- printf '%s\n' "$fingerprint" >"${MOCK_PROFILE_ROOT}/state/${worktree_id}/cluster.sha"
- printf '%s\n' \
- "PROFILE=${profile}" \
- "REPO_DIR=${profile_repo}" \
- "BRANCH=${profile_branch}" \
- "SHA_SHORT=${profile_sha}" \
- "DIRTY=${profile_dirty}" >"${MOCK_PROFILE_ROOT}/profile.env"
-
- E2E_KUBECONTEXT="$context"
- MINIKUBE_PROFILE="$minikube_profile"
- E2E_BRANCH_PROFILE_ENV="${MOCK_PROFILE_ROOT}/profile.env"
- E2E_PRE_GATE_STATE_ROOT="${MOCK_PROFILE_ROOT}/state"
- E2E_EXPECTED_PRE_GATE_GATE="$expected_gate"
- HCC_NS=control-plane
- SCRIPT_DIR="${ROOT}/scripts/e2e"
- MOCK_MARKER="$(
- jq -cn --arg worktreeId "$worktree_id" --arg gitHead "$head" \
- --arg clusterFingerprint "$marker_fingerprint" --arg gate "$marker_gate" \
- '{data:{worktreeId:$worktreeId,gitHead:$gitHead,
- clusterFingerprint:$clusterFingerprint,gate:$gate}}'
- )"
-
- is_branch_scoped_e2e_context() { return 0; }
- die() { exit 97; }
- git() {
- local command=$3
- shift 3
- case "$command $*" in
- "rev-parse --show-toplevel") printf '%s\n' "$repo_root" ;;
- "status --porcelain --untracked-files=normal") printf '%s\n' "$mock_worktree_dirty" ;;
- "rev-parse HEAD") printf '%s\n' "$head" ;;
- "rev-parse --short=8 HEAD") printf '%s\n' "$short_head" ;;
- "branch --show-current") printf '%s\n' "$branch" ;;
- *) return 96 ;;
- esac
- }
- kctl() { printf '%s\n' "$MOCK_MARKER"; }
- # shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-fixture.sh
- source "$FIXTURE_HELPER"
- require_branch_owned_hcc_gate "$HCC_NS"
-)
-
-if run_branch_gate_case success; then
- pass "shared branch gate accepts matching branch-profile evidence"
-else
- fail "shared branch gate rejects matching branch-profile evidence"
-fi
-if run_branch_gate_case reused-profile; then
- pass "shared branch gate accepts a reused same-branch profile after an exact-head pre-gate sync"
-else
- fail "shared branch gate rejects a reused same-branch profile despite exact-head pre-gate evidence"
-fi
-for case_name in detached minikube-profile-missing minikube-profile profile-name profile-repo profile-branch \
- profile-dirty dirty-worktree fingerprint gate; do
- if run_branch_gate_case "$case_name" >/dev/null 2>&1; then
- fail "shared branch gate accepted mismatched ${case_name} evidence"
- else
- pass "shared branch gate rejects mismatched ${case_name} evidence"
- fi
-done
-
-retry_progress_function="$(
- sed -n '/^external_egress_retry_progress_is_observed() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-retry_query_function="$(
- sed -n '/^dns_retry_query_observed_since_schedule() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-if (
- NEW_HCC_POD=hcc-pod
- HCC_NS=control-plane
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- CONTEXT_NAME=e2e-context
- MOCK_HCC_LOG='
-[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout
-[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms
-'
- kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }
- eval "$retry_progress_function"
- external_egress_retry_progress_is_observed
-); then
- pass "DNS failure is attributed to the dedicated external-egress retry lane"
-else
- fail "the injected DNS hold cannot prove dedicated external-egress retry progress"
-fi
-if (
- NEW_HCC_POD=hcc-pod
- HCC_NS=control-plane
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- CONTEXT_NAME=e2e-context
- MOCK_HCC_LOG='
-[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout
-[K8s] Scheduling external egress retry 1/3 for McpServer "different-server" in 5000ms
-[K8s] unrelated message for McpServer "e2e-held-server"
-'
- kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }
- eval "$retry_progress_function"
- ! external_egress_retry_progress_is_observed
-); then
- pass "external-egress retry evidence cannot combine a peer retry with the fixture server marker"
-else
- fail "external-egress retry evidence can be assembled from different log lines or servers"
-fi
-if (
- NEW_HCC_POD=hcc-pod
- HCC_NS=control-plane
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- CONTEXT_NAME=e2e-context
- MOCK_HCC_LOG='
-[K8s] Initial external egress reconciliation failed for mcp-server/e2e-held-server; runtime reconciliation will stay blocked until retry succeeds: dns timeout
-[K8s] Scheduling external egress retry 1/3 for McpServer "e2e-held-server" in 5000ms
-'
- for ((line = 0; line < 4096; line++)); do
- MOCK_HCC_LOG+=$'\n[K8s] unrelated high-volume log line'
- done
- kctl() { printf '%s\n' "$MOCK_HCC_LOG"; }
- eval "$retry_progress_function"
- external_egress_retry_progress_is_observed
-); then
- pass "an early matching retry line survives a high-volume log buffer without SIGPIPE ambiguity"
-else
- fail "a valid early retry marker is lost in a high-volume log buffer"
-fi
-if (
- DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3
- dns_hold_count() { printf '%s\n' 4; }
- eval "$retry_query_function"
- dns_retry_query_observed_since_schedule
-); then
- pass "a post-schedule DNS query proves the dedicated retry executed"
-else
- fail "a fresh DNS query after scheduling does not prove retry execution"
-fi
-if (
- DNS_HOLD_COUNT_AT_RETRY_SCHEDULE=3
- dns_hold_count() { printf '%s\n' 3; }
- eval "$retry_query_function"
- ! dns_retry_query_observed_since_schedule
-); then
- pass "a pre-schedule DNS query cannot satisfy retry execution"
-else
- fail "stale DNS evidence can masquerade as retry execution"
-fi
-if grep -Fq '{name:"timeout",value:"30"}' "$MCP_READINESS_GATE" &&
- grep -Fq '{name:"attempts",value:"1"}' "$MCP_READINESS_GATE"; then
- pass "the DNS fixture disables resolver retransmits that could counterfeit retry execution"
-else
- fail "the DNS fixture can mistake a resolver retransmit for a dedicated retry"
-fi
-
-# Full marker inventory: every runtime log marker any HCC gate consumes, paired
-# with the producer statement that emits it. Both sides are matched as fixed
-# source text, never as a rendered string: a template literal is pinned by its
-# fixed prefix plus the variable that supplies the dynamic identity, so a
-# marker that only ever exists rendered (`${config.hostNamespace}`,
-# `${reason}`, `${type}`, `${snapshot.channels.length}`) still resolves without
-# reporting false drift. Matching the consumer side too means a rename on
-# either side breaks the pair instead of silently disarming a live gate: the
-# marker no longer matches any log line, the count stays zero, and the gate
-# passes on evidence it never observed.
-# Entries are flat 4-tuples: consumer file, consumer text, producer file,
-# producer text.
-# shellcheck disable=SC2016
-marker_bindings=(
- "$WATCH_GATE" 'CommunicationChannel watch ended;'
- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'
-
- "$WATCH_GATE" 'Starting CommunicationChannel watch'
- "$HCC_K8S_CLIENT" '[K8s] Starting CommunicationChannel watch'
-
- "$WATCH_GATE" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'
- "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'
-
- "$WATCH_GATE" 'cache recovery failed;'
- "$HCC_K8S_CLIENT" 'CommunicationChannel cache recovery failed; stateless lifecycle remains held active'
-
- "$WATCH_GATE" 'Listing all CommunicationChannels in namespace ${CHANNEL_NS}'
- "$HCC_K8S_CLIENT" 'Listing all CommunicationChannels in namespace ${config.channelsNamespace}'
-
- "$WATCH_GATE" 'Listing all Hosts in namespace ${HOST_NS}'
- "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'
-
- "$WATCH_GATE" 'Completed Host reconciliation after (CommunicationChannel recovery|Host watch recovery convergence)$'
- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
-
- "$WATCH_GATE" 'CommunicationChannel watch event: DELETED for ${FIXTURE_CHANNEL}'
- "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'
-
- "$WATCH_GATE" 'CommunicationChannel watch event: ADDED for ${FIXTURE_CHANNEL}'
- "$HCC_K8S_CLIENT" 'CommunicationChannel watch event: ${type} for ${cc.name}'
-
- "$LOG_HELPER" 'CommunicationChannel watch ended;'
- "$HCC_K8S_CLIENT" 'CommunicationChannel watch ended; holding stateless lifecycle active'
-
- "$LOG_HELPER" 'Recovered [0-9]+ CommunicationChannel\(s\) into cache'
- "$HCC_K8S_CLIENT" 'Recovered ${snapshot.channels.length} CommunicationChannel(s) into cache'
-
- "$LOG_HELPER" 'Listing all Hosts in namespace '
- "$HCC_K8S_CLIENT" 'Listing all Hosts in namespace ${config.hostNamespace}'
-
- "$LOG_HELPER" 'Host(s) for lifecycle after CommunicationChannel recovery'
- "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) for lifecycle after ${reason}'
-
- "$LOG_HELPER" 'Host(s) after Host watch recovery convergence'
- "$HCC_K8S_CLIENT" 'Reconciling ${hosts.length} Host(s) after ${reason}'
-
- "$LOG_HELPER" 'Completed Host reconciliation after CommunicationChannel recovery$'
- "$HCC_K8S_CLIENT" "requestHostFleetReconcile('CommunicationChannel recovery'"
-
- "$LOG_HELPER" 'Completed Host reconciliation after Host watch recovery convergence$'
- "$HCC_K8S_CLIENT" "convergenceReason = 'Host watch recovery convergence'"
-
- "$LOG_HELPER" 'Periodic resync'
- "$HCC_K8S_CLIENT" "requestHostFleetReconcile('Periodic resync'"
-
- "$READINESS_GATE" "START_MARKER='Starting initial Host background convergence'"
- "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'
-
- "$READINESS_GATE" "COMPLETE_MARKER='Completed Host reconciliation after initial Host reconciliation'"
- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
-
- "$READINESS_GATE" "FAIL_MARKER='Host reconciliation after initial Host reconciliation failed'"
- "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'
-
- "$READINESS_GATE" 'after initial Host reconciliation'
- "$HCC_K8S_CLIENT" "'initial Host reconciliation',"
-
- "$HOST_BUNDLE_MEASURE" "HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'"
- "$HCC_K8S_CLIENT" 'Starting initial Host background convergence...'
-
- "$HOST_BUNDLE_MEASURE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"
- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
-
- "$HOST_BUNDLE_MEASURE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"
- "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'
-
- "$HOST_STORM_GATE" "HCC_PASS_COMPLETED_MARKER='Completed Host reconciliation after initial Host reconciliation'"
- "$HCC_K8S_CLIENT" 'Completed Host reconciliation after ${reason}'
-
- "$HOST_STORM_GATE" "HCC_PASS_FAILED_MARKER='Host reconciliation after initial Host reconciliation failed'"
- "$HCC_K8S_CLIENT" 'Host reconciliation after ${reason} failed:'
-
- "$MCP_READINESS_GATE" 'Initial external egress reconciliation failed for ${MCP_NS}/${MCP_NAME};'
- "$HCC_EGRESS_COORDINATOR" 'Initial external egress reconciliation failed for ${key};'
-
- "$MCP_READINESS_GATE" 'Scheduling external egress retry '
- "$HCC_EGRESS_COORDINATOR" 'Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}'
-
- "$MCP_READINESS_GATE" 'for McpServer \"${MCP_NAME}\"'
- "$HCC_EGRESS_COORDINATOR" 'for McpServer "${this.retryIntents.get(key)!.server.name}"'
-
- "$MCP_READINESS_GATE" 'Context watch event: MODIFIED for ${CONTEXT_NAME}'
- "$HCC_K8S_CLIENT" 'Context watch event: ${type} for ${context.name}'
-
- "$MCP_READINESS_GATE" '[NetPol] Reconciling context \"${CONTEXT_ID}\" — allowed servers: []'
- "$HCC_NETWORK_POLICY_RECONCILER" 'Reconciling context "${contextId}" — allowed servers: [${allowedServers.join('
-
- "$MCP_READINESS_GATE" "NETPOL_INITIAL_FAILED_MARKER='Initial NetworkPolicy background reconciliation failed:'"
- "$HCC_K8S_CLIENT" 'Initial NetworkPolicy background reconciliation failed:'
-
- "$MCP_READINESS_GATE" "NETPOL_ADDITIVE_FAILED_MARKER='Initial NetworkPolicy post-certification additive reconciliation failed:'"
- "$HCC_K8S_CLIENT" 'Initial NetworkPolicy post-certification additive reconciliation failed:'
-
- "$MCP_READINESS_GATE" "NETPOL_RETRY_SCHEDULED_MARKER='Scheduling initial NetworkPolicy background convergence retry'"
- "$HCC_K8S_CLIENT" 'Scheduling initial ${lane} background convergence retry'
-
- "$MCP_READINESS_GATE" "NETPOL_CONTEXT_FAILED_MARKER='NetworkPolicy reconciliation failed for context '"
- "$HCC_K8S_CLIENT" 'NetworkPolicy reconciliation failed for context ${context.name}:'
-)
-marker_drift=""
-marker_binding_count=0
-for ((marker_index = 0; marker_index < ${#marker_bindings[@]}; marker_index += 4)); do
- consumer_file="${marker_bindings[marker_index]}"
- consumer_marker="${marker_bindings[marker_index + 1]}"
- producer_file="${marker_bindings[marker_index + 2]}"
- producer_marker="${marker_bindings[marker_index + 3]}"
- marker_binding_count=$((marker_binding_count + 1))
- # `grep -Fq -- ""` matches every file, so an empty marker would disarm this
- # pair while still reporting a pass. Refuse it instead of searching for it.
- if [ -z "$consumer_marker" ] || [ -z "$producer_marker" ]; then
- marker_drift+=" marker pair at index ${marker_index} has an empty marker"$'\n'
- continue
- fi
- if ! grep -Fq -- "$consumer_marker" "$consumer_file"; then
- marker_drift+=" consumer $(basename "$consumer_file") no longer reads: ${consumer_marker}"$'\n'
- fi
- if ! grep -Fq -- "$producer_marker" "$producer_file"; then
- marker_drift+=" producer $(basename "$producer_file") no longer emits: ${producer_marker}"$'\n'
- fi
-done
-# The stride loop consumes 4 elements per pair, so an array whose length is not
-# a multiple of 4 silently shifts every entry after the edit and leaves the tail
-# unset. Assert the arity explicitly rather than relying on the pair count to
-# happen to notice.
-if [ $((${#marker_bindings[@]} % 4)) -eq 0 ] &&
- [ "$marker_binding_count" -eq 35 ] &&
- [ -z "$marker_drift" ]; then
- pass "all ${marker_binding_count} consumed HCC log markers stay bound to their runtime producers"
-else
- printf '%s' "$marker_drift" >&2
- fail "a consumed HCC log marker drifted from its runtime producer (${marker_binding_count} pairs checked)"
-fi
-
-# Completeness fence: every static single-quoted *_MARKER defined in the e2e
-# gates MUST be inventoried above as a consumer_marker, so a NEW marker cannot be
-# born outside the drift fence (the pinned count only catches removals). The scan
-# covers both the gates themselves and the _lib/ helpers they source — a marker
-# born in a sourced helper is the same escape class. Scoped to single-quoted
-# literals — a dynamic id like C_MARKER="refusal-c-${RUN_ID}" (double-quoted, a
-# per-run test id, not a cross-file HCC log marker) is not the class this fence
-# covers and is excluded by the quote style, not a hard-coded allow-list.
-marker_is_inventoried() {
- local needle="$1" i
- for ((i = 1; i < ${#marker_bindings[@]}; i += 4)); do
- [ "${marker_bindings[i]}" = "$needle" ] && return 0
- done
- return 1
-}
-uninventoried_markers=""
-while IFS= read -r marker_def; do
- [ -n "$marker_def" ] || continue
- marker_is_inventoried "$marker_def" ||
- uninventoried_markers+=" ${marker_def}"$'\n'
-done < <(
- grep -rhoE "^[[:space:]]*(readonly[[:space:]]+)?[A-Z_]+_MARKER='[^']*'" \
- "${SCRIPT_DIR}"/*.sh "${SCRIPT_DIR}"/_lib/*.sh |
- sed -E "s/^[[:space:]]*(readonly[[:space:]]+)?//"
-)
-if [ -z "$uninventoried_markers" ]; then
- pass "every single-quoted *_MARKER defined in the e2e gates is inventoried in the drift fence"
-else
- printf '%s' "$uninventoried_markers" >&2
- fail "an e2e gate defines a *_MARKER absent from marker_bindings (drift fence would miss it)"
-fi
-
-fixture_mcp_runtime_absent_function="$(
- sed -n '/^fixture_mcp_runtime_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-hcc_kubernetes_readiness_function="$(
- sed -n '/^hcc_kubernetes_readiness_is_exact() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-external_policy_protocol_function="$(
- sed -n '/^external_egress_policy_converged_with_protocol() {$/,/^}$/p' \
- "$MCP_READINESS_GATE"
-)"
-hcc_ready_after_revoke_function="$(
- sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-ready_probe_function="$(
- sed -n '/^probe_new_hcc_ready_endpoint() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-
-run_ready_probe_failure_case() (
- local expected=$1
- MOCK_READY_PROBE_OUTPUT=$2
- NEW_HCC_POD=hcc-pod
- HCC_NS=control-plane
- HCC_PORT=8080
- HCC_READY_PROBE_DIAGNOSTIC=""
- kctl() {
- printf '%s\n' "$MOCK_READY_PROBE_OUTPUT"
- return 1
- }
- eval "$ready_probe_function"
- ! probe_new_hcc_ready_endpoint &&
- [ "$HCC_READY_PROBE_DIAGNOSTIC" = "HCC_READY_PROBE category=${expected}" ]
-)
-
-if run_ready_probe_failure_case timeout 'HCC_READY_PROBE category=timeout' &&
- run_ready_probe_failure_case transport 'HCC_READY_PROBE category=transport' &&
- run_ready_probe_failure_case 'http-status status=503' \
- 'HCC_READY_PROBE category=http-status status=503' &&
- run_ready_probe_failure_case json-ready-false \
- 'HCC_READY_PROBE category=json-ready-false'; then
- pass "ready probe preserves distinct timeout, transport, HTTP, and ready=false diagnostics"
-else
- fail "ready probe collapses a failure boundary into an indiagnostic boolean"
-fi
-if (
- NEW_HCC_POD=hcc-pod
- HCC_NS=control-plane
- HCC_PORT=8080
- HCC_READY_PROBE_DIAGNOSTIC="stale"
- kctl() { return 0; }
- eval "$ready_probe_function"
- probe_new_hcc_ready_endpoint && [ -z "$HCC_READY_PROBE_DIAGNOSTIC" ]
-); then
- pass "successful ready probe clears stale failure diagnostics"
-else
- fail "successful ready probe retains a misleading prior diagnostic"
-fi
-
-# H6 gate-fix mutation (E4): the clean-window NetworkPolicy negative must FAIL
-# against an injection-window log (moving it inside the injected window would
-# false-fail — the scoping is load-bearing), PASS against a clean log, and fail
-# closed when kubectl errors. The gate itself can't run (minikube churn), so
-# this statically pins the predicate.
-clean_window_netpol_negatives_function="$(
- sed -n '/^clean_window_netpol_log_negatives_hold() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-clean_window_marker_defs="$(grep -E "^NETPOL_[A-Z_]+_MARKER='" "$MCP_READINESS_GATE")"
-run_clean_window_negative_case() (
- local mode=$1
- CONTEXT_NAME=alpha
- HCC_NS=control-plane
- eval "$clean_window_marker_defs"
- case $mode in
- injected)
- kctl() {
- printf '%s\n' \
- '[K8s] Initial NetworkPolicy post-certification additive reconciliation failed: dns timeout' \
- '[K8s] Scheduling initial NetworkPolicy background convergence retry 1 in 5000ms'
- } ;;
- clean) kctl() { printf '%s\n' '[K8s] NetworkPolicy convergence certified'; } ;;
- api-error) kctl() { return 1; } ;;
- esac
- eval "$clean_window_netpol_negatives_function"
- clean_window_netpol_log_negatives_hold hcc-pod
-)
-if ! run_clean_window_negative_case injected &&
- run_clean_window_negative_case clean &&
- ! run_clean_window_negative_case api-error; then
- pass "clean-window NetworkPolicy negative fails inside the injected window, passes clean, and fails closed"
-else
- fail "clean-window NetworkPolicy negative does not distinguish injected-window failures or fails open"
-fi
-
-# Static ordering fence: a clean-window negative invocation must exist in the
-# MAIN BODY (after `trap cleanup EXIT`) and before the DNS mutation. cleanup()
-# is defined earlier in the file than the mutation but runs on EXIT, so a plain
-# "first line < mutation line" test would be satisfied by the cleanup-window
-# invocation alone; excluding everything at or before the trap line pins the
-# pre-injection (window A) invocation specifically.
-trap_line="$(grep -n '^trap cleanup EXIT' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"
-dns_mutation_line="$(grep -n '^HCC_MUTATED=1' "$MCP_READINESS_GATE" | head -1 | cut -d: -f1)"
-body_clean_window_negative_line="$(
- grep -n 'clean_window_netpol_log_negatives_hold "' "$MCP_READINESS_GATE" |
- awk -F: -v t="${trap_line:-0}" -v m="${dns_mutation_line:-0}" \
- '$1 > t && $1 < m { print $1; exit }'
-)"
-if [ -n "$body_clean_window_negative_line" ]; then
- pass "clean-window NetworkPolicy negative runs in the body before the DNS fault injection"
-else
- fail "clean-window NetworkPolicy negative is missing from the pre-injection body window"
-fi
-if (
- NEW_HCC_POD=hcc-pod
- HCC_NS=control-plane
- HCC_PORT=8080
- HCC_READY_PROBE_DIAGNOSTIC=""
- kctl() {
- printf '%s\n' 'untrusted kubectl output containing super-secret material'
- return 1
- }
- eval "$ready_probe_function"
- ! probe_new_hcc_ready_endpoint &&
- [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=kubectl-exec' ]
-); then
- pass "ready probe replaces untrusted exec output with a bounded sanitized category"
-else
- fail "ready probe can leak untrusted exec output into diagnostics"
-fi
-# Literal source-code assertions guarantee one request and bounded categories,
-# without logging the response body, headers, URL, or transport error text.
-if [ "$(grep -Fc 'kctl exec' <<<"$ready_probe_function")" = 1 ] &&
- [[ "$ready_probe_function" == *'finish(4, "timeout")'* ]] &&
- [[ "$ready_probe_function" == *'finish(5, "transport")'* ]] &&
- [[ "$ready_probe_function" == *'finish(3, "http-status"'* ]] &&
- [[ "$ready_probe_function" == *'finish(3, "json-ready-false")'* ]] &&
- [[ "$ready_probe_function" == *"printf -v HCC_READY_PROBE_DIAGNOSTIC '%.240s'"* ]] &&
- [[ "$ready_probe_function" != *'console.error(body'* ]] &&
- [[ "$ready_probe_function" != *'error.message'* ]]; then
- pass "ready probe uses one request and emits only bounded non-secret failure categories"
-else
- fail "ready probe duplicates requests or exposes an unbounded diagnostic surface"
-fi
-
-if (
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- MCP_SERVER_LABEL=clerum.io/mcp-server
- POLICY_TYPE_LABEL=clerum.io/policy-type
- fixture_runtime_absent() { return 0; }
- kctl() { return 0; }
- eval "$fixture_mcp_runtime_absent_function"
- fixture_mcp_runtime_absent
-); then
- pass "fixture absence predicate accepts an absent runtime and external-egress policy"
-else
- fail "fixture absence predicate rejects an actually absent runtime"
-fi
-if (
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- MCP_SERVER_LABEL=clerum.io/mcp-server
- POLICY_TYPE_LABEL=clerum.io/policy-type
- fixture_runtime_absent() { return 0; }
- kctl() { printf '%s\n' 'networkpolicy.networking.k8s.io/stale-policy'; }
- eval "$fixture_mcp_runtime_absent_function"
- ! fixture_mcp_runtime_absent
-); then
- pass "fixture absence predicate rejects a surviving external-egress policy"
-else
- fail "fixture absence predicate can hide a surviving external-egress policy"
-fi
-if (
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- MCP_SERVER_LABEL=clerum.io/mcp-server
- POLICY_TYPE_LABEL=clerum.io/policy-type
- fixture_runtime_absent() { return 1; }
- kctl() { return 0; }
- eval "$fixture_mcp_runtime_absent_function"
- ! fixture_mcp_runtime_absent
-); then
- pass "fixture absence predicate rejects a surviving MCP runtime"
-else
- fail "fixture absence predicate can hide a surviving MCP runtime"
-fi
-
-if (
- NEW_HCC_POD=hcc-pod
- HCC_UID=pod-uid
- HCC_NS=control-plane
- HCC_DEPLOY='host-context-controller'
- kctl() {
- case "$2" in
- pod)
- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
- ;;
- deployment)
- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
- ;;
- *) return 1 ;;
- esac
- }
- eval "$hcc_kubernetes_readiness_function"
- hcc_kubernetes_readiness_is_exact
-); then
- pass "exact Kubernetes readiness accepts one current Ready HCC identity"
-else
- fail "exact Kubernetes readiness rejects current Ready HCC evidence"
-fi
-# Kubelet reports an indeterminate pod as Ready=Unknown, not Ready=False. Only
-# an Unknown fixture separates the exact `== "True"` readiness contract from a
-# weakened `!= "False"` one, so this red case pins the exact comparison instead
-# of the single status value both spellings already reject. Every other field is
-# healthy, which leaves the Ready condition as the only reason to refuse.
-if (
- NEW_HCC_POD=hcc-pod
- HCC_UID=pod-uid
- HCC_NS=control-plane
- HCC_DEPLOY='host-context-controller'
- kctl() {
- case "$2" in
- pod)
- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
- ;;
- deployment)
- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
- ;;
- *) return 1 ;;
- esac
- }
- eval "$hcc_kubernetes_readiness_function"
- ! hcc_kubernetes_readiness_is_exact
-); then
- pass "exact Kubernetes readiness rejects a current pod whose Ready condition is Unknown"
-else
- fail "exact Kubernetes readiness can accept an indeterminate HCC pod"
-fi
-if (
- NEW_HCC_POD=hcc-pod
- HCC_UID=pod-uid
- HCC_NS=control-plane
- HCC_DEPLOY='host-context-controller'
- kctl() {
- case "$2" in
- pod)
- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"False"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
- ;;
- deployment)
- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
- ;;
- *) return 1 ;;
- esac
- }
- eval "$hcc_kubernetes_readiness_function"
- ! hcc_kubernetes_readiness_is_exact
-); then
- pass "exact Kubernetes readiness rejects a current pod whose Ready condition is False"
-else
- fail "exact Kubernetes readiness can accept a non-Ready HCC pod"
-fi
-# The container-level readiness clause has its own red case: a pod whose Ready
-# condition is True while the host-context-controller container is not ready
-# must still be refused, so deleting or weakening that clause cannot pass.
-if (
- NEW_HCC_POD=hcc-pod
- HCC_UID=pod-uid
- HCC_NS=control-plane
- HCC_DEPLOY='host-context-controller'
- kctl() {
- case "$2" in
- pod)
- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":false}]}}'
- ;;
- deployment)
- printf '%s\n' '{"metadata":{"generation":4},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
- ;;
- *) return 1 ;;
- esac
- }
- eval "$hcc_kubernetes_readiness_function"
- ! hcc_kubernetes_readiness_is_exact
-); then
- pass "exact Kubernetes readiness rejects a Ready pod whose HCC container is not ready"
-else
- fail "exact Kubernetes readiness can accept an unready host-context-controller container"
-fi
-if (
- NEW_HCC_POD=hcc-pod
- HCC_UID=pod-uid
- HCC_NS=control-plane
- HCC_DEPLOY='host-context-controller'
- kctl() {
- case "$2" in
- pod)
- printf '%s\n' '{"metadata":{"uid":"pod-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"host-context-controller","ready":true}]}}'
- ;;
- deployment)
- printf '%s\n' '{"metadata":{"generation":5},"spec":{"replicas":1},"status":{"observedGeneration":4,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1,"unavailableReplicas":0}}'
- ;;
- *) return 1 ;;
- esac
- }
- eval "$hcc_kubernetes_readiness_function"
- ! hcc_kubernetes_readiness_is_exact
-); then
- pass "exact Kubernetes readiness rejects a stale Deployment generation"
-else
- fail "exact Kubernetes readiness can accept stale Deployment status"
-fi
-
-if (
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- MCP_SERVER_LABEL=clerum.io/mcp-server
- POLICY_TYPE_LABEL=clerum.io/policy-type
- MANAGED_BY_LABEL=app.kubernetes.io/managed-by
- MANAGED_BY_VALUE='host-context-controller'
- EXTERNAL_EGRESS_CIDR=203.0.113.10/32
- EXTERNAL_EGRESS_PORT=443
- kctl() {
- printf '%s\n' '{"items":[{"metadata":{"labels":{"app.kubernetes.io/managed-by":"host-context-controller","clerum.io/policy-type":"external-egress","clerum.io/mcp-server":"e2e-held-server"}},"spec":{"podSelector":{"matchLabels":{"clerum.io/mcp-server":"e2e-held-server"}},"policyTypes":["Egress"],"egress":[{"ports":[{"port":443,"protocol":"UDP"}],"to":[{"ipBlock":{"cidr":"203.0.113.10/32"}}]}]}}]}'
- }
- eval "$external_policy_protocol_function"
- external_egress_policy_converged_with_protocol UDP &&
- ! external_egress_policy_converged_with_protocol TCP
-); then
- pass "external-egress convergence binds the exact current protocol"
-else
- fail "external-egress convergence can accept a stale protocol"
-fi
-
-mcpserver_current_ready_function="$(
- sed -n '/^mcpserver_current_ready() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-mcpserver_current_ready_absent_function="$(
- sed -n '/^mcpserver_current_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-fixture_runtime_and_ready_absent_function="$(
- sed -n '/^fixture_runtime_and_ready_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-fixture_runtime_converged_function="$(
- sed -n '/^fixture_runtime_converged_with_protocol() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-peer_fleet_converged_function="$(
- sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-fixture_converged_function="$(
- sed -n '/^fixture_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-if (
- MCP_NS=mcp-server
- GENERATION=7
- OBSERVED_GENERATION=7
- kctl() {
- printf '{"metadata":{"generation":%s},"status":{"conditions":[{"type":"Ready","status":"True","observedGeneration":%s}]}}\n' \
- "$GENERATION" "$OBSERVED_GENERATION"
- }
- eval "$mcpserver_current_ready_function"
- eval "$mcpserver_current_ready_absent_function"
- mcpserver_current_ready current &&
- ! mcpserver_current_ready_absent current &&
- OBSERVED_GENERATION=6 &&
- ! mcpserver_current_ready stale &&
- mcpserver_current_ready_absent stale
-) &&
- [[ "$fixture_runtime_and_ready_absent_function" == *'mcpserver_current_ready_absent "$MCP_NAME"'* ]] &&
- [[ "$fixture_runtime_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]] &&
- [[ "$peer_fleet_converged_function" == *'mcpserver_current_ready "$server"'* ]] &&
- [[ "$fixture_converged_function" == *'mcpserver_current_ready "$MCP_NAME"'* ]]; then
- pass "McpServer runtime predicates accept only Ready status observed for the current generation"
-else
- fail "McpServer runtime predicates can accept stale Ready status or bypass the canonical check"
-fi
-
-if (
- MCP_NS=mcp-server
- MCP_NAME=e2e-held-server
- fixture_runtime_absent() { return 0; }
- kctl() { return 1; }
- eval "$mcpserver_current_ready_absent_function"
- eval "$fixture_runtime_and_ready_absent_function"
- ! fixture_runtime_and_ready_absent
-); then
- pass "McpServer Ready absence fails closed when the API read is unavailable"
-else
- fail "an unavailable McpServer API read can masquerade as Ready absence"
-fi
-
-if (
- probe_new_hcc_ready_endpoint() { return 0; }
- external_egress_policy_absent() { return 0; }
- eval "$hcc_ready_after_revoke_function"
- hcc_ready_after_stale_policy_revoked
-); then
- pass "divergent DNS hold permits global readiness only after stale-policy revocation"
-else
- fail "divergent DNS hold rejects globally safe readiness evidence"
-fi
-if (
- HCC_READY_PROBE_DIAGNOSTIC=""
- probe_new_hcc_ready_endpoint() { return 0; }
- external_egress_policy_absent() { return 1; }
- eval "$hcc_ready_after_revoke_function"
- rc=0
- hcc_ready_after_stale_policy_revoked || rc=$?
- # A 200 observed WITH the stale policy present is a fatal ordering violation
- # (rc=2), not a benign retryable "not ready" (rc=1).
- [ "$rc" -eq 2 ] &&
- [ "$HCC_READY_PROBE_DIAGNOSTIC" = 'HCC_READY_PROBE category=ready-200-with-stale-policy-present' ]
-); then
- pass "global readiness evidence flags a surviving divergent policy as a fatal rc=2 ordering violation"
-else
- fail "global readiness can self-confirm (or only soft-fail) while a divergent policy survives"
-fi
-# The loop must PROPAGATE the fatal rc=2 immediately, not retry until a later
-# clean poll turns the violation into a false PASS (the round-2 bug).
-wait_until_fast_function="$(
- sed -n '/^wait_until_fast() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-if (
- HCC_READY_PROBE_DIAGNOSTIC=""
- POLL=0
- probe_new_hcc_ready_endpoint() { return 0; }
- # Stale policy present for the first polls, revoked only later. The buggy
- # retry loop would poll past the fatal first observation and PASS here.
- external_egress_policy_absent() { POLL=$((POLL + 1)); [ "$POLL" -gt 4 ]; }
- eval "$hcc_ready_after_revoke_function"
- eval "$wait_until_fast_function"
- rc=0
- wait_until_fast 10 "rc=2 propagation" hcc_ready_after_stale_policy_revoked || rc=$?
- # rc must be the fatal 2, and it must have stopped inside the first predicate
- # poll (which calls external_egress_policy_absent twice) — POLL<3 proves no
- # retry occurred.
- [ "$rc" -eq 2 ] && [ "$POLL" -lt 3 ]
-); then
- pass "wait_until_fast propagates a fatal rc=2 on the first poll instead of retrying to a false PASS"
-else
- fail "wait_until_fast retried past a fatal ordering violation to a later clean poll"
-fi
-
-# M6: the third contract signal adjudicates a real /metrics scrape. Extract the
-# PURE predicate (the wrapper does kctl exec, untestable here) and feed it
-# synthetic scrapes. The family-name literals are exactly what a metric rename
-# would break — a rename leaves the vi.fn() unit tests green but must turn this
-# and the real gate red.
-safety_pass_metrics_predicate="$(
- sed -n '/^safety_pass_metrics_are_certified() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-[ -n "$safety_pass_metrics_predicate" ] ||
- fail "could not extract safety_pass_metrics_are_certified from the readiness gate"
-metrics_with_revocation='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1
-clerum_hcc_networkpolicy_safety_pass_policies_total{operation="listed"} 5
-clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 1'
-metrics_without_family='clerum_hcc_some_other_family_total 3'
-metrics_zero_revoked='clerum_hcc_networkpolicy_safety_pass_duration_seconds_count 1
-clerum_hcc_networkpolicy_safety_pass_policies_total{operation="revoked"} 0'
-if (
- eval "$safety_pass_metrics_predicate"
- safety_pass_metrics_are_certified "$metrics_with_revocation"
-); then
- pass "safety_pass metrics predicate certifies a scrape with >=1 revoked policy"
-else
- fail "safety_pass metrics predicate rejected a valid certifying scrape"
-fi
-if (
- eval "$safety_pass_metrics_predicate"
- ! safety_pass_metrics_are_certified "$metrics_without_family"
-); then
- pass "safety_pass metrics predicate rejects a scrape missing the duration family (rename guard)"
-else
- fail "safety_pass metrics predicate passed with the safety_pass family absent"
-fi
-if (
- eval "$safety_pass_metrics_predicate"
- ! safety_pass_metrics_are_certified "$metrics_zero_revoked"
-); then
- pass "safety_pass metrics predicate rejects a scrape with zero revoked policies"
-else
- fail "safety_pass metrics predicate passed with zero policies revoked"
-fi
-if (
- probe_new_hcc_ready_endpoint() { return 1; }
- external_egress_policy_absent() { return 0; }
- eval "$hcc_ready_after_revoke_function"
- ! hcc_ready_after_stale_policy_revoked
-); then
- pass "stale-policy revocation alone cannot substitute for global readiness"
-else
- fail "divergent hold evidence can pass without HCC readiness"
-fi
-
-baseline_policy_snapshot_function="$(
- sed -n '/^baseline_policy_snapshot() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-baseline_policies_unchanged_function="$(
- sed -n '/^baseline_policies_unchanged() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-valid_baseline_policies='{"items":[
- {"metadata":{"name":"deny-all-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"default-deny"}},"spec":{"podSelector":{},"policyTypes":["Ingress","Egress"]}},
- {"metadata":{"name":"allow-dns-egress-mcp-server","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"infrastructure"}},"spec":{"podSelector":{},"policyTypes":["Egress"],"egress":[{"ports":[{"port":53,"protocol":"UDP"},{"port":53,"protocol":"TCP"}]}]}},
- {"metadata":{"name":"allow-host-context-controller-api","labels":{"clerum.io/managed-by":"host-context-controller","clerum.io/policy-type":"allow-api"}},"spec":{"podSelector":{"matchLabels":{"app":"host-context-controller"}},"policyTypes":["Ingress"]}}
-]}'
-if (
- MCP_NS=mcp-server
- MANAGED_BY_VALUE='host-context-controller'
- MOCK_POLICY_JSON="$valid_baseline_policies"
- kctl() { printf '%s\n' "$MOCK_POLICY_JSON"; }
- eval "$baseline_policy_snapshot_function"
- eval "$baseline_policies_unchanged_function"
- BASELINE_POLICY_SNAPSHOT="$(baseline_policy_snapshot)"
- baseline_policies_unchanged &&
- MOCK_POLICY_JSON="$(jq -c '
- .items[0].metadata.labels["clerum.io/managed-by"] = "foreign-controller"
- ' <<<"$valid_baseline_policies")" &&
- ! baseline_policies_unchanged &&
- MOCK_POLICY_JSON="$(jq -c '.items[1].spec.egress[0].ports[0].port = 5353' \
- <<<"$valid_baseline_policies")" &&
- ! baseline_policies_unchanged
-); then
- pass "baseline safety proof rejects ownership and spec drift under stable policy names"
-else
- fail "baseline safety proof can pass after ownership or spec drift"
-fi
-
-peer_fleet_policies_absent_function="$(
- sed -n '/^peer_fleet_policies_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-fixture_resources_absent_function="$(
- sed -n '/^fixture_resources_absent() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-delete_peer_fleet_runtime_function="$(
- sed -n '/^delete_peer_fleet_runtime() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-if (
- MCP_NS=mcp-server
- HOST_NS=mcp-host
- RPC_PROXY_NS=rpc-proxy
- MCP_SERVER_LABEL=clerum.io/mcpserver
- POLICY_TYPE_LABEL=clerum.io/policy-type
- MCP_FLEET_NAMES=(primary peer)
- CONTEXT_FLEET_NAMES=(primary-context peer-context)
- SURVIVOR_NS=""
- resource_absent() { [ "$3" != "$SURVIVOR_NS" ]; }
- kctl() { return 0; }
- eval "$peer_fleet_policies_absent_function"
- peer_fleet_policies_absent &&
- SURVIVOR_NS=mcp-host &&
- ! peer_fleet_policies_absent &&
- SURVIVOR_NS=rpc-proxy &&
- ! peer_fleet_policies_absent
-) &&
- [[ "$fixture_resources_absent_function" == *'peer_fleet_policies_absent'* ]] &&
- [[ "$delete_peer_fleet_runtime_function" == *'"${context_policy}-egress"'* ]] &&
- [[ "$delete_peer_fleet_runtime_function" == *'"rpc-egress-${context}-${server}"'* ]]; then
- pass "peer fleet cleanup rejects surviving mcp-host or rpc-proxy policies"
-else
- fail "peer fleet cleanup can release ownership with residual derived policies"
-fi
-
# shellcheck source=scripts/e2e/_lib/hcc-watch-recovery-logs.sh
source "$LOG_HELPER"
HCC_LOG_BUFFER="$MOCK_LOG_FILE"
START_TIME=2026-07-14T00:00:00Z
HOST_NS=mcp-host
-blocker_holds_function="$(sed -n '/^blocker_holds_token_request() {$/,/^}$/p' "$READINESS_GATE")"
-blocker_count_function="$(sed -n '/^blocker_fixture_request_count() {$/,/^}$/p' "$READINESS_GATE")"
-if (
- BLOCKER_NAME=readiness-blocker
- HCC_NS=control-plane
- FIXTURE_HOST_PREFIX=e2e-hcc-ready-test
- TOKEN_REQUEST_LOG_PREFIX='holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host='
- kctl() { cat "$MOCK_LOG_FILE"; }
- eval "$blocker_holds_function"
- eval "$blocker_count_function"
- printf '%s\n' \
- 'holding-token GET /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-01' \
- 'holding-token POST /wrong/path host=e2e-hcc-ready-test-02' \
- 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \
- 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-03' \
- 'holding-token POST /api/v1/auth/mcp-host/mcp-host/standalone/tokens host=e2e-hcc-ready-test-04' \
- >"$MOCK_LOG_FILE"
- [ "$(blocker_fixture_request_count)" = 2 ] &&
- blocker_holds_token_request
-); then
- pass "blocker evidence counts unique fixtures only on the canonical token issuance method and path"
-else
- fail "wrong-method, wrong-path, or duplicate blocker traffic can satisfy the fleet evidence"
-fi
-
-create_fixtures_function="$(sed -n '/^create_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"
-if (
- RUN_ID=fixture-failure
- SUITE_NAME=hcc-readiness-bootstrap
- HOST_NS=mcp-host
- MCP_NS=mcp-server
- FIXTURE_SECRET=e2e-secret
- FIXTURE_CONTEXT=e2e-context
- FIXTURE_HOST_PREFIX=e2e-host
- FIXTURE_HOST_COUNT=4
- FIXTURE_HOST_NAMES=()
- FIXTURES_CREATED=0
- KCTL_APPLY_COUNT=0
- kctl() {
- KCTL_APPLY_COUNT=$((KCTL_APPLY_COUNT + 1))
- cat >/dev/null
- [ "$KCTL_APPLY_COUNT" -ne 3 ]
- }
- eval "$create_fixtures_function"
- if create_host_fixtures; then
- exit 1
- fi
- [ "$KCTL_APPLY_COUNT" = 3 ]
-); then
- pass "an intermediate Host apply failure aborts fixture creation even from an OR-list"
-else
- fail "fixture creation can continue or report success after an intermediate apply failure"
-fi
-
-bootstrap_runtime_absent_function="$(
- sed -n '/^fixture_runtime_absent() {$/,/^}$/p' "$READINESS_GATE"
-)"
-bootstrap_delete_fixtures_function="$(
- sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE"
-)"
-# Literal source-code assertions for macOS bash 3.2, where expanding an empty
-# array under set -u terminates the shell before cleanup can release its lock.
-# shellcheck disable=SC2016
-if [[ "$bootstrap_runtime_absent_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return 0'* ]] &&
- [[ "$bootstrap_delete_fixtures_function" == *'[ "${#FIXTURE_HOST_NAMES[@]}" -eq 0 ] && return "$failed"'* ]]; then
- pass "bootstrap cleanup guards empty fixture arrays before bash 3.2 expansion"
-else
- fail "bootstrap cleanup can abort on an empty fixture array before lock finalization"
-fi
-
-create_peer_fleet_function="$(
- sed -n '/^create_peer_fleet() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-peer_flag_line="$(grep -nF ' PEER_FLEET_CREATED=1' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"
-peer_loop_line="$(grep -nF ' for ((index = 1;' <<<"$create_peer_fleet_function" | head -1 | cut -d: -f1)"
-if [ -n "$peer_flag_line" ] && [ -n "$peer_loop_line" ] &&
- [ "$peer_flag_line" -lt "$peer_loop_line" ]; then
- pass "peer fleet cleanup is armed before the first partial apply can fail"
-else
- fail "peer fleet cleanup is armed only after all applies complete"
-fi
-
-divergent_ready_function="$(
- sed -n '/^hcc_ready_after_stale_policy_revoked() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-divergent_ready_line="$(
- grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-divergent_context_line="$(
- grep -nF 'wait_until 120 "all three latest-Context NetworkPolicies to converge during the MCP hold"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-divergent_release_line="$(
- grep -nF 'release_dns_blocker ||' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1
-)"
-udp_policy_line="$(
- grep -nF 'wait_until 120 "current UDP external-egress policy after DNS release"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-# shellcheck disable=SC2016
-if [[ "$divergent_ready_function" == *'probe_new_hcc_ready_endpoint || return 1'* ]] &&
- [[ "$divergent_ready_function" == *'external_egress_policy_absent'* ]] &&
- [ -n "$divergent_ready_line" ] && [ -n "$divergent_context_line" ] &&
- [ -n "$divergent_release_line" ] && [ -n "$udp_policy_line" ] &&
- [ "$divergent_ready_line" -lt "$divergent_context_line" ] &&
- [ "$divergent_context_line" -lt "$divergent_release_line" ] &&
- [ "$divergent_release_line" -lt "$udp_policy_line" ] &&
- ! grep -Fq 'probe_new_hcc_unready_endpoint' "$MCP_READINESS_GATE" &&
- ! grep -Fq '/ready is 503' "$MCP_READINESS_GATE"; then
- pass "divergent DNS hold proves global readiness, fail-closed runtime isolation, and post-release UDP convergence"
-else
- fail "the MCP gate still couples global readiness to divergent DNS replacement"
-fi
-
-initial_tcp_policy_line="$(
- grep -nF 'wait_until 180 "HCC to create the real initial TCP external-egress policy"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-# B3 retain-identity-stable contract: a same-intent restart RETAINS the
-# identity-stable DNS-derived allow (no deny window), certifies /ready with it
-# still present, keeps the affected runtime blocked, and converges the additive
-# TCP lane only AFTER the DNS hold releases — the inverse of the pre-B3
-# revoke-before-ready ordering this gate previously asserted.
-same_intent_snapshot_line="$(
- grep -nF 'SAME_INTENT_EGRESS_SNAPSHOT="$(external_egress_policy_snapshot)"' \
- "$MCP_READINESS_GATE" | head -1 | cut -d: -f1
-)"
-same_intent_retained_ready_line="$(
- grep -nF 'wait_until_fast 60 "HCC readiness while the same-intent identity-stable DNS policy is retained"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-same_intent_retain_guard_line="$(
- grep -nF 'the same-intent identity-stable DNS policy was revoked at readiness (retain regression / deny window)' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-same_intent_runtime_blocked_line="$(
- grep -nF 'the affected runtime or Ready status escaped the held same-intent DNS boundary' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-same_intent_release_line="$(
- grep -nF 'release_dns_blocker || die "could not release the same-intent DNS query"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-same_intent_tcp_policy_line="$(
- grep -nF 'wait_until 120 "current TCP external-egress policy after same-intent DNS release"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-divergent_revoke_line="$(
- grep -nF 'wait_until_fast 120 "HCC readiness after revoking the divergent stale policy"' \
- "$MCP_READINESS_GATE" | cut -d: -f1
-)"
-if [ -n "$initial_tcp_policy_line" ] && [ -n "$same_intent_snapshot_line" ] &&
- [ -n "$same_intent_retained_ready_line" ] && [ -n "$same_intent_retain_guard_line" ] &&
- [ -n "$same_intent_runtime_blocked_line" ] && [ -n "$same_intent_release_line" ] &&
- [ -n "$same_intent_tcp_policy_line" ] && [ -n "$divergent_revoke_line" ] &&
- [ "$initial_tcp_policy_line" -lt "$same_intent_snapshot_line" ] &&
- [ "$same_intent_snapshot_line" -lt "$same_intent_retained_ready_line" ] &&
- [ "$same_intent_retained_ready_line" -lt "$same_intent_retain_guard_line" ] &&
- [ "$same_intent_retain_guard_line" -lt "$same_intent_runtime_blocked_line" ] &&
- [ "$same_intent_runtime_blocked_line" -lt "$same_intent_release_line" ] &&
- [ "$same_intent_release_line" -lt "$same_intent_tcp_policy_line" ] &&
- [ "$same_intent_tcp_policy_line" -lt "$divergent_revoke_line" ] &&
- grep -Fq 'hcc_ready_with_identity_stable_policy_retained' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'HCC readiness after revoking the same-intent DNS policy' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'a real same-intent external-egress retry to be scheduled' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'CERTIFIED_POLICY_IDENTITY' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'external_egress_policy_identity' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'UID/spec' "$MCP_READINESS_GATE"; then
- pass "same-intent safety retains the identity-stable DNS policy at readiness; additive TCP convergence follows DNS release"
-else
- fail "the MCP readiness gate no longer proves B3 retain-identity-stable (or regressed to revoke-same-intent before readiness)"
-fi
-
printf '%s\n' \
'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
'[K8s] Listing all Hosts in namespace mcp-host' \
@@ -1329,30 +41,6 @@ else
fail "valid LIST-to-recovery ordering was rejected"
fi
-printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
- '[K8s] Listing all Hosts in namespace mcp-host' \
- '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
- '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \
- '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"
-if recovery_cycle_used_fresh_host_inventory 1 5; then
- pass "a causally ordered Host-watch pass may cover the recovered CC and fresh Host inventories"
-else
- fail "a valid covering Host-watch recovery pass was rejected"
-fi
-
-printf '%s\n' \
- '[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
- '[K8s] Listing all Hosts in namespace mcp-host' \
- '[K8s] Reconciling 5 Host(s) after Host watch recovery convergence' \
- '[K8s] Recovered 2 CommunicationChannel(s) into cache (ccCacheSynced=true)' \
- '[K8s] Completed Host reconciliation after Host watch recovery convergence' >"$MOCK_LOG_FILE"
-if recovery_cycle_used_fresh_host_inventory 1 5; then
- fail "a Host-watch pass selected before CC recovery satisfied the covering assertion"
-else
- pass "covering Host-watch recovery requires recovered CC state before pass selection"
-fi
-
printf '%s\n' \
'[K8s] Listing all Hosts in namespace mcp-host' \
'[K8s] CommunicationChannel watch ended; holding stateless lifecycle active until snapshot recovery' \
@@ -1394,272 +82,19 @@ else
fail "the second recovery cycle was not isolated from the first"
fi
-large_log_tail="$(awk 'BEGIN {
- for (i = 1; i <= 20000; i++) {
- printf "trailing-log-line-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n", i
- }
-}')"
-large_active_snapshot="${START_MARKER:-Starting initial Host background convergence}
-${large_log_tail}"
-if hcc_log_snapshot_contains \
- "$large_active_snapshot" 'Starting initial Host background convergence'; then
- pass "buffered marker lookup survives a log tail larger than a pipe buffer"
-else
- fail "large trailing logs can hide a marker from buffered lookup"
-fi
-
-if hcc_initial_pass_snapshot_is_active \
- "$large_active_snapshot" \
- 'Starting initial Host background convergence' \
- 'Completed Host reconciliation after initial Host reconciliation' \
- 'Host reconciliation after initial Host reconciliation failed'; then
- pass "Host pass snapshot is active only with START and no terminal marker"
-else
- fail "valid active Host pass snapshot was rejected"
-fi
-
-for terminal_marker in \
- 'Completed Host reconciliation after initial Host reconciliation' \
- 'Host reconciliation after initial Host reconciliation failed'; do
- if hcc_initial_pass_snapshot_is_active \
- "${large_active_snapshot}
-${terminal_marker}" \
- 'Starting initial Host background convergence' \
- 'Completed Host reconciliation after initial Host reconciliation' \
- 'Host reconciliation after initial Host reconciliation failed'; then
- fail "terminal Host marker '${terminal_marker}' can still satisfy the active-pass guard"
- else
- pass "terminal Host marker '${terminal_marker}' invalidates the active-pass guard"
- fi
-done
-
-readiness_log_function="$(
- sed -n '/^hcc_log_contains() {$/,/^}$/p' "$READINESS_GATE"
-)"
-running_pod_function="$(
- sed -n '/^running_hcc_pod() {$/,/^}$/p' "$READINESS_GATE"
-)"
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if [[ "$readiness_log_function" == *'logs="$(hcc_logs "$pod")"'* ]] &&
- [[ "$readiness_log_function" == *'hcc_log_snapshot_contains "$logs" "$marker"'* ]] &&
- [[ "$running_pod_function" == *'rows="$(kctl get pods'* ]] &&
- [[ "$running_pod_function" == *'<<<"$rows"'* ]]; then
- pass "readiness log and Running-pod selection buffer producers before early-exit consumers"
-else
- fail "readiness log or Running-pod selection reintroduced a pipefail/SIGPIPE path"
-fi
-
-active_guard_function="$(
- sed -n '/^initial_host_pass_is_active() {$/,/^}$/p' "$READINESS_GATE"
-)"
-initial_host_startup_producer="$(
- sed -n \
- '/Starting initial Host background convergence/,/const resyncSec = config.hostResyncIntervalSec/p' \
- "$HCC_K8S_CLIENT"
-)"
-# The bootstrap guard's negative COMPLETE/FAIL checks are safe only while all
-# three consumed markers remain bound to the runtime templates and the startup
-# call supplies the exact reason that renders the terminal pair.
-if grep -Fq 'Starting initial Host background convergence...' \
- "$HCC_K8S_CLIENT" &&
- grep -Fq 'Completed Host reconciliation after ${reason}' \
- "$HCC_K8S_CLIENT" &&
- grep -Fq 'Host reconciliation after ${reason} failed:' \
- "$HCC_K8S_CLIENT" &&
- [[ "$initial_host_startup_producer" == *'await this.recoverHostInventoryAndWatch('* ]] &&
- [[ "$initial_host_startup_producer" == *"'initial Host reconciliation',"* ]]; then
- pass "bootstrap START/COMPLETE/FAIL markers remain bound to the initial Host producer reason"
-else
- fail "bootstrap Host pass markers drifted from their runtime producer or reason binding"
-fi
-
-# The destructive Host-bundle measurement uses the same cold-start transition
-# as the bootstrap readiness gate. Keep its active-pass marker tied to the
-# emitted producer rather than an obsolete reason-string log line.
-if grep -Fq "readonly HCC_PASS_STARTED_MARKER='Starting initial Host background convergence'" \
- "$HOST_BUNDLE_MEASURE" &&
- grep -Fq 'Starting initial Host background convergence...' "$HCC_K8S_CLIENT"; then
- pass "Host-bundle measurement START marker remains bound to the cold-start producer"
-else
- fail "Host-bundle measurement can wait for a stale HCC START marker"
-fi
-# shellcheck disable=SC2016
-probe_result_line="$(grep -nF 'probe_result="$(kctl exec' "$READINESS_GATE" | cut -d: -f1)"
-# shellcheck disable=SC2016
-final_active_guard_line="$(
- grep -nF 'initial_host_pass_is_active "$new_hcc_pod"' "$READINESS_GATE" |
- tail -1 |
- cut -d: -f1
-)"
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if [[ "$active_guard_function" == *'"$COMPLETE_MARKER"'* ]] &&
- [[ "$active_guard_function" == *'"$FAIL_MARKER"'* ]] &&
- [ -n "$probe_result_line" ] &&
- [ -n "$final_active_guard_line" ] &&
- [ "$final_active_guard_line" -gt "$probe_result_line" ]; then
- pass "final readiness assertion rejects both failed and completed Host passes"
-else
- fail "final readiness assertion does not enforce both terminal Host markers"
-fi
-
-cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$READINESS_GATE")"
-fixture_cleanup_body="$(sed -n '/^delete_host_fixtures() {$/,/^}$/p' "$READINESS_GATE")"
-# shellcheck disable=SC2016
-cleanup_stop_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0' \
- <<<"$cleanup_body" | head -1 | cut -d: -f1)"
-cleanup_fixture_line="$(grep -nF 'delete_host_fixtures' <<<"$cleanup_body" | head -1 | cut -d: -f1)"
-# shellcheck disable=SC2016
-cleanup_restore_line="$(grep -nF 'kctl set env deployment/"$HCC_DEPLOY"' \
- <<<"$cleanup_body" | head -1 | cut -d: -f1)"
+acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock || die' "$GATE" | cut -d: -f1)"
# Literal source-code assertion.
# shellcheck disable=SC2016
-if grep -Fq 'FIXTURE_HOST_COUNT=$((HOST_RECONCILE_CONCURRENCY * 2 + 1))' "$READINESS_GATE" &&
- grep -Fq 'kind: Host' "$READINESS_GATE" &&
- grep -Fq "host='+host" "$READINESS_GATE" &&
- grep -Fq 'blocker_fixture_request_count' "$READINESS_GATE" &&
- [[ "$fixture_cleanup_body" == *'clerum.io/managed-by=host-context-controller,clerum.io/host=${host}'* ]] &&
- [[ "$fixture_cleanup_body" == *'fixture_resources_absent'* ]] &&
- ! grep -Fq 'RUNTIME_SECRET' "$READINESS_GATE" &&
- ! grep -Fq 'HOST_REF' "$READINESS_GATE" &&
- [ -n "$cleanup_stop_line" ] &&
- [ -n "$cleanup_fixture_line" ] &&
- [ -n "$cleanup_restore_line" ] &&
- [ "$cleanup_stop_line" -lt "$cleanup_fixture_line" ] &&
- [ "$cleanup_fixture_line" -lt "$cleanup_restore_line" ]; then
- pass "readiness gate uses a multi-wave owned Host fleet and deletes it while HCC is stopped"
-else
- fail "readiness gate can mutate a real Host Secret or restore HCC before fixture deletion"
-fi
-
-if [[ "$cleanup_body" == *'header "HCC readiness bootstrap gate passed"'* ]] &&
- grep -Fq 'header "HCC readiness assertions passed; restoring branch-owned runtime"' \
- "$READINESS_GATE" &&
- [ "$(grep -Fc 'header "HCC readiness bootstrap gate passed"' "$READINESS_GATE")" = 1 ]; then
- pass "readiness gate emits its final pass banner only after verified restoration and lock finalization"
-else
- fail "readiness gate can announce a final pass before cleanup and ownership release finish"
-fi
-
-mcp_cleanup_body="$(sed -n '/^cleanup() {$/,/^}$/p' "$MCP_READINESS_GATE")"
-# shellcheck disable=SC2016
-mcp_input_absence_line="$(
- grep -nF 'fixture_inputs_absent' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
-)"
-# shellcheck disable=SC2016
-mcp_restore_line="$(
- grep -nF 'restore_patch="$(jq -cn' <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
-)"
-if [[ "$mcp_cleanup_body" == *'fixture_inputs_removed=0'* ]] &&
- [[ "$mcp_cleanup_body" == *'if [ "$fixture_inputs_removed" = 1 ] && [ "$restore_ok" = 1 ]; then'* ]] &&
- [ -n "$mcp_input_absence_line" ] &&
- [ -n "$mcp_restore_line" ] &&
- [ "$mcp_input_absence_line" -lt "$mcp_restore_line" ]; then
- pass "MCP readiness cleanup keeps HCC stopped until fixture input deletion is verified"
-else
- fail "MCP readiness cleanup can restart HCC after an unverified Context or McpServer deletion"
-fi
-
-mcp_finalize_line="$(
- grep -nF 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' \
- <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
-)"
-mcp_pass_line="$(
- grep -nF 'header "HCC McpServer/Context readiness gate passed"' \
- <<<"$mcp_cleanup_body" | head -1 | cut -d: -f1
-)"
-if [ "$(grep -Fc 'header "HCC McpServer/Context readiness gate passed"' \
- "$MCP_READINESS_GATE")" = 1 ] &&
- [ -n "$mcp_finalize_line" ] &&
- [ -n "$mcp_pass_line" ] &&
- [ "$mcp_finalize_line" -lt "$mcp_pass_line" ]; then
- pass "MCP readiness gate announces PASS only after cleanup and lock finalization"
-else
- fail "MCP readiness gate can announce PASS before cleanup and lock finalization"
-fi
-
-wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$READINESS_GATE")"
+mutation_line="$(grep -nF 'kctl get deployment "$HCC_DEPLOY"' "$GATE" | head -1 | cut -d: -f1)"
# Literal source-code assertion.
# shellcheck disable=SC2016
-if [[ "$wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&
- ! grep -Fq -- '--for=delete' "$READINESS_GATE" &&
- ! grep -Fq 'port: 8081' "$READINESS_GATE" &&
- grep -Fq 'HCC_E2E_PORT' "$READINESS_GATE"; then
- pass "readiness gate uses wall-clock deadlines, absence waits, and the deployed HCC port"
-else
- fail "readiness gate retains a command-latency, zero-pod wait, or hard-coded-port edge"
-fi
-
-mcp_wait_until_body="$(sed -n '/^wait_until() {$/,/^}$/p' "$MCP_READINESS_GATE")"
-mcp_port_probe_count="$(grep -Fc 'env "HCC_E2E_PORT=${HCC_PORT}"' "$MCP_READINESS_GATE")"
-mcp_pod_absence_count="$(grep -Fc 'hcc_pods_absent' "$MCP_READINESS_GATE")"
-# Literal source-code assertions.
-# shellcheck disable=SC2016
-if grep -Fq 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \
- "$MCP_READINESS_GATE" &&
- [ "$(grep -Fc 'require_branch_owned_hcc_gate "$HCC_NS"' "$MCP_READINESS_GATE")" = 1 ] &&
- ! grep -Fq 'HCC_BRANCH_GATE_SYNC_MARKER' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'cluster_fingerprint_file=' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'profile_env=' "$MCP_READINESS_GATE" &&
- [[ "$mcp_wait_until_body" == *'deadline=$(( $(date +%s) + timeout ))'* ]] &&
- grep -Fq 'hcc_pods_absent' "$MCP_READINESS_GATE" &&
- ! grep -Fq -- '--for=delete' "$MCP_READINESS_GATE" &&
- ! grep -Fq 'port: 8081' "$MCP_READINESS_GATE" &&
- [ "$mcp_port_probe_count" = 4 ] &&
- [ "$mcp_pod_absence_count" = 4 ]; then
- pass "MCP readiness gate shares exact-head ownership and uses bounded portable runtime probes"
-else
- fail "MCP readiness gate duplicates ownership or retains a deadline, pod-absence, or port edge"
-fi
-
-# A single held McpServer proves the minimal failure, but this gate must also
-# cross the controller's bounded worker width with independent, gate-owned
-# McpServer/Context pairs. Keep this as a static contract so future edits do
-# not silently reduce the fleet scenario back to one fixture. The fixture
-# must exist while HCC is stopped, otherwise it would not exercise startup.
-mcp_peer_create_line="$(grep -nF 'create_peer_fleet || die' "$MCP_READINESS_GATE" | cut -d: -f1)"
-mcp_restart_line="$(grep -nF 'kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1' "$MCP_READINESS_GATE" | tail -1 | cut -d: -f1)"
-peer_fleet_converged_body="$(
- sed -n '/^peer_fleet_converged() {$/,/^}$/p' "$MCP_READINESS_GATE"
-)"
-if grep -Fq 'MCP_FLEET_SIZE="${E2E_HCC_MCP_FLEET_SIZE:-11}"' "$MCP_READINESS_GATE" &&
- grep -Fq '[ "$MCP_FLEET_SIZE" -gt 10 ]' "$MCP_READINESS_GATE" &&
- grep -Fq 'create_peer_fleet' "$MCP_READINESS_GATE" &&
- grep -Fq 'peer_fleet_converged' "$MCP_READINESS_GATE" &&
- [[ "$peer_fleet_converged_body" == *'mcpserver_current_ready "$server"'* ]] &&
- [[ "$peer_fleet_converged_body" == *'context_policies_converged "$context" "$context" "$server"'* ]] &&
- grep -Fq 'all peer MCP/Context runtimes to converge after releasing the held primary egress' "$MCP_READINESS_GATE" &&
- [ -n "$mcp_peer_create_line" ] && [ -n "$mcp_restart_line" ] &&
- [ "$mcp_peer_create_line" -lt "$mcp_restart_line" ]; then
- pass "MCP readiness gate crosses the bounded worker width with gate-owned MCP and Context peers"
+if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] && [ "$acquire_line" -lt "$mutation_line" ] &&
+ grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1' "$GATE"; then
+ pass "gate acquires before HCC access and finalizes ownership from cleanup"
else
- fail "MCP readiness gate does not prove a fleet larger than the worker width"
+ fail "gate lock is not held across the full HCC fault-injection window"
fi
-for gate in "${GATES[@]}"; do
- acquire_line="$(grep -nF 'acquire_hcc_watch_gate_lock' "$gate" | tail -1 | cut -d: -f1)"
- case "$gate" in
- "$WATCH_GATE")
- # Literal source-code assertion.
- # shellcheck disable=SC2016
- mutation_line="$(grep -nF 'kctl patch deployment "$HCC_DEPLOY"' "$gate" | tail -1 | cut -d: -f1)"
- ;;
- *)
- mutation_line="$(grep -nF 'HCC_MUTATED=1' "$gate" | tail -1 | cut -d: -f1)"
- ;;
- esac
- # The grep pattern intentionally matches literal shell variable references.
- # shellcheck disable=SC2016
- if [ -n "$acquire_line" ] && [ -n "$mutation_line" ] &&
- [ "$acquire_line" -lt "$mutation_line" ] &&
- grep -Fq 'finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok"' "$gate"; then
- pass "$(basename "$gate") acquires before mutation and finalizes ownership from cleanup"
- else
- fail "$(basename "$gate") does not hold the shared lock across its full fault-injection window"
- fi
-done
-
kctl() {
local action=$1 body expected_rv current_rv current_uid next_rv
case "$action" in
@@ -1703,7 +138,7 @@ truncate_rfc1123() { printf '%.63s' "$1"; }
source "$LOCK_HELPER"
HCC_NS=control-plane
-HCC_DEPLOY="host-context-controller"
+HCC_DEPLOY=host-context-controller
E2E_KUBECONTEXT=clerum-codex-lock-test-1234abcd
RUN_ID=owner-1
HCC_GATE_LOCK_ACQUIRED=0
@@ -1818,62 +253,12 @@ else
fi
unset MOCK_CREATE_WITHOUT_UID
-cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$WATCH_GATE" | cut -d: -f1)"
-results_line="$(grep -nF ' print_results' "$WATCH_GATE" | head -1 | cut -d: -f1)"
+cleanup_fail_line="$(grep -nF 'fail "fixture cleanup, HCC restoration, or lock finalization did not complete' "$GATE" | cut -d: -f1)"
+results_line="$(grep -nF ' print_results' "$GATE" | head -1 | cut -d: -f1)"
if [ -n "$cleanup_fail_line" ] && [ -n "$results_line" ] && [ "$cleanup_fail_line" -lt "$results_line" ]; then
pass "cleanup failure is counted before the final result summary"
else
fail "cleanup can still print a false all-passed summary"
fi
-# --- H3: SIGPIPE/pipefail sweep on HCC pass-marker log probes -------------
-# A `printf '%s[\n]' "$logs" | grep -Fq "$..._MARKER"` probe is a false-green
-# trap: on a large log with the marker near the top, `grep -Fq` exits early,
-# `printf` takes SIGPIPE, and under `set -o pipefail` the whole pipeline reports
-# failure — so a PRESENT marker reads as MISSING (deterministic 3/3 repro in the
-# round-2 storm gate). The safe idiom is a here-string: no producer to signal.
-# grep -Fq -- "$MARKER" <<<"$logs"
-#
-# Scope: HCC pass-marker gates only (storm + measure). Explicitly OUT OF SCOPE
-# (tracked separately — a different failure class, or a non-HCC gate outside
-# this PR's blast radius):
-# - e2e-gke-secrets.sh:291-294 grep -oE|wc leak COUNT: reads the whole log,
-# no early-exit, so it is NOT the SIGPIPE class.
-# - e2e-plugin-workload-sdk.sh SDK markers (non-HCC).
-# - e2e-lib.sh / e2e-prod-lib.sh / e2e-channel-reader-per-host.sh (non-HCC).
-HCC_STORM_GATE="${SCRIPT_DIR}/e2e-host-storm-gate.sh"
-# shellcheck disable=SC2016
-sigpipe_offenders="$(
- grep -nE 'printf[^|]*\| *grep -Fq[^<]*_MARKER' \
- "$HCC_STORM_GATE" "$HOST_BUNDLE_MEASURE" 2>/dev/null || true
-)"
-if [ -z "$sigpipe_offenders" ]; then
- pass "HCC pass-marker probes use here-strings, not pipefail/SIGPIPE pipes"
-else
- fail "HCC pass-marker probe reintroduced a printf|grep -Fq SIGPIPE path:
-${sigpipe_offenders}"
-fi
-
-# Corroborating repro of the mechanism the sweep guards against. The here-string
-# form MUST find a marker on line 1 of a 1.6 MB log (hard invariant). The pipe
-# form is expected to miss it under pipefail (SIGPIPE) — that miss is the bug,
-# demonstrated, not a standing assertion, so a platform that does not reproduce
-# it is not a suite failure: the static sweep above is the real enforcement.
-h3_marker='HCC_PASS_COMPLETED: h3 repro sentinel'
-h3_bulk="$(yes 'xxxxxxxxxxxxxxxx' | head -c 1600000)"
-h3_log="${h3_marker}
-${h3_bulk}"
-if grep -Fq -- "$h3_marker" <<<"$h3_log"; then
- pass "here-string probe finds a marker on line 1 of a 1.6 MB log"
-else
- fail "here-string probe MISSED a present marker on a large log (regression)"
-fi
-h3_pipe_found=0
-( set -o pipefail; printf '%s\n' "$h3_log" | grep -Fq -- "$h3_marker" ) && h3_pipe_found=1
-if [ "$h3_pipe_found" -eq 0 ]; then
- pass "pipe+pipefail probe misses the same marker (SIGPIPE repro confirmed)"
-else
- pass "pipe probe did not SIGPIPE on this platform; static sweep still enforces the idiom"
-fi
-
exit "$FAIL"
diff --git a/scripts/tests/test-host-storm-metric-gate.sh b/scripts/tests/test-host-storm-metric-gate.sh
deleted file mode 100755
index e5d994bb5..000000000
--- a/scripts/tests/test-host-storm-metric-gate.sh
+++ /dev/null
@@ -1,296 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2016 # literal source-contract assertions below
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-GATE="${ROOT}/scripts/e2e/e2e-host-storm-gate.sh"
-WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/host-storm-metric-test.XXXXXX")"
-ADJUDICATOR="${WORK_DIR}/adjudicator.py"
-trap 'rm -rf "$WORK_DIR"' EXIT
-
-fail() {
- echo "FAIL: $*" >&2
- exit 1
-}
-
-extract_adjudicator() {
- awk '
- /python3 >"\$METRIC_VERDICTS" <<'\''PY'\''/ { capture = 1; next }
- capture && /^PY$/ { exit }
- capture { print }
- ' "$GATE" >"$ADJUDICATOR"
- [ -s "$ADJUDICATOR" ] || fail "could not extract host-storm metric adjudicator"
-}
-
-write_metrics() {
- local path=$1 le_05=$2 le_5=$3 le_15=$4 total=$5 fleet_started=${6:-0}
- local fleet_coalesced=${7:-0} fleet_trailing=${8:-0} fleet_failed=${9:-0}
- cat >"$path" <<EOF
-# TYPE clerum_hcc_host_delete_cleanup_total counter
-clerum_hcc_host_delete_cleanup_total{outcome="queued"} 0
-clerum_hcc_host_delete_cleanup_total{outcome="confirmed"} 0
-clerum_hcc_host_delete_cleanup_total{outcome="completed"} 0
-clerum_hcc_host_delete_cleanup_total{outcome="retried"} 0
-clerum_hcc_host_delete_cleanup_total{outcome="superseded"} 0
-# TYPE clerum_hcc_host_fleet_requests_total counter
-clerum_hcc_host_fleet_requests_total{result="started"} ${fleet_started}
-clerum_hcc_host_fleet_requests_total{result="coalesced"} ${fleet_coalesced}
-clerum_hcc_host_fleet_requests_total{result="trailing"} ${fleet_trailing}
-clerum_hcc_host_fleet_requests_total{result="failed"} ${fleet_failed}
-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="0.5"} ${le_05}
-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="5"} ${le_5}
-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="15"} ${le_15}
-clerum_hcc_host_reconcile_queue_wait_seconds_bucket{lane="urgent",outcome="success",le="+Inf"} ${total}
-EOF
-}
-
-extract_adjudicator
-
-for scrape in a b c1 c2 d wake_pre wake_post; do
- write_metrics "${WORK_DIR}/${scrape}.prom" 0 0 0 0
-done
-
-cat >>"${WORK_DIR}/c1.prom" <<'EOF'
-clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="15"} 1
-clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="success",le="+Inf"} 1
-clerum_hcc_host_watch_recovery_seconds_bucket{phase="total",outcome="failure",le="+Inf"} 0
-EOF
-
-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 1
-
-# The post-recovery direct-watch window is healthy.
-write_metrics "${WORK_DIR}/d.prom" 10 10 10 10
-
-# The wake window contains one sample over the 5s hard gate plus nineteen
-# fast samples. A combined p95 would be green (19/20 == 95%), but the
-# supported wake journey violated its binding urgent queue budget.
-write_metrics "${WORK_DIR}/wake_post.prom" 19 19 20 20
-
-run_adjudicator() {
- local wake_post=$1 fleet_completion_proven=${2:-1}
- SCRAPE_A="${WORK_DIR}/a.prom" \
- SCRAPE_B="${WORK_DIR}/b.prom" \
- SCRAPE_C1="${WORK_DIR}/c1.prom" \
- SCRAPE_C2="${WORK_DIR}/c2.prom" \
- SCRAPE_D="${WORK_DIR}/d.prom" \
- SCRAPE_WAKE_PRE="${WORK_DIR}/wake_pre.prom" \
- SCRAPE_WAKE_POST="$wake_post" \
- QUEUE_WAIT_METRIC=clerum_hcc_host_reconcile_queue_wait_seconds \
- WATCH_RECOVERY_METRIC=clerum_hcc_host_watch_recovery_seconds \
- FLEET_METRIC=clerum_hcc_host_fleet_requests_total \
- DELETE_CLEANUP_METRIC=clerum_hcc_host_delete_cleanup_total \
- P95_BUDGET_S=5 \
- RECOVERY_BUDGET_S=15 \
- MIN_URGENT_SAMPLES=10 \
- FLEET_COMPLETION_PROVEN="$fleet_completion_proven" \
- python3 "$ADJUDICATOR"
-}
-
-OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_post.prom")"
-
-if ! grep -Fq 'VERDICT|A2-direct-watch-urgent-p95|PASS|' <<<"$OUTPUT" &&
- ! grep -Fq 'VERDICT|A2-urgent-queue-wait-p95|PASS|' <<<"$OUTPUT"; then
- fail "healthy post-recovery direct-watch window did not pass"
-fi
-grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|FAIL|' <<<"$OUTPUT" ||
- fail "slow wake window was diluted into a false PASS"
-
-write_metrics "${WORK_DIR}/wake_healthy.prom" 20 20 20 20
-HEALTHY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
-grep -Fq 'VERDICT|A2-wake-urgent-hard-gate|PASS|' <<<"$HEALTHY_OUTPUT" ||
- fail "healthy wake window did not pass its hard gate"
-
-# A7 is fenced to the replacement pod's explicit initial-fleet COMPLETED log.
-# A second non-overlapping request can therefore be legitimate activity after
-# recovery; an absolute started==1 assertion must not false-red it.
-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2
-TWO_STARTS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$TWO_STARTS_OUTPUT" ||
- fail "healthy replacement-pod recovery false-reds when a second sequential fleet request starts"
-
-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0
-NO_START_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|PASS|' <<<"$NO_START_OUTPUT" ||
- fail "A7 still treats a scheduled counter as stronger than causal completion evidence"
-
-NO_COMPLETION_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom" 0)"
-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$NO_COMPLETION_OUTPUT" ||
- fail "A7 false-passed without replacement-pod fleet completion evidence"
-
-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 0 0 1
-FAILED_PASS_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$FAILED_PASS_OUTPUT" ||
- fail "A7 false-passed a failed replacement-pod fleet pass"
-
-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 2 1 2
-UNBOUNDED_TRAILING_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$UNBOUNDED_TRAILING_OUTPUT" ||
- fail "A7 false-passed trailing fleet work without a matching coalesced request"
-
-# M8: without the FLEET_METRIC family registered on the running image, counter()
-# returns 0 for every result label and A7 would pass vacuously (all-zero looks
-# like "no failures"). Strip the family's # TYPE line so it drops out of
-# c2_types (mirrors a metric rename / an image that never registered it) and
-# assert A7 FAILs on the presence check, not vacuously passes.
-write_metrics "${WORK_DIR}/c2.prom" 0 0 0 0 0
-grep -v '^# TYPE clerum_hcc_host_fleet_requests_total ' "${WORK_DIR}/c2.prom" \
- >"${WORK_DIR}/c2.nofleet.prom"
-mv "${WORK_DIR}/c2.nofleet.prom" "${WORK_DIR}/c2.prom"
-MISSING_FAMILY_OUTPUT="$(run_adjudicator "${WORK_DIR}/wake_healthy.prom")"
-grep -Fq 'VERDICT|A7-fleet-recovery-bounded|FAIL|' <<<"$MISSING_FAMILY_OUTPUT" ||
- fail "A7 vacuously passed with the FLEET_METRIC family absent from the scrape"
-grep -Fq 'not registered on the running image' <<<"$MISSING_FAMILY_OUTPUT" ||
- fail "A7 failed for the wrong reason when the FLEET_METRIC family was absent"
-
-WAKE_EVIDENCE_FUNCTION="$(
- sed -n '/^_wake_urgent_evidence_recorded() {$/,/^}$/p' "$GATE"
-)"
-WAKE_GENERATION_FUNCTION="$(
- sed -n '/^wake_generation_is_handled() {$/,/^}$/p' "$GATE"
-)"
-POST_WAKE_FUNCTION="$(
- sed -n '/^post_wake() {$/,/^}$/p' "$GATE"
-)"
-
-[[ -n "$WAKE_GENERATION_FUNCTION" ]] ||
- fail "runtime wake evidence has no generation-correlation predicate"
-
-# Exercise the exact pure predicate embedded in the E2E gate. The requested
-# generation is the business identifier returned by the public wake route;
-# an older handled generation must never satisfy that request.
-eval "$WAKE_GENERATION_FUNCTION"
-if wake_generation_is_handled 7 6; then
- fail "wake generation predicate accepted an older handled generation"
-fi
-wake_generation_is_handled 7 7 ||
- fail "wake generation predicate rejected the exact handled generation"
-wake_generation_is_handled 7 8 ||
- fail "wake generation predicate rejected a newer handled generation"
-if wake_generation_is_handled 0 8 || wake_generation_is_handled invalid 8 ||
- wake_generation_is_handled 7 invalid; then
- fail "wake generation predicate accepted malformed generation evidence"
-fi
-
-# Literal source assertions intentionally match unexpanded shell variables.
-# shellcheck disable=SC2016
-if ! grep -Fq '"$current_pod" == "$HCC_POD_WAKE"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
- ! grep -Fq '"$lifecycle_state" != '\''suspended'\''' <<<"$WAKE_EVIDENCE_FUNCTION" ||
- ! grep -Fq 'handled_generation="$(wake_handled_generation)"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
- ! grep -Fq 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
- ! grep -Fq 'metric="${IN_FLIGHT_METRIC}"' <<<"$WAKE_EVIDENCE_FUNCTION" ||
- ! grep -Fq '(after - before) >= 1 && active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then
- fail "runtime wake evidence is not fenced to the same pod, requested generation, metric delta, and idle urgent lane"
-fi
-generation_check_line="$(
- grep -nF 'wake_generation_is_handled "$WAKE_GENERATION" "$handled_generation"' \
- <<<"$WAKE_EVIDENCE_FUNCTION" | cut -d: -f1
-)"
-metrics_scrape_line="$(
- grep -nF 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \
- <<<"$WAKE_EVIDENCE_FUNCTION" | head -n1 | cut -d: -f1
-)"
-if [[ -z "$generation_check_line" || -z "$metrics_scrape_line" ]] ||
- (( generation_check_line >= metrics_scrape_line )); then
- fail "wake generation is not correlated before accepting shared urgent metrics"
-fi
-wake_scrape_count="$(
- grep -Fc 'scrape_hcc_metrics "$SCRAPE_WAKE_POST" "$HCC_POD_WAKE"' \
- <<<"$WAKE_EVIDENCE_FUNCTION"
-)"
-if (( wake_scrape_count < 2 )) ||
- ! grep -Fq 'fence_in_flight=' <<<"$WAKE_EVIDENCE_FUNCTION" ||
- ! grep -Fq 'active == 0' <<<"$WAKE_EVIDENCE_FUNCTION"; then
- fail "wake evidence lacks a completed-lane fence followed by a fresh metric scrape"
-fi
-
-# shellcheck disable=SC2016
-if ! grep -Fq 'WAKE_GENERATION=' <<<"$POST_WAKE_FUNCTION" ||
- ! grep -Fq '"$WAKE_STATUS" == '\''202'\''' <<<"$POST_WAKE_FUNCTION" ||
- ! grep -Fq '.status // empty' <<<"$POST_WAKE_FUNCTION" ||
- ! grep -Fq '.wakeGeneration // empty' <<<"$POST_WAKE_FUNCTION"; then
- fail "public wake response is not validated as an accepted wake-generation contract"
-fi
-
-# A7 must start from a fresh scrape immediately before the kill. Reusing the
-# earlier wake-evidence scrape creates an unobserved interval in which a fleet
-# failure could occur and escape the pre/post recovery counter delta.
-# shellcheck disable=SC2016
-if ! grep -Fq 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" ||
- grep -Fq 'cp "$SCRAPE_WAKE_POST" "$SCRAPE_B"' "$GATE"; then
- fail "pre-kill scrape B is not freshly captured after wake adjudication"
-fi
-fresh_scrape_line="$(
- grep -nF 'scrape_hcc_metrics "$SCRAPE_B" "$HCC_POD_B"' "$GATE" | cut -d: -f1
-)"
-kill_line="$(grep -nF 'T_KILL_MS="$(now_ms)"' "$GATE" | cut -d: -f1)"
-if [[ -z "$fresh_scrape_line" || -z "$kill_line" ]] ||
- (( fresh_scrape_line >= kill_line )); then
- fail "fresh scrape B does not precede the HCC kill"
-fi
-
-# The destructive Host storm must use the same exact-HEAD/profile/fingerprint
-# ownership contract and exclusive compare-and-swap lock as every HCC gate.
-# shellcheck disable=SC2016
-for required in \
- 'source "${SCRIPT_DIR}/e2e-lib.sh"' \
- 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"' \
- 'source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"' \
- 'require_branch_owned_hcc_gate "$CONTROL_NS"' \
- 'acquire_hcc_watch_gate_lock' \
- 'finalize_hcc_watch_gate_lock'; do
- grep -Fq "$required" "$GATE" ||
- fail "Host storm omits shared HCC gate contract: ${required}"
-done
-grep -Fq '.metadata.labels["minikube.k8s.io/name"] == $context' "$GATE" ||
- fail "Host storm does not bind the node's exact minikube profile label"
-grep -Fq 'MINIKUBE_PROFILE must explicitly select context' \
- "${ROOT}/scripts/e2e/_lib/hcc-watch-recovery-fixture.sh" ||
- fail "shared profile ownership helper no longer binds MINIKUBE_PROFILE"
-
-# Branch-owned URLs must come from the selected profile's ports.env. Fixed
-# machine-global service or metrics ports are a cross-worktree false-green risk.
-grep -Fq 'CLERUM_PROFILE_PORTS_ENV' "$GATE" ||
- fail "Host storm does not load branch-profile port evidence"
-grep -Fq 'profile_env_value EXTERNAL_REST_API_URL' "$GATE" ||
- fail "Host storm does not derive External REST from branch ports.env"
-grep -Fq 'profile_env_value RPC_PROXY_URL' "$GATE" ||
- fail "Host storm does not derive RPC proxy from branch ports.env"
-if grep -Eq '127\.0\.0\.1:(8091|8094|18082)' "$GATE"; then
- fail "Host storm retains a fixed machine-global service or metrics port"
-fi
-grep -Fq '"pod/${pod}" ":${HCC_PORT}"' "$GATE" ||
- fail "HCC metric scrape no longer asks kubectl for a process-owned random port"
-
-# KEEP_FIXTURES may be useful interactively, but it cannot produce PASS evidence
-# or leave a lock-free mutated profile that could later be mistaken for T2.
-keep_guard_line="$(grep -nF 'KEEP_FIXTURES=1 is diagnostic-only and cannot produce a probative T2 Host-storm result' "$GATE" | cut -d: -f1)"
-branch_gate_line="$(grep -nF 'require_branch_owned_hcc_gate "$CONTROL_NS"' "$GATE" | cut -d: -f1)"
-if [[ -z "$keep_guard_line" || -z "$branch_gate_line" ]] ||
- (( keep_guard_line >= branch_gate_line )); then
- fail "KEEP_FIXTURES is not rejected before branch verification and mutation"
-fi
-if grep -Fq 'leaving fixtures in place' "$GATE"; then
- fail "Host storm can still report through the fixture-retention path"
-fi
-
-# Completion must be read from the exact replacement pod and established
-# before C1/C2 metric adjudication. Merely observing `started` is not proof.
-FLEET_COMPLETION_FUNCTION="$(
- sed -n '/^_replacement_fleet_completed_probe() {$/,/^}$/p' "$GATE"
-)"
-grep -Fq '"$current_pod" == "$HCC_POD_C"' <<<"$FLEET_COMPLETION_FUNCTION" ||
- fail "fleet completion probe is not fenced to the replacement pod"
-grep -Fq '"pod/${HCC_POD_C}"' <<<"$FLEET_COMPLETION_FUNCTION" ||
- fail "fleet completion probe reads a deployment aggregate instead of the replacement pod"
-grep -Fq '$HCC_PASS_COMPLETED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||
- fail "fleet completion probe does not require the explicit COMPLETED marker"
-grep -Fq '$HCC_PASS_FAILED_MARKER' <<<"$FLEET_COMPLETION_FUNCTION" ||
- fail "fleet completion probe does not fail loud on the explicit FAILED marker"
-completion_wait_line="$(grep -nF '"replacement HCC initial Host fleet reconciliation completion"' "$GATE" | cut -d: -f1)"
-c1_line="$(grep -nF 'scrape_hcc_metrics "$SCRAPE_C1" "$HCC_POD_C"' "$GATE" | head -n1 | cut -d: -f1)"
-if [[ -z "$completion_wait_line" || -z "$c1_line" ]] ||
- (( completion_wait_line >= c1_line )); then
- fail "replacement fleet completion is not causally established before post-recovery metrics"
-fi
-
-echo "PASS: host-storm oracle binds exact branch/profile ownership, shared fault locking, random endpoints, wake generation, and causal fleet completion"
diff --git a/scripts/tests/test-lint-networkpolicies.sh b/scripts/tests/test-lint-networkpolicies.sh
index 3153dd5dd..2400d8517 100755
--- a/scripts/tests/test-lint-networkpolicies.sh
+++ b/scripts/tests/test-lint-networkpolicies.sh
@@ -3,23 +3,7 @@ set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
TMP_DIR="$(mktemp -d)"
-
-# The minikube k8s-api-ip patch is a machine-specific, gitignored file rendered
-# at setup time. On a developer box with a live minikube it may already exist;
-# this test shadows it with a dummy render below. Save the real one (if any) and
-# restore it on EXIT so the test leaves the working tree exactly as it found it.
-MINIKUBE_API_IP_PATCH=""
-MINIKUBE_API_IP_PATCH_BACKUP=""
-MINIKUBE_API_IP_PATCH_PREEXISTING=0
-cleanup() {
- if [[ "${MINIKUBE_API_IP_PATCH_PREEXISTING}" -eq 1 ]]; then
- cp "${MINIKUBE_API_IP_PATCH_BACKUP}" "${MINIKUBE_API_IP_PATCH}"
- elif [[ -n "${MINIKUBE_API_IP_PATCH:-}" && -f "${MINIKUBE_API_IP_PATCH}" ]]; then
- rm -f "${MINIKUBE_API_IP_PATCH}"
- fi
- rm -rf "${TMP_DIR}"
-}
-trap cleanup EXIT
+trap 'rm -rf "${TMP_DIR}"' EXIT
write_policy() {
local path="$1"
@@ -78,7 +62,7 @@ write_policy "${TMP_DIR}/safe-specific.yaml" ' - ports:
protocol: TCP
to:
- ipBlock:
- cidr: 198.51.100.10/32'
+ cidr: 203.0.113.10/32'
run_lint "${TMP_DIR}/safe-specific.yaml" >/dev/null
echo "PASS: lint-networkpolicies allows explicit port plus specific destination"
@@ -151,30 +135,8 @@ ${PUBLIC_EXCEPT_BLOCK}"
run_lint "${TMP_DIR}/public-with-excepts.yaml" >/dev/null
echo "PASS: lint-networkpolicies allows public-only exception set with explicit ports"
-# The minikube overlay references patches/k8s-api-ip.yaml, which is rendered
-# from a gitignored template at setup time (deploy/scripts/minikube-detect-k8s-api-ip.sh)
-# and absent on a fresh checkout — so `kubectl kustomize` would abort with a
-# "no such file" evalsymlink error before ever reaching a lint verdict. Render
-# it inline here from the template with a placeholder CIDR, exactly as the
-# single-writer CI step does (.github/workflows/ci-public.yml), so this test
-# exercises the real rendered overlay instead of failing on a missing env file.
-MINIKUBE_API_IP_TEMPLATE="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml.template"
-MINIKUBE_API_IP_PATCH="${ROOT}/deploy/overlays/minikube/patches/k8s-api-ip.yaml"
-if [[ ! -f "${MINIKUBE_API_IP_TEMPLATE}" ]]; then
- echo "FAIL: missing ${MINIKUBE_API_IP_TEMPLATE}; cannot render the minikube k8s-api-ip patch" >&2
- exit 1
-fi
-if [[ -f "${MINIKUBE_API_IP_PATCH}" ]]; then
- # A developer's real rendered patch is present — preserve it so the dummy
- # render below doesn't clobber it; the EXIT trap restores it verbatim.
- MINIKUBE_API_IP_PATCH_BACKUP="${TMP_DIR}/k8s-api-ip.yaml.orig"
- cp "${MINIKUBE_API_IP_PATCH}" "${MINIKUBE_API_IP_PATCH_BACKUP}"
- MINIKUBE_API_IP_PATCH_PREEXISTING=1
-fi
-sed 's#__K8S_API_IP__#10.96.0.1#g' "${MINIKUBE_API_IP_TEMPLATE}" >"${MINIKUBE_API_IP_PATCH}"
-
-MINIKUBE_RENDERED="${TMP_DIR}/minikube-rendered.yaml"
-kubectl kustomize "${ROOT}/deploy/overlays/minikube" >"${MINIKUBE_RENDERED}"
+GCP_DEV_RENDERED="${TMP_DIR}/gcp-dev-rendered.yaml"
+kubectl kustomize "${ROOT}/deploy/overlays/gcp-dev" >"${GCP_DEV_RENDERED}"
RUBYOPT=--disable=gems ruby -ryaml -e '
forbidden = %w[allow-dns-egress-sandbox-ui deny-all-sandbox-ui]
required = %w[sandbox-ui-static-deny-all sandbox-ui-static-dns-egress]
@@ -189,6 +151,7 @@ RUBYOPT=--disable=gems ruby -ryaml -e '
found << name if forbidden.include?(name)
if name == "sandbox-ui-static-dns-egress"
saw_dns_peer = false
+ saw_dns_ipblock = false
Array(doc.dig("spec", "egress")).each_with_index do |rule, rule_idx|
ports = Array(rule["ports"]).map { |port| [port.fetch("protocol", "TCP"), port["port"]] }.sort
expected_ports = [["TCP", 53], ["UDP", 53]]
@@ -201,20 +164,27 @@ RUBYOPT=--disable=gems ruby -ryaml -e '
peers.each_with_index do |peer, peer_idx|
ns_name = peer.dig("namespaceSelector", "matchLabels", "kubernetes.io/metadata.name")
dns_selector = peer.dig("podSelector", "matchLabels", "k8s-app")
+ dns_ipblock = peer.dig("ipBlock", "cidr")
if ns_name == "kube-system" && dns_selector == "kube-dns"
saw_dns_peer = true
next
end
- abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods")
+ if dns_ipblock == "203.0.113.10/32"
+ saw_dns_ipblock = true
+ next
+ end
+
+ abort("sandbox-ui static DNS egress peer #{rule_idx}/#{peer_idx} must stay scoped to kube-system kube-dns pods or the example-dev kube-dns ClusterIP")
end
end
abort("sandbox-ui static DNS egress must include a kube-system kube-dns peer") unless saw_dns_peer
+ abort("sandbox-ui static DNS egress must include the example-dev kube-dns ClusterIP") unless saw_dns_ipblock
end
end
abort("sandbox-ui static slice must not render HCC-owned policies: #{found.join(", ")}") unless found.empty?
missing = required - names
abort("sandbox-ui static slice must render renamed baseline policies: #{missing.join(", ")}") unless missing.empty?
-' "${MINIKUBE_RENDERED}"
-echo "PASS: minikube sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and retain kube-dns-only DNS egress"
+' "${GCP_DEV_RENDERED}"
+echo "PASS: gcp-dev sandbox-ui static policies do not take HCC-owned deny-all/DNS ownership and include kube-dns ClusterIP"
diff --git a/scripts/tests/test-postgres-single-writer-strategy.sh b/scripts/tests/test-postgres-single-writer-strategy.sh
deleted file mode 100755
index 40174bf46..000000000
--- a/scripts/tests/test-postgres-single-writer-strategy.sh
+++ /dev/null
@@ -1,160 +0,0 @@
-#!/usr/bin/env bash
-# Regression test for deploy/scripts/validate-postgres-single-writer-strategy.sh
-#
-# Captures the [R4-D] fix: a Deployment this repo DEFINES (deploy/base/<ns>/
-# exists) that is ABSENT from a rendered overlay must FAIL LOUDLY, not be
-# NOTICE-and-skipped. Before the fix the validator skipped any absent deploy,
-# so a local overlay that stopped rendering control-plane/host-context-controller
-# validated the single-writer rollout invariant over an empty set and still
-# printed "validation passed" (a fail-open / false green).
-#
-# The CI step "Validate single-writer rollout invariant on rendered overlays"
-# already runs the validator against the real minikube overlay (a happy-path
-# check). This test is complementary: it fabricates the absent-deploy condition
-# in a hermetic fixture so the fail-loud behavior itself is under test.
-#
-# The test builds a hermetic fixture ROOT_DIR, copies the validator into it so
-# its ROOT_DIR ($(dirname)/../..) resolves to the fixture, and renders overlays
-# OFFLINE with `kubectl kustomize` — the same path the validator uses. No cluster.
-#
-# Fail-loud / fail-fast: every assertion fails the suite immediately, and the
-# reporter refuses to pass unless a non-zero number of assertions actually ran.
-set -uo pipefail
-
-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-VALIDATOR="${REPO_ROOT}/deploy/scripts/validate-postgres-single-writer-strategy.sh"
-
-for tool in kubectl yq jq; do
- command -v "${tool}" >/dev/null 2>&1 ||
- { echo "FAIL: required tool '${tool}' is missing" >&2; exit 1; }
-done
-[ -f "${VALIDATOR}" ] ||
- { echo "FAIL: validator not found at ${VALIDATOR}" >&2; exit 1; }
-
-TESTS_RUN=0
-FAILURES=0
-pass() { TESTS_RUN=$((TESTS_RUN + 1)); echo "ok - $1"; }
-fail() { TESTS_RUN=$((TESTS_RUN + 1)); FAILURES=$((FAILURES + 1)); echo "FAIL - $1" >&2; }
-
-WORK="$(mktemp -d "${TMPDIR:-/tmp}/single-writer-test.XXXXXX")"
-trap 'rm -rf "${WORK}"' EXIT
-
-ROOT="${WORK}/repo"
-mkdir -p "${ROOT}/deploy/scripts"
-cp "${VALIDATOR}" "${ROOT}/deploy/scripts/"
-FIXTURE_VALIDATOR="${ROOT}/deploy/scripts/$(basename "${VALIDATOR}")"
-
-# control-plane is a namespace this fixture repo DEFINES (base tree present).
-# registry is intentionally NOT defined here -> it is FOREIGN (NOTICE-and-skip).
-mkdir -p "${ROOT}/deploy/base/control-plane"
-: >"${ROOT}/deploy/base/control-plane/.keep"
-
-write_kustomization() { # dir, resource-file...
- local dir="$1"; shift
- {
- echo 'apiVersion: kustomize.config.k8s.io/v1beta1'
- echo 'kind: Kustomization'
- echo 'resources:'
- local r
- for r in "$@"; do echo " - ${r}"; done
- } >"${dir}/kustomization.yaml"
-}
-
-write_control_postgres() { # dir (valid stateful DB deploy: PVC + Recreate + replicas 1)
- cat >"$1/control-postgres.yaml" <<'YAML'
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: control-postgres
- namespace: control-plane
-spec:
- replicas: 1
- strategy:
- type: Recreate
- template:
- spec:
- volumes:
- - name: data
- persistentVolumeClaim:
- claimName: control-postgres-data
-YAML
-}
-
-write_hcc() { # dir (valid single-writer deploy: replicas 1 + Recreate)
- cat >"$1/host-context-controller.yaml" <<'YAML'
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: host-context-controller
- namespace: control-plane
-spec:
- replicas: 1
- strategy:
- type: Recreate
- template:
- spec:
- containers:
- - name: host-context-controller
- image: example/hcc:test
-YAML
-}
-
-# --- Overlay A: LOCAL single-writer deploy ABSENT (the fail-open trigger) ---
-# Renders a valid control-postgres but OMITS host-context-controller.
-OV_MISSING="${ROOT}/deploy/overlays/local-missing"
-mkdir -p "${OV_MISSING}"
-write_control_postgres "${OV_MISSING}"
-write_kustomization "${OV_MISSING}" control-postgres.yaml
-
-# --- Overlay B: every LOCAL deploy PRESENT and valid (foreign still skipped) ---
-OV_OK="${ROOT}/deploy/overlays/all-local-present"
-mkdir -p "${OV_OK}"
-write_control_postgres "${OV_OK}"
-write_hcc "${OV_OK}"
-write_kustomization "${OV_OK}" control-postgres.yaml host-context-controller.yaml
-
-run_validator() { # overlays-value ; prints combined output, returns exit code
- SINGLE_WRITER_OVERLAYS="$1" bash "${FIXTURE_VALIDATOR}" 2>&1
-}
-
-# 1) A locally-defined single-writer deploy absent from a rendered overlay MUST
-# fail loudly (this is exactly what the pre-fix NOTICE-and-skip let pass).
-out="$(run_validator "deploy/overlays/local-missing")"; rc=$?
-if [ "${rc}" -ne 0 ] && printf '%s' "${out}" |
- grep -q "renders no Deployment control-plane/host-context-controller"; then
- pass "absent locally-defined single-writer deploy fails loudly (exit ${rc})"
-else
- fail "absent local single-writer deploy did NOT fail loudly (exit ${rc}); output: ${out}"
-fi
-
-# 2) With every locally-defined deploy present and valid, a still-absent FOREIGN
-# deploy (registry/registry-postgres) is NOTICE-and-skipped, and the script
-# passes — the fix must not over-fail on foreign deploys.
-out="$(run_validator "deploy/overlays/all-local-present")"; rc=$?
-if [ "${rc}" -eq 0 ] &&
- printf '%s' "${out}" | grep -q "Single-writer rollout strategy validation passed" &&
- printf '%s' "${out}" | grep -q "does not render foreign Deployment registry/registry-postgres"; then
- pass "all local deploys present -> passes; absent foreign deploy is skipped"
-else
- fail "expected pass with foreign-skip (exit ${rc}); output: ${out}"
-fi
-
-# 3) Zero overlays must not pass vacuously.
-out="$(run_validator " ")"; rc=$?
-if [ "${rc}" -ne 0 ] && printf '%s' "${out}" | grep -q "zero overlays"; then
- pass "whitespace-only overlay set fails loudly (no vacuous pass)"
-else
- fail "empty overlay set did not fail loudly (exit ${rc}); output: ${out}"
-fi
-
-# --- Reporter (fail-loud / zero-tests-is-never-success) ---
-echo "1..${TESTS_RUN}"
-if [ "${TESTS_RUN}" -lt 3 ]; then
- echo "FAIL: expected at least 3 assertions, ran ${TESTS_RUN}" >&2
- exit 1
-fi
-if [ "${FAILURES}" -ne 0 ]; then
- echo "FAIL: ${FAILURES}/${TESTS_RUN} assertions failed" >&2
- exit 1
-fi
-echo "All ${TESTS_RUN} single-writer validator assertions passed"
diff --git a/scripts/tests/test-wrc-hcc-contracts-gate.sh b/scripts/tests/test-wrc-hcc-contracts-gate.sh
deleted file mode 100755
index a0d6180bc..000000000
--- a/scripts/tests/test-wrc-hcc-contracts-gate.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-set -u
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-GATE="${ROOT}/scripts/e2e/e2e-wrc-hcc-contracts.sh"
-FAIL=0
-
-pass() { echo "PASS: $1"; }
-fail() { echo "FAIL: $1"; FAIL=1; }
-
-if bash -n "$GATE"; then
- pass "WRC-HCC contracts gate has valid bash syntax"
-else
- fail "WRC-HCC contracts gate has invalid bash syntax"
-fi
-
-# The Phase 2 fixture has steps, so the admission policy requires an explicit
-# trigger. Keep this immediately alongside the fixture's agent/steps contract
-# so the E2E cannot regress into an admission-only false negative.
-phase_two_fixture="$(sed -n '/^header "Phase 2 - WRC toolsCalled args status contract"$/,/^YAML$/p' "$GATE")"
-if [[ "$phase_two_fixture" == *$' triggers:\n onDemand: {}'* ]] &&
- [[ "$phase_two_fixture" == *$' steps:'* ]]; then
- pass "WRC-HCC Phase 2 fixture declares its required on-demand trigger"
-else
- fail "WRC-HCC Phase 2 fixture can be rejected before its status contract runs"
-fi
-
-exit "$FAIL"
diff --git a/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh b/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
deleted file mode 100755
index 98f9a9177..000000000
--- a/scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-set -u
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-GATE="${ROOT}/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh"
-FAIL=0
-
-pass() { echo "PASS: $1"; }
-fail() { echo "FAIL: $1"; FAIL=1; }
-
-if bash -n "$GATE"; then
- pass "WRC internal-dependency gate has valid bash syntax"
-else
- fail "WRC internal-dependency gate has invalid bash syntax"
-fi
-
-# WRC persists its generated workload resource names in status before
-# materializing workloads. The gate must use that public CRD contract rather
-# than assuming a workload ID is also a Deployment or Service name.
-# shellcheck disable=SC2016
-if grep -Fq 'wait_for_workload_instance() {' "$GATE" &&
- grep -Fq 'SOURCE_DEPLOYMENT="$(wait_for_workload_instance "$SOURCE_ID"' "$GATE" &&
- grep -Fq 'BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID"' "$GATE" &&
- ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$SOURCE_ID"' "$GATE" &&
- ! grep -Fq 'wait_for_deployment "$SANDBOX_NS" "$BACKEND_ID"' "$GATE" &&
- grep -Fq 'deploy/${SOURCE_DEPLOYMENT}' "$GATE" &&
- grep -Fq '${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local' "$GATE"; then
- pass "WRC gate resolves persisted workload instances before runtime assertions"
-else
- fail "WRC gate assumes raw workload IDs are runtime resource names"
-fi
-
-exit "$FAIL"
diff --git a/tests/e2e/integration/network-policies.test.ts b/tests/e2e/integration/network-policies.test.ts
index 07925e9db..aba9302b2 100644
--- a/tests/e2e/integration/network-policies.test.ts
+++ b/tests/e2e/integration/network-policies.test.ts
@@ -8,50 +8,12 @@
* the NetworkPolicy is blocking traffic as expected. A successful response or
* connection refused (port not listening) would indicate a policy gap.
*/
-import { beforeAll, describe, expect, it } from 'vitest'
+import { describe, expect, it } from 'vitest'
import { execFileSync } from 'child_process'
// Short timeout for negative tests — we expect these to time out
const NP_TIMEOUT = 3
-// ─── Cluster reachability gate ───────────────────────────────────────────────
-// This suite drives kubectl against a live minikube cluster (Calico CNI + all
-// services deployed). Mirrors the opt-in-skip convention already used by the
-// sibling integration suites — channel-reader-via-api.test.ts and
-// mcpCredentialRotation.helpers.ts (E2E_SKIP_IF_CLUSTER_UNREACHABLE) plus
-// registry-pull-secret-runtime.test.ts (E2E_REQUIRE_CLUSTER / CI):
-// - Default (no flag): fail-fast when the cluster is unreachable. A silent
-// pass would hide a broken E2E setup, so the suite must NOT be neutralized
-// unconditionally.
-// - E2E_SKIP_IF_CLUSTER_UNREACHABLE=1: opt in to a clean skip for the tier
-// that runs integration WITHOUT a cluster (T1).
-// - E2E_REQUIRE_CLUSTER=1 or CI=true: force a hard failure even if the skip
-// flag is also set — CI must never skip.
-const SKIP_IF_UNREACHABLE = process.env.E2E_SKIP_IF_CLUSTER_UNREACHABLE === '1'
-const REQUIRE_CLUSTER = process.env.E2E_REQUIRE_CLUSTER === '1' || process.env.CI === 'true'
-
-let clusterReachable = false
-
-/**
- * One-shot reachability probe: succeeds only when kubectl can talk to the API
- * server. This is a precondition probe (the established sibling idiom —
- * registry-pull-secret-runtime.test.ts's `kubectlOrNull`), NOT an error
- * swallow: its boolean result drives the fail-hard-or-skip decision in
- * `beforeAll`, which stays loud in the default and CI paths.
- */
-function probeClusterReachable(): boolean {
- try {
- execFileSync('kubectl', ['get', 'ns', 'default', '-o', 'name'], {
- encoding: 'utf-8',
- timeout: 15_000,
- stdio: ['ignore', 'pipe', 'pipe'],
- })
- return true
- } catch {
- return false
- }
-}
-
interface ConnectResult {
reachable: boolean
statusCode?: number
@@ -179,32 +141,8 @@ function testConnectivity(
}
describe('NetworkPolicy Enforcement (Calico)', () => {
- beforeAll(() => {
- clusterReachable = probeClusterReachable()
- if (clusterReachable) return
-
- const msg =
- '[network-policies] kubernetes cluster not reachable (kubectl get ns default failed)'
- if (REQUIRE_CLUSTER) {
- throw new Error(
- `${msg}. Refusing to skip: E2E_REQUIRE_CLUSTER/CI is set, and a silent skip here ` +
- 'would report success while asserting nothing about NetworkPolicy enforcement. ' +
- 'Bring up the cluster (e.g. `make minikube-setup`) or unset the flag.'
- )
- }
- if (SKIP_IF_UNREACHABLE) {
- console.log(`${msg} — tests will be skipped (E2E_SKIP_IF_CLUSTER_UNREACHABLE=1)`)
- return
- }
- throw new Error(
- `${msg}. This E2E suite requires a minikube cluster with Calico CNI and all services ` +
- 'deployed. Run `make minikube-setup` first, or set E2E_SKIP_IF_CLUSTER_UNREACHABLE=1 to skip.'
- )
- })
-
// ─── Prerequisite Check ──────────────────────────────────────────────
- it('Calico CNI is active', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('Calico CNI is active', () => {
const pods = execFileSync(
'kubectl',
['get', 'pods', '-n', 'kube-system', '-l', 'k8s-app=calico-node', '--no-headers'],
@@ -213,8 +151,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(pods).toContain('Running')
})
- it('deny-all policies exist in secured namespaces', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('deny-all policies exist in secured namespaces', () => {
const namespaces = ['mcp-host', 'mcp-server', 'rpc-proxy', 'sandbox-recipes']
for (const ns of namespaces) {
let nps: string
@@ -247,8 +184,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
// ─── POSITIVE TESTS (Allowed Traffic) ────────────────────────────────
describe('Allowed traffic (positive)', () => {
- it('control-api → registry-api:8085 (cross-namespace, registry access)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('control-api → registry-api:8085 (cross-namespace, registry access)', () => {
const result = testConnectivity(
'control-plane',
'control-api',
@@ -257,8 +193,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.reachable).toBe(true)
})
- it('control-ui → control-api:8090 (UI backend path)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('control-ui → control-api:8090 (UI backend path)', () => {
const result = testConnectivity(
'control-plane',
'control-ui',
@@ -267,8 +202,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.reachable).toBe(true)
})
- it('external-rest-api → profile-control-funnel:8080 (internal control path)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('external-rest-api → profile-control-funnel:8080 (internal control path)', () => {
const result = testConnectivity(
'profiles',
'external-rest-api',
@@ -277,14 +211,12 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.reachable).toBe(true)
})
- it('rpc-proxy → mcp-host:8080 (message relay)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('rpc-proxy → mcp-host:8080 (message relay)', () => {
const result = testConnectivity('rpc-proxy', 'rpc-proxy', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(true)
})
- it('mcp-host can reach the HCC gateway readiness endpoint', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('mcp-host can reach the HCC gateway readiness endpoint', () => {
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -298,8 +230,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
// ─── NEGATIVE TESTS (Blocked by Design) ──────────────────────────────
describe('Blocked traffic (negative — deny-all enforced)', () => {
- it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('mcp-host → rpc-proxy:8094 (reverse direction blocked)', () => {
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -309,8 +240,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('mcp-host → control-api:8090 (no direct access to control-plane)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('mcp-host → control-api:8090 (no direct access to control-plane)', () => {
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -320,12 +250,11 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('channel-reader → workflow approval gateway:8092 (must go through mcp-host)', () => {
const podName = getChannelReaderPodName()
if (!podName) {
console.warn('SKIPPED: no running channel-reader pod in channels')
- ctx.skip()
+ return
}
const result = testConnectivity(
@@ -337,8 +266,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('mcp-host → registry-api:8085 (no direct registry access)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('mcp-host → registry-api:8085 (no direct registry access)', () => {
const result = testConnectivity(
'mcp-host',
getMcpHostRuntimeName(),
@@ -348,8 +276,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('control-api → rpc-proxy:8094 (no direct control-plane hop)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('control-api → rpc-proxy:8094 (no direct control-plane hop)', () => {
const result = testConnectivity(
'control-plane',
'control-api',
@@ -359,8 +286,7 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('external-rest-api → rpc-proxy:8094 (desktop app reaches rpc-proxy directly, not via backend)', () => {
const result = testConnectivity(
'profiles',
'external-rest-api',
@@ -370,22 +296,19 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('control-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {
const result = testConnectivity('control-plane', 'control-api', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(false)
expect(result.error).toContain('timeout')
})
- it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('external-rest-api → mcp-host:8080 directly (must go through rpc-proxy)', () => {
const result = testConnectivity('profiles', 'external-rest-api', getMcpHostRuntimeUrl())
expect(result.reachable).toBe(false)
expect(result.error).toContain('timeout')
})
- it('mcp-proxy → external-rest-api:3000 (no reverse path)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('mcp-proxy → external-rest-api:3000 (no reverse path)', () => {
const result = testConnectivity(
'mcp-server',
'mcp-proxy',
@@ -425,12 +348,11 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
}
}
- it('sandbox-recipes → control-api:8090 (isolated sandbox)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('sandbox-recipes → control-api:8090 (isolated sandbox)', () => {
const podName = findExecPod('sandbox-recipes', 'app=db')
if (!podName) {
console.warn('SKIPPED: no running db pod in sandbox-recipes')
- ctx.skip()
+ return
}
// sandbox-recipes has deny-all; only DNS + HCC API egress allowed
const result = testConnectivity(
@@ -442,12 +364,11 @@ describe('NetworkPolicy Enforcement (Calico)', () => {
expect(result.error).toContain('timeout')
})
- it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', (ctx) => {
- if (!clusterReachable) ctx.skip()
+ it('sandbox-recipes → rpc-proxy:8094 (isolated sandbox)', () => {
const podName = findExecPod('sandbox-recipes', 'app=db')
if (!podName) {
console.warn('SKIPPED: no running db pod in sandbox-recipes')
- ctx.skip()
+ return
}
const result = testConnectivity(
'sandbox-recipes',
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index 9981d0aa7..558c31760 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.65",
+ "version": "0.1.66",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.65",
+ "version": "0.1.66",
"devDependencies": {
"@types/jsonwebtoken": "^9.0.10",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index 63c62d203..19145d619 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.65",
+ "version": "0.1.66",
"private": true,
"type": "module",
"scripts": {