Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): add the HCC watch-churn readiness gate (minikube reproducer)

Alfredo Lopez4db02c06946d
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 08541e5bc..f318ba33a 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -390,6 +390,9 @@ jobs:
- name: Test HCC watch-recovery gate coordination
run: bash scripts/tests/test-hcc-watch-recovery-gate.sh
+ - name: Test HCC watch-churn readiness gate contract
+ run: bash scripts/tests/test-hcc-watch-churn-readiness-gate.sh
+
- name: Test HCC API gateway readiness separation
run: bash scripts/tests/test-hcc-api-gateway-readiness.sh
diff --git a/Makefile b/Makefile
index 107633d70..7334f00fc 100644
--- a/Makefile
+++ b/Makefile
@@ -943,6 +943,12 @@ test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host
@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
E2E_HCC_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-readiness-bootstrap.sh
+.PHONY: test-e2e-hcc-watch-churn-readiness
+test-e2e-hcc-watch-churn-readiness: ## Prove HCC readiness survives sustained apiserver watch churn (PR #205 GKE livelock). EXPECT_LIVELOCK=1 reproduces the bug.
+ @echo "Running HCC watch-churn readiness gate..."
+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
+ E2E_HCC_WATCH_FAULT_INJECTION=1 EXPECT_LIVELOCK=$(EXPECT_LIVELOCK) E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+
.PHONY: test-e2e-hcc-mcp-context-readiness
test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Context/NetworkPolicy initial convergence
@echo "Running HCC MCP/Context/NetworkPolicy readiness gate..."
diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
new file mode 100755
index 000000000..c62777dee
--- /dev/null
+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+# Helpers for the HCC watch-churn readiness gate: a self-flapping Kubernetes API
+# TCP proxy that periodically resets long-lived (watch) connections, a synthetic
+# clerum-dev-scale CR fleet, and a non-destructive HCC redirect/restore.
+#
+# The proxy runs the HCC image (already loaded in the node), needs no privileges,
+# and — unlike scale_proxy in the recovery gate — cuts connections on its own
+# clock, so it reproduces GKE's sustained `Premature close` regime rather than a
+# single manual outage. NetworkPolicy forces HCC's ONLY apiserver egress through
+# it, so the cuts hit exactly the HCC watches and nothing else.
+#
+# Variables (PROXY_NAME, HCC_NS, HCC_DEPLOY, HCC_IMAGE, MCP_NS, HOST_NS,
+# FLEET_PREFIX, FLEET_SECRET, RUN_ID, PROXY_EGRESS_NP, HCC_PROXY_NP, and the
+# *_CREATED flags) are owned by the sourcing gate; this file only defines helpers.
+
+# shellcheck disable=SC2034
+CHURN_PROXY_PORT=8443
+
+create_hcc_churn_proxy() {
+ # $1 = churn period ms (cut cadence), $2 = min connection age ms before a cut.
+ local churn_period_ms=$1 churn_min_age_ms=$2
+ PROXY_CREATED=1
+ kctl apply -f - >/dev/null <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ${PROXY_NAME}
+ namespace: ${HCC_NS}
+ labels: {e2e.clerum.io/suite: hcc-watch-churn}
+spec:
+ replicas: 1
+ selector: {matchLabels: {app: ${PROXY_NAME}}}
+ template:
+ metadata: {labels: {app: ${PROXY_NAME}, e2e.clerum.io/suite: hcc-watch-churn}}
+ spec:
+ automountServiceAccountToken: false
+ terminationGracePeriodSeconds: 1
+ containers:
+ - name: proxy
+ image: ${HCC_IMAGE}
+ imagePullPolicy: IfNotPresent
+ env:
+ - {name: CHURN_PERIOD_MS, value: "${churn_period_ms}"}
+ - {name: CHURN_MIN_AGE_MS, value: "${churn_min_age_ms}"}
+ command: [node, -e]
+ args:
+ # A relay to kubernetes.default.svc:443 that tracks every live upstream
+ # socket with its birth timestamp, and every CHURN_PERIOD_MS destroys the
+ # ones older than CHURN_MIN_AGE_MS. Short requests (LIST/GET/POST, ms)
+ # complete untouched; long-lived watches (>min age) are reset — exactly
+ # GKE's balancer idle-timeout behaviour. The listener never dies, so HCC
+ # reconnects, re-LISTs, and bumps its watch generation on every cut.
+ - >-
+ const net=require('net');
+ const period=Number(process.env.CHURN_PERIOD_MS);
+ const minAge=Number(process.env.CHURN_MIN_AGE_MS);
+ const live=new Set();
+ net.createServer(c=>{
+ const born=Date.now();
+ const u=net.connect(443,'kubernetes.default.svc');
+ const rec={c,u,born};live.add(rec);
+ const close=()=>{live.delete(rec);c.destroy();u.destroy()};
+ c.on('error',close);u.on('error',close);c.on('close',close);u.on('close',close);
+ c.pipe(u);u.pipe(c);
+ }).listen(${CHURN_PROXY_PORT},'0.0.0.0');
+ setInterval(()=>{const now=Date.now();
+ for(const r of [...live]){ if(now-r.born>=minAge){ r.c.destroy();r.u.destroy();live.delete(r);} }
+ },period);
+ readinessProbe: {tcpSocket: {port: ${CHURN_PROXY_PORT}}, periodSeconds: 1}
+ resources:
+ requests: {cpu: 10m, memory: 24Mi}
+ limits: {cpu: 100m, memory: 64Mi}
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities: {drop: [ALL]}
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+ runAsUser: 1000
+ runAsGroup: 1000
+ seccompProfile: {type: RuntimeDefault}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ${PROXY_NAME}
+ namespace: ${HCC_NS}
+ labels: {e2e.clerum.io/suite: hcc-watch-churn}
+spec:
+ selector: {app: ${PROXY_NAME}}
+ ports: [{name: tls, port: 443, targetPort: ${CHURN_PROXY_PORT}, protocol: TCP}]
+EOF
+
+ # Clone the control-plane API egress policy for the proxy, and restrict proxy
+ # ingress to the HCC pod only (identical isolation to the recovery gate).
+ kctl get networkpolicy allow-k8s-api-egress-control-plane -n "$HCC_NS" -o json |
+ jq --arg name "$PROXY_EGRESS_NP" --arg app "$PROXY_NAME" '
+ del(.metadata.creationTimestamp, .metadata.generation, .metadata.resourceVersion,
+ .metadata.uid, .metadata.managedFields) |
+ .metadata.name = $name |
+ .metadata.labels = {"e2e.clerum.io/suite": "hcc-watch-churn"} |
+ .spec.podSelector = {matchLabels: {app: $app}} |
+ .spec.policyTypes = ["Ingress", "Egress"] |
+ .spec.ingress = [{from: [{podSelector: {matchLabels: {app: "host-context-controller"}}}],
+ ports: [{port: '"${CHURN_PROXY_PORT}"', protocol: "TCP"}]}]' |
+ kctl apply -f - >/dev/null
+
+ kctl apply -f - >/dev/null <<EOF
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${HCC_PROXY_NP}
+ namespace: ${HCC_NS}
+ labels: {e2e.clerum.io/suite: hcc-watch-churn}
+spec:
+ podSelector: {matchLabels: {app: host-context-controller}}
+ policyTypes: [Egress]
+ egress:
+ - to: [{podSelector: {matchLabels: {app: ${PROXY_NAME}}}}]
+ ports:
+ - {port: 443, protocol: TCP}
+ - {port: ${CHURN_PROXY_PORT}, protocol: TCP}
+EOF
+ kctl rollout status deployment "$PROXY_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||
+ die "churn proxy did not become ready"
+}
+
+# create_synthetic_fleet CONTEXTS MCPSERVERS HOSTS — HCC must be scaled to 0 first.
+create_synthetic_fleet() {
+ local n_ctx=$1 n_mcp=$2 n_host=$3 i ctx applied
+ FLEET_CREATED=1
+
+ # LLM secret shared by all synthetic Hosts (label-scoped for cleanup).
+ kctl create secret generic "$FLEET_SECRET" -n "$HOST_NS" \
+ --from-literal=api-key=e2e-churn --dry-run=client -o yaml |
+ kctl label --local -f - e2e.clerum.io/suite=hcc-watch-churn -o yaml |
+ kctl apply -f - >/dev/null
+
+ # Contexts (one stream).
+ { for ((i = 1; i <= n_ctx; i++)); do
+ ctx="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" "$i")"
+ cat <<EOF
+---
+apiVersion: clerum.io/v1alpha1
+kind: Context
+metadata: {name: ${ctx}, namespace: ${MCP_NS}, labels: {e2e.clerum.io/suite: hcc-watch-churn, e2e.clerum.io/run: "${RUN_ID}"}}
+spec: {contextId: ${ctx}, description: "churn fleet context", mcpServers: []}
+EOF
+ done; } | kctl apply -f - >/dev/null
+
+ # McpServers (one stream) — inventory + NetworkPolicies, no heavy pods.
+ { for ((i = 1; i <= n_mcp; i++)); do
+ ctx="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" $(((i % n_ctx) + 1)))"
+ cat <<EOF
+---
+apiVersion: clerum.io/v1alpha1
+kind: McpServer
+metadata: {name: $(printf '%s-mcp-%03d' "$FLEET_PREFIX" "$i"), namespace: ${MCP_NS}, labels: {e2e.clerum.io/suite: hcc-watch-churn, e2e.clerum.io/run: "${RUN_ID}"}}
+spec:
+ contextRef: ${ctx}
+ description: "churn fleet mcp server"
+ image: ${HCC_IMAGE}
+ imagePullPolicy: Never
+ transport: {type: streamableHttp, url: "http://127.0.0.1:3000/mcp", port: 3000}
+EOF
+ done; } | kctl apply -f - >/dev/null
+
+ # Hosts (one stream) — these materialize Deployments; keep the count modest.
+ { for ((i = 1; i <= n_host; i++)); do
+ cat <<EOF
+---
+apiVersion: clerum.io/v1alpha1
+kind: Host
+metadata: {name: $(printf '%s-host-%02d' "$FLEET_PREFIX" "$i"), namespace: ${HOST_NS}, labels: {e2e.clerum.io/suite: hcc-watch-churn, e2e.clerum.io/run: "${RUN_ID}"}}
+spec:
+ host: $(printf '%s-host-%02d' "$FLEET_PREFIX" "$i")
+ contextRef: $(printf '%s-ctx-%03d' "$FLEET_PREFIX" 1)
+ secretRef: ${FLEET_SECRET}
+ model: {provider: openai, name: e2e-churn}
+EOF
+ done; } | kctl apply -f - >/dev/null
+
+ # Fail-loud count check: refuse to proceed on a partial fleet.
+ applied="$(kctl get mcpserver -n "$MCP_NS" -l e2e.clerum.io/run="$RUN_ID" -o name | wc -l | tr -d ' ')"
+ [ "$applied" = "$n_mcp" ] || die "synthetic fleet incomplete: applied ${applied}/${n_mcp} McpServers"
+}
+
+delete_synthetic_fleet() {
+ local failed=0 kind
+ for kind in mcpserver context host; do
+ kctl delete "$kind" -A -l e2e.clerum.io/suite=hcc-watch-churn \
+ --ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 || failed=1
+ done
+ kctl delete secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found >/dev/null 2>&1 || failed=1
+ # HCC-managed runtime left behind by the fleet Hosts.
+ kctl delete deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \
+ -A -l "clerum.io/managed-by=host-context-controller" --ignore-not-found >/dev/null 2>&1 || true
+ return "$failed"
+}
+
+# Non-destructive restore: undo the env overrides and hostAliases redirect.
+restore_hcc_after_churn() {
+ kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1
+ kctl patch deployment/"$HCC_DEPLOY" -n "$HCC_NS" --type=merge \
+ -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1
+ [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] || return 1
+}
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
new file mode 100755
index 000000000..fd4c37438
--- /dev/null
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -0,0 +1,277 @@
+#!/usr/bin/env bash
+# HCC readiness under SUSTAINED apiserver watch churn (Premature close regime).
+#
+# Reproduces the PR #205 GKE livelock in minikube — a regime minikube does NOT
+# exhibit by default (single stable apiserver, no balancer idle-timeout). A
+# clerum-dev-scale synthetic fleet widens each NetworkPolicy revocation pass; a
+# self-flapping API proxy resets the HCC watches mid-pass, bumping the watch
+# generation so a generation-pinned isReadinessInventoryAuthoritative() could
+# never certify. The content-identity fix must certify through the churn.
+#
+# EXPECT_LIVELOCK=1 -> proves the reproducer: /ready stays 503 + divergent
+# generation pattern (guards against a vacuous test).
+# EXPECT_LIVELOCK=0 -> proves the fix: /ready reaches 200 (1/1) under the
+# SAME churn, and stays 200 through STABILITY_WINDOW_SEC.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+source "${SCRIPT_DIR}/e2e-lib.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/hcc-watch-churn-fixture.sh"
+
+# ── Fail-closed guards (identical contract to the sibling HCC gates) ──
+[ -n "$E2E_KUBECONTEXT" ] || {
+ echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2
+ exit 1
+}
+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
+ echo "Refusing churn injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
+ exit 1
+}
+require_safe_kube_context
+command -v jq >/dev/null 2>&1 || {
+ echo "jq is required" >&2
+ exit 1
+}
+[ "${E2E_HCC_WATCH_FAULT_INJECTION:-0}" = 1 ] || {
+ echo "Set E2E_HCC_WATCH_FAULT_INJECTION=1 to acknowledge fault injection." >&2
+ exit 1
+}
+kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $c)' >/dev/null ||
+ {
+ echo "Refusing churn injection: target is not this profile's minikube node." >&2
+ exit 1
+ }
+
+# ── Tunables ──
+HCC_NS="${HCC_NS:-control-plane}"
+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
+MCP_NS="${MCP_SERVER_NS:-mcp-server}"
+HOST_NS="${MCP_HOST_NS:-mcp-host}"
+CHURN_PERIOD_MS="${CHURN_PERIOD_MS:-7000}"
+CHURN_MIN_AGE_MS="${CHURN_MIN_AGE_MS:-3000}"
+CHURN_MIN_CUTS="${CHURN_MIN_CUTS:-3}"
+READINESS_BUDGET_SEC="${READINESS_BUDGET_SEC:-360}"
+STABILITY_WINDOW_SEC="${STABILITY_WINDOW_SEC:-30}"
+FLEET_CONTEXTS="${FLEET_CONTEXTS:-24}"
+FLEET_MCPSERVERS="${FLEET_MCPSERVERS:-115}"
+FLEET_HOSTS="${FLEET_HOSTS:-8}"
+EXPECT_LIVELOCK="${EXPECT_LIVELOCK:-0}"
+
+RUN_ID="$(date +%s)-$$"
+PROXY_NAME="$(truncate_rfc1123 "e2e-hcc-churn-proxy-${RUN_ID}")"
+PROXY_EGRESS_NP="$(truncate_rfc1123 "${PROXY_NAME}-api")"
+HCC_PROXY_NP="$(truncate_rfc1123 "${PROXY_NAME}-from-hcc")"
+FLEET_PREFIX="$(truncate_rfc1123 "e2e-hcc-churn-${RUN_ID}")"
+FLEET_SECRET="$(truncate_rfc1123 "${FLEET_PREFIX}-llm")"
+LOG_ARTIFACT="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn.XXXXXX")"
+ORIGINAL_REPLICAS=""
+HCC_IMAGE=""
+K8S_API_CIDR=""
+PROXY_CREATED=0
+FLEET_CREATED=0
+HCC_PATCHED=0
+# shellcheck disable=SC2034
+HCC_GATE_LOCK_ACQUIRED=0 HCC_GATE_LOCK_NAME="" HCC_GATE_LOCK_UID="" HCC_GATE_FINALIZATION_FAILURE=""
+
+die() {
+ fail "$*"
+ exit 1
+}
+
+wait_until() {
+ local timeout=$1 description=$2
+ shift 2
+ local deadline now
+ deadline=$(($(date +%s) + timeout))
+ while :; do
+ "$@" && return 0
+ now="$(date +%s)"
+ [ "$now" -lt "$deadline" ] || break
+ sleep 1
+ done
+ echo "Timed out after ${timeout}s waiting for ${description}" >&2
+ return 1
+}
+
+running_hcc_pod() {
+ local rows
+ rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
+ --field-selector=status.phase=Running \
+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \
+ 2>/dev/null)" || return 1
+ awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"
+}
+
+hcc_pods_absent() {
+ local pods
+ pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" || return 1
+ [ -z "$pods" ]
+}
+
+ready_status() {
+ local pod=$1
+ kctl exec "pod/$pod" -n "$HCC_NS" -c host-context-controller -- \
+ wget -T 10 -t 1 -qO- http://127.0.0.1:8081/ready >/dev/null 2>&1 && echo 200 || echo 503
+}
+
+print_repair_instructions() {
+ cat >&2 <<EOF
+HCC watch-churn gate cleanup could not restore a verified clean state.
+Context: ${E2E_KUBECONTEXT}
+HCC: ${HCC_NS}/${HCC_DEPLOY}
+Churn proxy: ${HCC_NS}/${PROXY_NAME}
+Synthetic fleet label: e2e.clerum.io/suite=hcc-watch-churn (run ${RUN_ID})
+
+Inspect before changing anything:
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/suite=hcc-watch-churn
+
+Restore the HCC deployment (remove redirect env + hostAliases, restore replicas):
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} \\
+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS-
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} patch deployment/${HCC_DEPLOY} --type=merge \\
+ -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}'
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s
+EOF
+}
+
+cleanup() {
+ local status=$? cleanup_failed=0 restore_ok=1
+ trap - EXIT
+ set +e
+ if [ "$HCC_PATCHED" = 1 ]; then
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1
+ wait_until 120 "HCC stop" hcc_pods_absent >/dev/null 2>&1 || restore_ok=0
+ fi
+ [ "$FLEET_CREATED" = 1 ] && { delete_synthetic_fleet || cleanup_failed=1; }
+ if [ "$HCC_PATCHED" = 1 ]; then
+ restore_hcc_after_churn || restore_ok=0
+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1
+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1 || restore_ok=0
+ fi
+ [ "$PROXY_CREATED" = 1 ] && kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1
+ kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1
+ [ "$restore_ok" = 1 ] || {
+ print_repair_instructions
+ cleanup_failed=1
+ }
+ finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
+ print_results || status=1
+ [ "$cleanup_failed" = 0 ] || status=1
+ exit "$status"
+}
+trap cleanup EXIT
+
+# ── FASE A: guards + snapshot ──
+require_branch_owned_hcc_gate "$HCC_NS"
+acquire_hcc_watch_gate_lock
+ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"
+[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
+[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||
+ die "HCC already has hostAliases; refusing a non-restorable injection"
+HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
+[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"
+api_ip="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \
+ printenv KUBERNETES_SERVICE_HOST)" || die "could not read API service IP"
+K8S_API_CIDR="${api_ip}/32"
+
+# ── FASE B: redirect through the self-flapping proxy (HCC still healthy) ──
+log "Creating self-flapping API proxy (period=${CHURN_PERIOD_MS}ms, min-age=${CHURN_MIN_AGE_MS}ms)"
+create_hcc_churn_proxy "$CHURN_PERIOD_MS" "$CHURN_MIN_AGE_MS"
+verify_hcc_proxy_network_policy
+proxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"
+[ -n "$proxy_ip" ] || die "proxy Service has no ClusterIP"
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
+wait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop before fleet creation"
+
+# ── FASE C: synthetic clerum-dev-scale fleet (HCC stopped) ──
+log "Creating synthetic fleet: ${FLEET_CONTEXTS} Contexts / ${FLEET_MCPSERVERS} McpServers / ${FLEET_HOSTS} Hosts"
+create_synthetic_fleet "$FLEET_CONTEXTS" "$FLEET_MCPSERVERS" "$FLEET_HOSTS"
+ok "synthetic fleet created (${FLEET_MCPSERVERS} McpServers)"
+
+# ── FASE D: start HCC under churn and poll /ready ──
+log "Redirecting HCC through the churn proxy and starting it under load"
+patch="$(jq -cn --arg ip "$proxy_ip" --arg cidr "$K8S_API_CIDR" '{spec:{template:{spec:{
+ hostAliases:[{ip:$ip,hostnames:["kubernetes.default.svc"]}],
+ containers:[{name:"host-context-controller",env:[
+ {name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},
+ {name:"KUBERNETES_SERVICE_PORT",value:"443"},
+ {name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$cidr}]}]}}}}')"
+kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null
+HCC_PATCHED=1
+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
+start_hcc_recovery_log_stream
+
+deadline=$(($(date +%s) + READINESS_BUDGET_SEC))
+first_ready=""
+last_status=""
+while [ "$(date +%s)" -lt "$deadline" ]; do
+ pod="$(running_hcc_pod)" || {
+ sleep 2
+ continue
+ }
+ [ -n "$pod" ] || {
+ sleep 2
+ continue
+ }
+ last_status="$(ready_status "$pod")"
+ if [ "$last_status" = 200 ]; then
+ first_ready="$((READINESS_BUDGET_SEC - (deadline - $(date +%s))))"
+ break
+ fi
+ sleep 3
+done
+cuts="$(hcc_recovery_logs | grep -Ec 'watch ended|authority changed|recovering authoritative inventory' || true)"
+gen_pattern="$(hcc_recovery_logs | grep -Ec 'authority changed before .* admission' || true)"
+{
+ echo "=== HCC deploy ==="
+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide
+ echo "=== last /ready: ${last_status} cuts=${cuts} gen-diverge=${gen_pattern} first-ready=${first_ready:-never} ==="
+ hcc_recovery_logs | grep -E 'watch ended|authority changed|Recovered|inventory' | tail -60
+} >"$LOG_ARTIFACT" 2>&1
+
+# ── Verdict ──
+if [ "$EXPECT_LIVELOCK" = 1 ]; then
+ [ "$cuts" -ge "$CHURN_MIN_CUTS" ] && ok "churn cut ${cuts} watch(es) (>= ${CHURN_MIN_CUTS})" ||
+ fail "churn cut only ${cuts} watch(es) — reproducer is VACUOUS"
+ [ "$gen_pattern" -ge 1 ] && ok "observed divergent-generation pattern ${gen_pattern}x" ||
+ fail "no divergent-generation pattern — bug mechanism NOT exercised"
+ [ -z "$first_ready" ] && ok "livelock reproduced: /ready stayed 503 for ${READINESS_BUDGET_SEC}s under churn" ||
+ fail "expected livelock but /ready reached 200 — reproducer does not exercise the bug"
+else
+ [ "$cuts" -ge "$CHURN_MIN_CUTS" ] && ok "fix validated UNDER churn (${cuts} cuts)" ||
+ fail "fix validated WITHOUT churn (${cuts} cuts) — result is worthless"
+ if [ -n "$first_ready" ]; then
+ ok "fix converged to Ready under sustained Premature close (time-to-Ready=${first_ready}s, cuts=${cuts})"
+ stable=1
+ s_deadline=$(($(date +%s) + STABILITY_WINDOW_SEC))
+ while [ "$(date +%s)" -lt "$s_deadline" ]; do
+ pod="$(running_hcc_pod)" || {
+ stable=0
+ break
+ }
+ [ "$(ready_status "$pod")" = 200 ] || {
+ stable=0
+ break
+ }
+ sleep 3
+ done
+ [ "$stable" = 1 ] && ok "readiness stable for ${STABILITY_WINDOW_SEC}s under continuous churn" ||
+ fail "readiness oscillated 200->503 under churn — fix not stable"
+ else
+ fail "fix did NOT reach Ready under churn within ${READINESS_BUDGET_SEC}s — livelock regression (see ${LOG_ARTIFACT})"
+ fi
+fi
+stop_hcc_recovery_log_stream
+log "Evidence artifact: ${LOG_ARTIFACT}"
+# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.
diff --git a/scripts/tests/test-hcc-watch-churn-readiness-gate.sh b/scripts/tests/test-hcc-watch-churn-readiness-gate.sh
new file mode 100755
index 000000000..09fd2004f
--- /dev/null
+++ b/scripts/tests/test-hcc-watch-churn-readiness-gate.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Offline contract gate for the HCC watch-churn readiness e2e. Runs in CI: no
+# cluster. Proves the gate keeps its fail-loud, branch-scoped, restorable shape
+# and its anti-vacuity assertions, so a refactor cannot quietly gut the one test
+# that reproduces the clerum-dev livelock under real watch churn.
+set -u
+FAIL=0
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+GATE="${ROOT}/scripts/e2e/e2e-hcc-watch-churn-readiness.sh"
+FIXTURE="${ROOT}/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"
+pass() { echo "PASS: $1"; }
+fail() {
+ echo "FAIL: $1"
+ FAIL=1
+}
+
+for s in "$GATE" "$FIXTURE"; do
+ bash -n "$s" && pass "$(basename "$s") valid bash" || fail "$(basename "$s") invalid bash"
+done
+
+# 1. Anti-vacuity: both regimes assert a minimum number of real watch cuts.
+[ "$(grep -Ec 'cuts.*-ge.*CHURN_MIN_CUTS' "$GATE")" -ge 2 ] &&
+ pass "both livelock and fix regimes assert >= CHURN_MIN_CUTS real cuts" ||
+ fail "a regime can pass without proving churn actually cut the watches"
+
+# 2. Livelock regime demands the divergent-generation evidence.
+grep -Fq 'authority changed before .* admission' "$GATE" &&
+ pass "livelock regime keys on the real generation-divergence log" ||
+ fail "livelock regime does not check the generation-divergence signal"
+
+# 3. Fix regime demands sustained stability, not a transient 200.
+grep -Fq 'readiness oscillated 200->503' "$GATE" &&
+ pass "fix regime asserts sustained readiness under continuous churn" ||
+ fail "fix regime accepts a transient 200 between cuts"
+
+# 4. Branch-scoped + fault-injection ack + single-writer + restorable, like siblings.
+grep -Fq 'is_branch_scoped_e2e_context' "$GATE" &&
+ grep -Fq 'E2E_HCC_WATCH_FAULT_INJECTION' "$GATE" &&
+ grep -Fq 'require_branch_owned_hcc_gate' "$GATE" &&
+ grep -Fq 'ORIGINAL_REPLICAS' "$GATE" &&
+ grep -Fq 'restore_hcc_after_churn' "$GATE" &&
+ pass "gate keeps branch-scope, ack, single-writer and restore contract" ||
+ fail "gate drops a branch-scope / ack / single-writer / restore guard"
+
+# 5. Churn cadence is parameterized with an explicit default.
+grep -Eq 'CHURN_PERIOD_MS:-[0-9]{3,5}' "$GATE" &&
+ pass "churn cadence is parameterized with an explicit default" ||
+ fail "churn cadence is not parameterized"
+
+# 6. The node label is bound to the explicit profile (no non-null wildcard).
+grep -Fq 'minikube.k8s.io/name"] == $c' "$GATE" &&
+ ! grep -Fq 'minikube.k8s.io/name"] != null' "$GATE" &&
+ pass "gate binds the minikube node label to the explicit profile" ||
+ fail "gate can accept a node owned by another profile"
+
+# 7. The single-writer invariant is enforced (exactly one replica).
+grep -Fq 'expected exactly one HCC replica' "$GATE" &&
+ pass "gate refuses to run unless HCC is single-replica" ||
+ fail "gate does not enforce the single-writer invariant"
+
+exit "$FAIL"