Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): add the HCC watch-churn readiness gate (minikube reproducer)
Alfredo Lopez4db02c06946d
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 08541e5bc..f318ba33a 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -390,6 +390,9 @@ jobs:- name: Test HCC watch-recovery gate coordinationrun: bash scripts/tests/test-hcc-watch-recovery-gate.sh+ - name: Test HCC watch-churn readiness gate contract+ run: bash scripts/tests/test-hcc-watch-churn-readiness-gate.sh+- name: Test HCC API gateway readiness separationrun: bash scripts/tests/test-hcc-api-gateway-readiness.shdiff --git a/Makefile b/Makefileindex 107633d70..7334f00fc 100644--- a/Makefile+++ b/Makefile@@ -943,6 +943,12 @@ test-e2e-hcc-readiness-bootstrap: ## Prove HCC readiness while its initial Host@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }E2E_HCC_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-readiness-bootstrap.sh+.PHONY: test-e2e-hcc-watch-churn-readiness+test-e2e-hcc-watch-churn-readiness: ## Prove HCC readiness survives sustained apiserver watch churn (PR #205 GKE livelock). EXPECT_LIVELOCK=1 reproduces the bug.+ @echo "Running HCC watch-churn readiness gate..."+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }+ E2E_HCC_WATCH_FAULT_INJECTION=1 EXPECT_LIVELOCK=$(EXPECT_LIVELOCK) E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-watch-churn-readiness.sh+.PHONY: test-e2e-hcc-mcp-context-readinesstest-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Context/NetworkPolicy initial convergence@echo "Running HCC MCP/Context/NetworkPolicy readiness gate..."diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.shnew file mode 100755index 000000000..c62777dee--- /dev/null+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh@@ -0,0 +1,207 @@+#!/usr/bin/env bash+# Helpers for the HCC watch-churn readiness gate: a self-flapping Kubernetes API+# TCP proxy that periodically resets long-lived (watch) connections, a synthetic+# clerum-dev-scale CR fleet, and a non-destructive HCC redirect/restore.+#+# The proxy runs the HCC image (already loaded in the node), needs no privileges,+# and — unlike scale_proxy in the recovery gate — cuts connections on its own+# clock, so it reproduces GKE's sustained `Premature close` regime rather than a+# single manual outage. NetworkPolicy forces HCC's ONLY apiserver egress through+# it, so the cuts hit exactly the HCC watches and nothing else.+#+# Variables (PROXY_NAME, HCC_NS, HCC_DEPLOY, HCC_IMAGE, MCP_NS, HOST_NS,+# FLEET_PREFIX, FLEET_SECRET, RUN_ID, PROXY_EGRESS_NP, HCC_PROXY_NP, and the+# *_CREATED flags) are owned by the sourcing gate; this file only defines helpers.++# shellcheck disable=SC2034+CHURN_PROXY_PORT=8443++create_hcc_churn_proxy() {+ # $1 = churn period ms (cut cadence), $2 = min connection age ms before a cut.+ local churn_period_ms=$1 churn_min_age_ms=$2+ PROXY_CREATED=1+ kctl apply -f - >/dev/null <<EOF+apiVersion: apps/v1+kind: Deployment+metadata:+ name: ${PROXY_NAME}+ namespace: ${HCC_NS}+ labels: {e2e.clerum.io/suite: hcc-watch-churn}+spec:+ replicas: 1+ selector: {matchLabels: {app: ${PROXY_NAME}}}+ template:+ metadata: {labels: {app: ${PROXY_NAME}, e2e.clerum.io/suite: hcc-watch-churn}}+ spec:+ automountServiceAccountToken: false+ terminationGracePeriodSeconds: 1+ containers:+ - name: proxy+ image: ${HCC_IMAGE}+ imagePullPolicy: IfNotPresent+ env:+ - {name: CHURN_PERIOD_MS, value: "${churn_period_ms}"}+ - {name: CHURN_MIN_AGE_MS, value: "${churn_min_age_ms}"}+ command: [node, -e]+ args:+ # A relay to kubernetes.default.svc:443 that tracks every live upstream+ # socket with its birth timestamp, and every CHURN_PERIOD_MS destroys the+ # ones older than CHURN_MIN_AGE_MS. Short requests (LIST/GET/POST, ms)+ # complete untouched; long-lived watches (>min age) are reset — exactly+ # GKE's balancer idle-timeout behaviour. The listener never dies, so HCC+ # reconnects, re-LISTs, and bumps its watch generation on every cut.+ - >-+ const net=require('net');+ const period=Number(process.env.CHURN_PERIOD_MS);+ const minAge=Number(process.env.CHURN_MIN_AGE_MS);+ const live=new Set();+ net.createServer(c=>{+ const born=Date.now();+ const u=net.connect(443,'kubernetes.default.svc');+ const rec={c,u,born};live.add(rec);+ const close=()=>{live.delete(rec);c.destroy();u.destroy()};+ c.on('error',close);u.on('error',close);c.on('close',close);u.on('close',close);+ c.pipe(u);u.pipe(c);+ }).listen(${CHURN_PROXY_PORT},'0.0.0.0');+ setInterval(()=>{const now=Date.now();+ for(const r of [...live]){ if(now-r.born>=minAge){ r.c.destroy();r.u.destroy();live.delete(r);} }+ },period);+ readinessProbe: {tcpSocket: {port: ${CHURN_PROXY_PORT}}, periodSeconds: 1}+ resources:+ requests: {cpu: 10m, memory: 24Mi}+ limits: {cpu: 100m, memory: 64Mi}+ securityContext:+ allowPrivilegeEscalation: false+ capabilities: {drop: [ALL]}+ readOnlyRootFilesystem: true+ runAsNonRoot: true+ runAsUser: 1000+ runAsGroup: 1000+ seccompProfile: {type: RuntimeDefault}+---+apiVersion: v1+kind: Service+metadata:+ name: ${PROXY_NAME}+ namespace: ${HCC_NS}+ labels: {e2e.clerum.io/suite: hcc-watch-churn}+spec:+ selector: {app: ${PROXY_NAME}}+ ports: [{name: tls, port: 443, targetPort: ${CHURN_PROXY_PORT}, protocol: TCP}]+EOF++ # Clone the control-plane API egress policy for the proxy, and restrict proxy+ # ingress to the HCC pod only (identical isolation to the recovery gate).+ kctl get networkpolicy allow-k8s-api-egress-control-plane -n "$HCC_NS" -o json |+ jq --arg name "$PROXY_EGRESS_NP" --arg app "$PROXY_NAME" '+ del(.metadata.creationTimestamp, .metadata.generation, .metadata.resourceVersion,+ .metadata.uid, .metadata.managedFields) |+ .metadata.name = $name |+ .metadata.labels = {"e2e.clerum.io/suite": "hcc-watch-churn"} |+ .spec.podSelector = {matchLabels: {app: $app}} |+ .spec.policyTypes = ["Ingress", "Egress"] |+ .spec.ingress = [{from: [{podSelector: {matchLabels: {app: "host-context-controller"}}}],+ ports: [{port: '"${CHURN_PROXY_PORT}"', protocol: "TCP"}]}]' |+ kctl apply -f - >/dev/null++ kctl apply -f - >/dev/null <<EOF+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${HCC_PROXY_NP}+ namespace: ${HCC_NS}+ labels: {e2e.clerum.io/suite: hcc-watch-churn}+spec:+ podSelector: {matchLabels: {app: host-context-controller}}+ policyTypes: [Egress]+ egress:+ - to: [{podSelector: {matchLabels: {app: ${PROXY_NAME}}}}]+ ports:+ - {port: 443, protocol: TCP}+ - {port: ${CHURN_PROXY_PORT}, protocol: TCP}+EOF+ kctl rollout status deployment "$PROXY_NAME" -n "$HCC_NS" --timeout=90s >/dev/null ||+ die "churn proxy did not become ready"+}++# create_synthetic_fleet CONTEXTS MCPSERVERS HOSTS — HCC must be scaled to 0 first.+create_synthetic_fleet() {+ local n_ctx=$1 n_mcp=$2 n_host=$3 i ctx applied+ FLEET_CREATED=1++ # LLM secret shared by all synthetic Hosts (label-scoped for cleanup).+ kctl create secret generic "$FLEET_SECRET" -n "$HOST_NS" \+ --from-literal=api-key=e2e-churn --dry-run=client -o yaml |+ kctl label --local -f - e2e.clerum.io/suite=hcc-watch-churn -o yaml |+ kctl apply -f - >/dev/null++ # Contexts (one stream).+ { for ((i = 1; i <= n_ctx; i++)); do+ ctx="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" "$i")"+ cat <<EOF+---+apiVersion: clerum.io/v1alpha1+kind: Context+metadata: {name: ${ctx}, namespace: ${MCP_NS}, labels: {e2e.clerum.io/suite: hcc-watch-churn, e2e.clerum.io/run: "${RUN_ID}"}}+spec: {contextId: ${ctx}, description: "churn fleet context", mcpServers: []}+EOF+ done; } | kctl apply -f - >/dev/null++ # McpServers (one stream) — inventory + NetworkPolicies, no heavy pods.+ { for ((i = 1; i <= n_mcp; i++)); do+ ctx="$(printf '%s-ctx-%03d' "$FLEET_PREFIX" $(((i % n_ctx) + 1)))"+ cat <<EOF+---+apiVersion: clerum.io/v1alpha1+kind: McpServer+metadata: {name: $(printf '%s-mcp-%03d' "$FLEET_PREFIX" "$i"), namespace: ${MCP_NS}, labels: {e2e.clerum.io/suite: hcc-watch-churn, e2e.clerum.io/run: "${RUN_ID}"}}+spec:+ contextRef: ${ctx}+ description: "churn fleet mcp server"+ image: ${HCC_IMAGE}+ imagePullPolicy: Never+ transport: {type: streamableHttp, url: "http://127.0.0.1:3000/mcp", port: 3000}+EOF+ done; } | kctl apply -f - >/dev/null++ # Hosts (one stream) — these materialize Deployments; keep the count modest.+ { for ((i = 1; i <= n_host; i++)); do+ cat <<EOF+---+apiVersion: clerum.io/v1alpha1+kind: Host+metadata: {name: $(printf '%s-host-%02d' "$FLEET_PREFIX" "$i"), namespace: ${HOST_NS}, labels: {e2e.clerum.io/suite: hcc-watch-churn, e2e.clerum.io/run: "${RUN_ID}"}}+spec:+ host: $(printf '%s-host-%02d' "$FLEET_PREFIX" "$i")+ contextRef: $(printf '%s-ctx-%03d' "$FLEET_PREFIX" 1)+ secretRef: ${FLEET_SECRET}+ model: {provider: openai, name: e2e-churn}+EOF+ done; } | kctl apply -f - >/dev/null++ # Fail-loud count check: refuse to proceed on a partial fleet.+ applied="$(kctl get mcpserver -n "$MCP_NS" -l e2e.clerum.io/run="$RUN_ID" -o name | wc -l | tr -d ' ')"+ [ "$applied" = "$n_mcp" ] || die "synthetic fleet incomplete: applied ${applied}/${n_mcp} McpServers"+}++delete_synthetic_fleet() {+ local failed=0 kind+ for kind in mcpserver context host; do+ kctl delete "$kind" -A -l e2e.clerum.io/suite=hcc-watch-churn \+ --ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 || failed=1+ done+ kctl delete secret "$FLEET_SECRET" -n "$HOST_NS" --ignore-not-found >/dev/null 2>&1 || failed=1+ # HCC-managed runtime left behind by the fleet Hosts.+ kctl delete deployment,service,serviceaccount,role,rolebinding,secret,pvc,networkpolicy \+ -A -l "clerum.io/managed-by=host-context-controller" --ignore-not-found >/dev/null 2>&1 || true+ return "$failed"+}++# Non-destructive restore: undo the env overrides and hostAliases redirect.+restore_hcc_after_churn() {+ kctl set env deployment/"$HCC_DEPLOY" -n "$HCC_NS" \+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS- >/dev/null || return 1+ kctl patch deployment/"$HCC_DEPLOY" -n "$HCC_NS" --type=merge \+ -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}' >/dev/null || return 1+ [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] || return 1+}diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.shnew file mode 100755index 000000000..fd4c37438--- /dev/null+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh@@ -0,0 +1,277 @@+#!/usr/bin/env bash+# HCC readiness under SUSTAINED apiserver watch churn (Premature close regime).+#+# Reproduces the PR #205 GKE livelock in minikube — a regime minikube does NOT+# exhibit by default (single stable apiserver, no balancer idle-timeout). A+# clerum-dev-scale synthetic fleet widens each NetworkPolicy revocation pass; a+# self-flapping API proxy resets the HCC watches mid-pass, bumping the watch+# generation so a generation-pinned isReadinessInventoryAuthoritative() could+# never certify. The content-identity fix must certify through the churn.+#+# EXPECT_LIVELOCK=1 -> proves the reproducer: /ready stays 503 + divergent+# generation pattern (guards against a vacuous test).+# EXPECT_LIVELOCK=0 -> proves the fix: /ready reaches 200 (1/1) under the+# SAME churn, and stays 200 through STABILITY_WINDOW_SEC.+set -euo pipefail+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+# shellcheck source=scripts/e2e/e2e-lib.sh+source "${SCRIPT_DIR}/e2e-lib.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-logs.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/hcc-watch-churn-fixture.sh"++# ── Fail-closed guards (identical contract to the sibling HCC gates) ──+[ -n "$E2E_KUBECONTEXT" ] || {+ echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2+ exit 1+}+is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {+ echo "Refusing churn injection on non-branch context '${E2E_KUBECONTEXT}'." >&2+ exit 1+}+require_safe_kube_context+command -v jq >/dev/null 2>&1 || {+ echo "jq is required" >&2+ exit 1+}+[ "${E2E_HCC_WATCH_FAULT_INJECTION:-0}" = 1 ] || {+ echo "Set E2E_HCC_WATCH_FAULT_INJECTION=1 to acknowledge fault injection." >&2+ exit 1+}+kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $c)' >/dev/null ||+ {+ echo "Refusing churn injection: target is not this profile's minikube node." >&2+ exit 1+ }++# ── Tunables ──+HCC_NS="${HCC_NS:-control-plane}"+HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"+MCP_NS="${MCP_SERVER_NS:-mcp-server}"+HOST_NS="${MCP_HOST_NS:-mcp-host}"+CHURN_PERIOD_MS="${CHURN_PERIOD_MS:-7000}"+CHURN_MIN_AGE_MS="${CHURN_MIN_AGE_MS:-3000}"+CHURN_MIN_CUTS="${CHURN_MIN_CUTS:-3}"+READINESS_BUDGET_SEC="${READINESS_BUDGET_SEC:-360}"+STABILITY_WINDOW_SEC="${STABILITY_WINDOW_SEC:-30}"+FLEET_CONTEXTS="${FLEET_CONTEXTS:-24}"+FLEET_MCPSERVERS="${FLEET_MCPSERVERS:-115}"+FLEET_HOSTS="${FLEET_HOSTS:-8}"+EXPECT_LIVELOCK="${EXPECT_LIVELOCK:-0}"++RUN_ID="$(date +%s)-$$"+PROXY_NAME="$(truncate_rfc1123 "e2e-hcc-churn-proxy-${RUN_ID}")"+PROXY_EGRESS_NP="$(truncate_rfc1123 "${PROXY_NAME}-api")"+HCC_PROXY_NP="$(truncate_rfc1123 "${PROXY_NAME}-from-hcc")"+FLEET_PREFIX="$(truncate_rfc1123 "e2e-hcc-churn-${RUN_ID}")"+FLEET_SECRET="$(truncate_rfc1123 "${FLEET_PREFIX}-llm")"+LOG_ARTIFACT="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn.XXXXXX")"+ORIGINAL_REPLICAS=""+HCC_IMAGE=""+K8S_API_CIDR=""+PROXY_CREATED=0+FLEET_CREATED=0+HCC_PATCHED=0+# shellcheck disable=SC2034+HCC_GATE_LOCK_ACQUIRED=0 HCC_GATE_LOCK_NAME="" HCC_GATE_LOCK_UID="" HCC_GATE_FINALIZATION_FAILURE=""++die() {+ fail "$*"+ exit 1+}++wait_until() {+ local timeout=$1 description=$2+ shift 2+ local deadline now+ deadline=$(($(date +%s) + timeout))+ while :; do+ "$@" && return 0+ now="$(date +%s)"+ [ "$now" -lt "$deadline" ] || break+ sleep 1+ done+ echo "Timed out after ${timeout}s waiting for ${description}" >&2+ return 1+}++running_hcc_pod() {+ local rows+ rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \+ --field-selector=status.phase=Running \+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}' \+ 2>/dev/null)" || return 1+ awk -F '\t' '$1 != "" && $2 == "" { print $1; exit }' <<<"$rows"+}++hcc_pods_absent() {+ local pods+ pods="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o name 2>/dev/null)" || return 1+ [ -z "$pods" ]+}++ready_status() {+ local pod=$1+ kctl exec "pod/$pod" -n "$HCC_NS" -c host-context-controller -- \+ wget -T 10 -t 1 -qO- http://127.0.0.1:8081/ready >/dev/null 2>&1 && echo 200 || echo 503+}++print_repair_instructions() {+ cat >&2 <<EOF+HCC watch-churn gate cleanup could not restore a verified clean state.+Context: ${E2E_KUBECONTEXT}+HCC: ${HCC_NS}/${HCC_DEPLOY}+Churn proxy: ${HCC_NS}/${PROXY_NAME}+Synthetic fleet label: e2e.clerum.io/suite=hcc-watch-churn (run ${RUN_ID})++Inspect before changing anything:+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment,service,networkpolicy -l e2e.clerum.io/suite=hcc-watch-churn++Restore the HCC deployment (remove redirect env + hostAliases, restore replicas):+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set env deployment/${HCC_DEPLOY} \\+ KUBERNETES_SERVICE_HOST- KUBERNETES_SERVICE_PORT- CONTEXT_MAPPER_K8S_API_CIDRS-+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} patch deployment/${HCC_DEPLOY} --type=merge \\+ -p '{"spec":{"template":{"spec":{"hostAliases":null}}}}'+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}+ kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=180s+EOF+}++cleanup() {+ local status=$? cleanup_failed=0 restore_ok=1+ trap - EXIT+ set +e+ if [ "$HCC_PATCHED" = 1 ]; then+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null 2>&1+ wait_until 120 "HCC stop" hcc_pods_absent >/dev/null 2>&1 || restore_ok=0+ fi+ [ "$FLEET_CREATED" = 1 ] && { delete_synthetic_fleet || cleanup_failed=1; }+ if [ "$HCC_PATCHED" = 1 ]; then+ restore_hcc_after_churn || restore_ok=0+ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1+ kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1 || restore_ok=0+ fi+ [ "$PROXY_CREATED" = 1 ] && kctl delete deployment,service "$PROXY_NAME" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1+ kctl delete networkpolicy "$PROXY_EGRESS_NP" "$HCC_PROXY_NP" -n "$HCC_NS" --ignore-not-found >/dev/null 2>&1+ [ "$restore_ok" = 1 ] || {+ print_repair_instructions+ cleanup_failed=1+ }+ finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1+ print_results || status=1+ [ "$cleanup_failed" = 0 ] || status=1+ exit "$status"+}+trap cleanup EXIT++# ── FASE A: guards + snapshot ──+require_branch_owned_hcc_gate "$HCC_NS"+acquire_hcc_watch_gate_lock+ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"+[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"+[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.hostAliases}')" ] ||+ die "HCC already has hostAliases; refusing a non-restorable injection"+HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"+[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"+api_ip="$(kctl exec deployment/"$HCC_DEPLOY" -n "$HCC_NS" -c host-context-controller -- \+ printenv KUBERNETES_SERVICE_HOST)" || die "could not read API service IP"+K8S_API_CIDR="${api_ip}/32"++# ── FASE B: redirect through the self-flapping proxy (HCC still healthy) ──+log "Creating self-flapping API proxy (period=${CHURN_PERIOD_MS}ms, min-age=${CHURN_MIN_AGE_MS}ms)"+create_hcc_churn_proxy "$CHURN_PERIOD_MS" "$CHURN_MIN_AGE_MS"+verify_hcc_proxy_network_policy+proxy_ip="$(kctl get service "$PROXY_NAME" -n "$HCC_NS" -o jsonpath='{.spec.clusterIP}')"+[ -n "$proxy_ip" ] || die "proxy Service has no ClusterIP"+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null+wait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop before fleet creation"++# ── FASE C: synthetic clerum-dev-scale fleet (HCC stopped) ──+log "Creating synthetic fleet: ${FLEET_CONTEXTS} Contexts / ${FLEET_MCPSERVERS} McpServers / ${FLEET_HOSTS} Hosts"+create_synthetic_fleet "$FLEET_CONTEXTS" "$FLEET_MCPSERVERS" "$FLEET_HOSTS"+ok "synthetic fleet created (${FLEET_MCPSERVERS} McpServers)"++# ── FASE D: start HCC under churn and poll /ready ──+log "Redirecting HCC through the churn proxy and starting it under load"+patch="$(jq -cn --arg ip "$proxy_ip" --arg cidr "$K8S_API_CIDR" '{spec:{template:{spec:{+ hostAliases:[{ip:$ip,hostnames:["kubernetes.default.svc"]}],+ containers:[{name:"host-context-controller",env:[+ {name:"KUBERNETES_SERVICE_HOST",value:"kubernetes.default.svc"},+ {name:"KUBERNETES_SERVICE_PORT",value:"443"},+ {name:"CONTEXT_MAPPER_K8S_API_CIDRS",value:$cidr}]}]}}}}')"+kctl patch deployment "$HCC_DEPLOY" -n "$HCC_NS" --type=strategic -p "$patch" >/dev/null+HCC_PATCHED=1+kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null+start_hcc_recovery_log_stream++deadline=$(($(date +%s) + READINESS_BUDGET_SEC))+first_ready=""+last_status=""+while [ "$(date +%s)" -lt "$deadline" ]; do+ pod="$(running_hcc_pod)" || {+ sleep 2+ continue+ }+ [ -n "$pod" ] || {+ sleep 2+ continue+ }+ last_status="$(ready_status "$pod")"+ if [ "$last_status" = 200 ]; then+ first_ready="$((READINESS_BUDGET_SEC - (deadline - $(date +%s))))"+ break+ fi+ sleep 3+done+cuts="$(hcc_recovery_logs | grep -Ec 'watch ended|authority changed|recovering authoritative inventory' || true)"+gen_pattern="$(hcc_recovery_logs | grep -Ec 'authority changed before .* admission' || true)"+{+ echo "=== HCC deploy ==="+ kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide+ echo "=== last /ready: ${last_status} cuts=${cuts} gen-diverge=${gen_pattern} first-ready=${first_ready:-never} ==="+ hcc_recovery_logs | grep -E 'watch ended|authority changed|Recovered|inventory' | tail -60+} >"$LOG_ARTIFACT" 2>&1++# ── Verdict ──+if [ "$EXPECT_LIVELOCK" = 1 ]; then+ [ "$cuts" -ge "$CHURN_MIN_CUTS" ] && ok "churn cut ${cuts} watch(es) (>= ${CHURN_MIN_CUTS})" ||+ fail "churn cut only ${cuts} watch(es) — reproducer is VACUOUS"+ [ "$gen_pattern" -ge 1 ] && ok "observed divergent-generation pattern ${gen_pattern}x" ||+ fail "no divergent-generation pattern — bug mechanism NOT exercised"+ [ -z "$first_ready" ] && ok "livelock reproduced: /ready stayed 503 for ${READINESS_BUDGET_SEC}s under churn" ||+ fail "expected livelock but /ready reached 200 — reproducer does not exercise the bug"+else+ [ "$cuts" -ge "$CHURN_MIN_CUTS" ] && ok "fix validated UNDER churn (${cuts} cuts)" ||+ fail "fix validated WITHOUT churn (${cuts} cuts) — result is worthless"+ if [ -n "$first_ready" ]; then+ ok "fix converged to Ready under sustained Premature close (time-to-Ready=${first_ready}s, cuts=${cuts})"+ stable=1+ s_deadline=$(($(date +%s) + STABILITY_WINDOW_SEC))+ while [ "$(date +%s)" -lt "$s_deadline" ]; do+ pod="$(running_hcc_pod)" || {+ stable=0+ break+ }+ [ "$(ready_status "$pod")" = 200 ] || {+ stable=0+ break+ }+ sleep 3+ done+ [ "$stable" = 1 ] && ok "readiness stable for ${STABILITY_WINDOW_SEC}s under continuous churn" ||+ fail "readiness oscillated 200->503 under churn — fix not stable"+ else+ fail "fix did NOT reach Ready under churn within ${READINESS_BUDGET_SEC}s — livelock regression (see ${LOG_ARTIFACT})"+ fi+fi+stop_hcc_recovery_log_stream+log "Evidence artifact: ${LOG_ARTIFACT}"+# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.diff --git a/scripts/tests/test-hcc-watch-churn-readiness-gate.sh b/scripts/tests/test-hcc-watch-churn-readiness-gate.shnew file mode 100755index 000000000..09fd2004f--- /dev/null+++ b/scripts/tests/test-hcc-watch-churn-readiness-gate.sh@@ -0,0 +1,61 @@+#!/usr/bin/env bash+# Offline contract gate for the HCC watch-churn readiness e2e. Runs in CI: no+# cluster. Proves the gate keeps its fail-loud, branch-scoped, restorable shape+# and its anti-vacuity assertions, so a refactor cannot quietly gut the one test+# that reproduces the clerum-dev livelock under real watch churn.+set -u+FAIL=0+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+GATE="${ROOT}/scripts/e2e/e2e-hcc-watch-churn-readiness.sh"+FIXTURE="${ROOT}/scripts/e2e/_lib/hcc-watch-churn-fixture.sh"+pass() { echo "PASS: $1"; }+fail() {+ echo "FAIL: $1"+ FAIL=1+}++for s in "$GATE" "$FIXTURE"; do+ bash -n "$s" && pass "$(basename "$s") valid bash" || fail "$(basename "$s") invalid bash"+done++# 1. Anti-vacuity: both regimes assert a minimum number of real watch cuts.+[ "$(grep -Ec 'cuts.*-ge.*CHURN_MIN_CUTS' "$GATE")" -ge 2 ] &&+ pass "both livelock and fix regimes assert >= CHURN_MIN_CUTS real cuts" ||+ fail "a regime can pass without proving churn actually cut the watches"++# 2. Livelock regime demands the divergent-generation evidence.+grep -Fq 'authority changed before .* admission' "$GATE" &&+ pass "livelock regime keys on the real generation-divergence log" ||+ fail "livelock regime does not check the generation-divergence signal"++# 3. Fix regime demands sustained stability, not a transient 200.+grep -Fq 'readiness oscillated 200->503' "$GATE" &&+ pass "fix regime asserts sustained readiness under continuous churn" ||+ fail "fix regime accepts a transient 200 between cuts"++# 4. Branch-scoped + fault-injection ack + single-writer + restorable, like siblings.+grep -Fq 'is_branch_scoped_e2e_context' "$GATE" &&+ grep -Fq 'E2E_HCC_WATCH_FAULT_INJECTION' "$GATE" &&+ grep -Fq 'require_branch_owned_hcc_gate' "$GATE" &&+ grep -Fq 'ORIGINAL_REPLICAS' "$GATE" &&+ grep -Fq 'restore_hcc_after_churn' "$GATE" &&+ pass "gate keeps branch-scope, ack, single-writer and restore contract" ||+ fail "gate drops a branch-scope / ack / single-writer / restore guard"++# 5. Churn cadence is parameterized with an explicit default.+grep -Eq 'CHURN_PERIOD_MS:-[0-9]{3,5}' "$GATE" &&+ pass "churn cadence is parameterized with an explicit default" ||+ fail "churn cadence is not parameterized"++# 6. The node label is bound to the explicit profile (no non-null wildcard).+grep -Fq 'minikube.k8s.io/name"] == $c' "$GATE" &&+ ! grep -Fq 'minikube.k8s.io/name"] != null' "$GATE" &&+ pass "gate binds the minikube node label to the explicit profile" ||+ fail "gate can accept a node owned by another profile"++# 7. The single-writer invariant is enforced (exactly one replica).+grep -Fq 'expected exactly one HCC replica' "$GATE" &&+ pass "gate refuses to run unless HCC is single-replica" ||+ fail "gate does not enforce the single-writer invariant"++exit "$FAIL"