Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(e2e): measure churn convergence, not the absence of 503, in the readiness gate

Alfredo Lopezce19fc8c4511
diff --git a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
index c62777dee..0a17bd1eb 100755
--- a/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
+++ b/scripts/e2e/_lib/hcc-watch-churn-fixture.sh
@@ -50,8 +50,13 @@ spec:
# complete untouched; long-lived watches (>min age) are reset — exactly
# GKE's balancer idle-timeout behaviour. The listener never dies, so HCC
# reconnects, re-LISTs, and bumps its watch generation on every cut.
+ # The cut loop re-checks /churn-ctl/paused (emptyDir) on every tick, so
+ # the gate can freeze the churn IN PLACE via pause_hcc_churn_proxy —
+ # no env patch, no Deployment rollout, no final cut — while the relay
+ # and every live socket stay up.
- >-
const net=require('net');
+ const fs=require('fs');
const period=Number(process.env.CHURN_PERIOD_MS);
const minAge=Number(process.env.CHURN_MIN_AGE_MS);
const live=new Set();
@@ -63,7 +68,8 @@ spec:
c.on('error',close);u.on('error',close);c.on('close',close);u.on('close',close);
c.pipe(u);u.pipe(c);
}).listen(${CHURN_PROXY_PORT},'0.0.0.0');
- setInterval(()=>{const now=Date.now();
+ setInterval(()=>{if(fs.existsSync('/churn-ctl/paused'))return;
+ const now=Date.now();
for(const r of [...live]){ if(now-r.born>=minAge){ r.c.destroy();r.u.destroy();live.delete(r);} }
},period);
readinessProbe: {tcpSocket: {port: ${CHURN_PROXY_PORT}}, periodSeconds: 1}
@@ -78,6 +84,10 @@ spec:
runAsUser: 1000
runAsGroup: 1000
seccompProfile: {type: RuntimeDefault}
+ volumeMounts:
+ - {name: churn-ctl, mountPath: /churn-ctl}
+ volumes:
+ - {name: churn-ctl, emptyDir: {}}
---
apiVersion: v1
kind: Service
@@ -124,6 +134,27 @@ EOF
die "churn proxy did not become ready"
}
+# Freeze the churn WITHOUT restarting anything: write the pause flag that the
+# relay's cut loop re-checks on every tick. Alternatives all restart something:
+# an env patch on the proxy Deployment rolls the proxy pod (one final cut plus
+# a rollout race against the readiness probe), and `scale --replicas=0` severs
+# the HCC's ONLY apiserver egress — both contaminate the post-churn
+# measurement. `kubectl exec` reaches the pod via apiserver->kubelet, so the
+# gate's NetworkPolicies do not constrain it, and `node` is guaranteed on PATH
+# because it is the container's own command. The flag lives on an emptyDir
+# (writable under readOnlyRootFilesystem, pod-lifetime persistent), and the
+# write is verified read-after-write so a silent no-op cannot pass.
+pause_hcc_churn_proxy() {
+ local pod
+ pod="$(kctl get pods -n "$HCC_NS" -l "app=${PROXY_NAME}" \
+ --field-selector=status.phase=Running \
+ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" || return 1
+ [ -n "$pod" ] || return 1
+ kctl exec "pod/${pod}" -n "$HCC_NS" -c proxy -- node -e \
+ 'const fs=require("fs");fs.writeFileSync("/churn-ctl/paused","1");if(!fs.existsSync("/churn-ctl/paused"))process.exit(1)' \
+ >/dev/null 2>&1
+}
+
# create_synthetic_fleet CONTEXTS MCPSERVERS HOSTS — HCC must be scaled to 0 first.
create_synthetic_fleet() {
local n_ctx=$1 n_mcp=$2 n_host=$3 i ctx applied
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
index b4416dd68..50995d8fe 100755
--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -8,10 +8,21 @@
# generation so a generation-pinned isReadinessInventoryAuthoritative() could
# never certify. The content-identity fix must certify through the churn.
#
-# EXPECT_LIVELOCK=1 -> proves the reproducer: /ready stays 503 + divergent
-# generation pattern (guards against a vacuous test).
-# EXPECT_LIVELOCK=0 -> proves the fix: /ready reaches 200 (1/1) under the
-# SAME churn, and stays 200 through STABILITY_WINDOW_SEC.
+# The fail-closed contract means a cut watch drops /ready to 503 TRANSIENTLY
+# (~recovery-timer + re-LIST) and re-certifies with NO content change; under
+# sustained churn the correct regime is a bounded 503/200 duty cycle, NOT an
+# unbroken 200. The gate therefore measures streaks, not the absence of 503s:
+#
+# EXPECT_LIVELOCK=1 -> proves the reproducer: an UNBROKEN 503 streak for the
+# whole budget (>> RECOVERY_BUDGET_SEC), kubelet
+# cross-check, plus the divergent-generation pattern
+# (guards against a vacuous test).
+# EXPECT_LIVELOCK=0 -> proves the fix: /ready reaches 200 under churn, every
+# in-churn 503 streak closes within RECOVERY_BUDGET_SEC
+# with repeated 503->200 recoveries, and once the churn
+# is paused (flag file — no proxy or HCC restart) the
+# SAME pod re-certifies in <= POST_STOP_READY_SEC and
+# holds 200 for STABILITY_WINDOW_SEC with zero 503s.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/e2e/e2e-lib.sh
@@ -60,6 +71,22 @@ CHURN_MIN_AGE_MS="${CHURN_MIN_AGE_MS:-3000}"
CHURN_MIN_CUTS="${CHURN_MIN_CUTS:-3}"
READINESS_BUDGET_SEC="${READINESS_BUDGET_SEC:-360}"
STABILITY_WINDOW_SEC="${STABILITY_WINDOW_SEC:-30}"
+# Longest tolerated CONTINUOUS 503 streak under churn: 5s recovery timer +
+# re-LIST + one retried cut cycle. A livelock produces an unbounded streak.
+RECOVERY_BUDGET_SEC="${RECOVERY_BUDGET_SEC:-15}"
+# 1Hz observation window under churn after first Ready (fix mode): ~8 cut
+# cycles at the default 7s cadence, enough for repeated 503->200 evidence.
+CHURN_OBSERVE_SEC="${CHURN_OBSERVE_SEC:-60}"
+# Post-churn convergence budget once the proxy stops cutting: one recovery
+# timer + LIST + margin. The absorbing-state detector.
+POST_STOP_READY_SEC="${POST_STOP_READY_SEC:-15}"
+# >= 2 full cycles x 3 informers (Context/McpServer/Host) must have been cut.
+WATCH_CUT_MIN_LINES="${WATCH_CUT_MIN_LINES:-6}"
+# Minimum distinct 503->200 transitions under churn: one recovery could be a
+# fluke; repeated recovery refutes the livelock.
+MIN_CHURN_RECOVERIES="${MIN_CHURN_RECOVERIES:-2}"
+# Livelock signature: an unbroken 503 streak at least this long (3x budget).
+LIVELOCK_MIN_STREAK_SEC="${LIVELOCK_MIN_STREAK_SEC:-45}"
FLEET_CONTEXTS="${FLEET_CONTEXTS:-24}"
FLEET_MCPSERVERS="${FLEET_MCPSERVERS:-115}"
FLEET_HOSTS="${FLEET_HOSTS:-8}"
@@ -76,6 +103,9 @@ PROBE_EGRESS_NP="$(truncate_rfc1123 "${PROXY_NAME}-from-probe")"
FLEET_PREFIX="$(truncate_rfc1123 "e2e-hcc-churn-${RUN_ID}")"
FLEET_SECRET="$(truncate_rfc1123 "${FLEET_PREFIX}-llm")"
LOG_ARTIFACT="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn.XXXXXX")"
+# 1Hz /ready sample series: "<epoch> <status> <phase>" per line. Streaks and
+# transitions are derived from this file at verdict time.
+READY_SERIES="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn-series.XXXXXX")"
ORIGINAL_REPLICAS=""
HCC_IMAGE=""
K8S_API_CIDR=""
@@ -88,8 +118,13 @@ HCC_PATCHED=0
HCC_LOG_BUFFER="$(mktemp "${TMPDIR:-/tmp}/hcc-watch-churn-stream.XXXXXX")"
HCC_LOG_STREAM_PID=""
START_TIME=""
-# Verdict-evidence counters; assigned by recount_churn_evidence() before any read.
-cuts=0 gen_pattern=0 total_watch_cut_lines=0 stable=""
+# Verdict-evidence counters; assigned by recount_churn_evidence() and the
+# series readers before any read.
+cuts=0 gen_pattern=0 total_watch_cut_lines=0
+churn_total=0 churn_200=0 churn_503=0 churn_maxstreak=0 churn_transitions=0
+hold_total=0 hold_503=0
+post_converged=0 post_ready_secs=""
+hcc_pod_at_stop="" hcc_restarts_at_stop=""
# shellcheck disable=SC2034
HCC_GATE_LOCK_ACQUIRED=0 HCC_GATE_LOCK_NAME="" HCC_GATE_LOCK_UID="" HCC_GATE_FINALIZATION_FAILURE=""
@@ -134,6 +169,75 @@ ready_status() {
wget -T 10 -t 1 -qO- http://127.0.0.1:8081/ready >/dev/null 2>&1 && echo 200 || echo 503
}
+hcc_restart_count() {
+ local pod=$1
+ kctl get pod "$pod" -n "$HCC_NS" \
+ -o jsonpath='{.status.containerStatuses[?(@.name=="host-context-controller")].restartCount}' 2>/dev/null
+}
+
+# sample_ready_series DURATION_SEC PHASE STOP_ON_READY
+# Appends "<epoch> <status> <phase>" lines to READY_SERIES at ~1Hz (each probe
+# is an exec round-trip, so real spacing is 1-2s; all streak math uses the
+# recorded timestamps, never sample counts). A missing or terminating pod is
+# recorded as 503: a crash-looping HCC is DOWN, and hiding that would let a
+# livelock-by-crash impersonate recovery. STOP_ON_READY=1 returns at the first
+# 200 sample (readiness search); 0 runs the full duration (observation).
+sample_ready_series() {
+ local duration=$1 phase=$2 stop_on_ready=$3
+ local s_deadline t0 now pod status
+ s_deadline=$(($(date +%s) + duration))
+ while t0="$(date +%s)"; [ "$t0" -lt "$s_deadline" ]; do
+ status=503
+ if pod="$(running_hcc_pod)" && [ -n "$pod" ]; then
+ status="$(ready_status "$pod")"
+ fi
+ printf '%s %s %s\n' "$t0" "$status" "$phase" >>"$READY_SERIES"
+ last_status="$status"
+ if [ "$status" = 200 ] && [ "$stop_on_ready" = 1 ]; then
+ return 0
+ fi
+ now="$(date +%s)"
+ [ $((now - t0)) -ge 1 ] || sleep 1
+ done
+ return 0
+}
+
+# series_metrics PHASE -> prints "total n200 n503 max503streak_sec transitions"
+# for that phase slice. A 503 streak is measured in wall seconds from its first
+# 503 sample to the next 200 sample (or to end-of-slice + 1s if it never
+# closed); transitions counts distinct 503->200 recoveries.
+series_metrics() {
+ awk -v phase="$1" '
+ $3 == phase {
+ total += 1
+ if ($2 == 200) {
+ n200 += 1
+ if (in503) {
+ streak = $1 - streak_start
+ if (streak > max_streak) max_streak = streak
+ transitions += 1
+ in503 = 0
+ }
+ } else {
+ n503 += 1
+ if (!in503) { in503 = 1; streak_start = $1 }
+ }
+ last_ts = $1
+ }
+ END {
+ if (in503) {
+ streak = last_ts + 1 - streak_start
+ if (streak > max_streak) max_streak = streak
+ }
+ printf "%d %d %d %d %d\n", total + 0, n200 + 0, n503 + 0, max_streak + 0, transitions + 0
+ }' "$READY_SERIES"
+}
+
+# first_200_epoch PHASE -> epoch of the first 200 sample in that phase, if any.
+first_200_epoch() {
+ awk -v phase="$1" '$3 == phase && $2 == 200 { print $1; exit }' "$READY_SERIES"
+}
+
print_repair_instructions() {
cat >&2 <<EOF
HCC watch-churn gate cleanup could not restore a verified clean state.
@@ -181,7 +285,7 @@ cleanup() {
finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
print_results || status=1
[ "$cleanup_failed" = 0 ] || status=1
- rm -f "$HCC_LOG_BUFFER"
+ rm -f "$HCC_LOG_BUFFER" "$READY_SERIES"
exit "$status"
}
trap cleanup EXIT
@@ -229,25 +333,17 @@ kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
START_TIME="$(python3 -c 'import datetime; print(datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))')"
start_hcc_recovery_log_stream
-deadline=$(($(date +%s) + READINESS_BUDGET_SEC))
-first_ready=""
+# Readiness search: sample at 1Hz until the first 200 or the budget expires.
+# Every sample (including the pre-Ready 503s) lands in the churn-phase series.
last_status=""
-while [ "$(date +%s)" -lt "$deadline" ]; do
- pod="$(running_hcc_pod)" || {
- sleep 2
- continue
- }
- [ -n "$pod" ] || {
- sleep 2
- continue
- }
- last_status="$(ready_status "$pod")"
- if [ "$last_status" = 200 ]; then
- first_ready="$((READINESS_BUDGET_SEC - (deadline - $(date +%s))))"
- break
- fi
- sleep 3
-done
+churn_probe_t0="$(date +%s)"
+sample_ready_series "$READINESS_BUDGET_SEC" churn 1
+first_ready=""
+first_ready_epoch="$(first_200_epoch churn)"
+if [ -n "$first_ready_epoch" ]; then
+ first_ready=$((first_ready_epoch - churn_probe_t0))
+fi
+
# ── Churn accounting over the FULL raw stream ──
# hcc_recovery_logs() pre-filters the buffer with the RECOVERY gate's patterns
# (CommunicationChannel/Listing all Hosts/...). The McpServer, Context and Host
@@ -268,37 +364,71 @@ recount_churn_evidence() {
write_evidence_artifact() {
# Diagnostics ONLY — nothing here feeds the verdict (cuts/gen_pattern/
- # first_ready/stable are computed before this runs), so every command is
- # deliberately best-effort: under `set -euo pipefail`, an empty grep in this
- # block aborted the whole gate BEFORE the verdict on the first live run.
+ # first_ready/series metrics are computed before this runs), so every command
+ # is deliberately best-effort: under `set -euo pipefail`, an empty grep in
+ # this block aborted the whole gate BEFORE the verdict on the first live run.
{
echo "=== HCC deploy ==="
kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide || true
echo "=== last /ready: ${last_status} cut-cycles=${cuts} watch-cut-lines=${total_watch_cut_lines} gen-diverge=${gen_pattern} first-ready=${first_ready:-never} ==="
+ echo "=== churn phase: samples=${churn_total} 200=${churn_200} 503=${churn_503} max-503-streak=${churn_maxstreak}s 503->200=${churn_transitions} ==="
+ echo "=== post-churn: converged=${post_converged} ready-after=${post_ready_secs:-never}s hold: samples=${hold_total} 503=${hold_503} ==="
+ echo "=== /ready series (epoch status phase) ==="
+ cat "$READY_SERIES" || true
+ echo "=== HCC log tail ==="
grep -E 'watch ended|authority changed|Recovered|inventory' "$HCC_LOG_BUFFER" | tail -60 || true
} >"$LOG_ARTIFACT" 2>&1 || true
}
-# ── Stability window (fix mode only) — the churn keeps cutting while we hold ──
+# ── Churn observation window (fix mode) — the proxy keeps cutting while we
+# sample at 1Hz. This is where the bounded-recovery evidence accumulates: 503
+# streaks must close within RECOVERY_BUDGET_SEC and recovery must repeat
+# (503->200 transitions). The pre-redesign zero-tolerance stability window
+# asserted a property the fail-closed contract deliberately violates: a cut
+# watch drops /ready to 503 transiently while it re-syncs, so "no 503 after
+# first 200" failed the CORRECT regime and measured the wrong thing.
if [ "$EXPECT_LIVELOCK" = 0 ] && [ -n "$first_ready" ]; then
- stable=1
- s_deadline=$(($(date +%s) + STABILITY_WINDOW_SEC))
- while [ "$(date +%s)" -lt "$s_deadline" ]; do
- pod="$(running_hcc_pod)" || { stable=0; break; }
- [ -n "$pod" ] || { stable=0; break; }
- [ "$(ready_status "$pod")" = 200 ] || { stable=0; break; }
- sleep 3
- done
+ log "Observing /ready at 1Hz for ${CHURN_OBSERVE_SEC}s under continuous churn"
+ sample_ready_series "$CHURN_OBSERVE_SEC" churn 0
+fi
+
+# ── Pause the churn WITHOUT restarting the proxy or the HCC ──
+# The relay checks its pause flag on every tick, so pause_hcc_churn_proxy
+# freezes the cut loop in place: no Deployment rollout, no final cut, no HCC
+# restart. The post-churn phase must prove IN-SITU convergence (same process,
+# no content change), so the HCC pod identity and restart count are pinned
+# here and re-verified in the verdict.
+hcc_pod_at_stop="$(running_hcc_pod || true)"
+if [ -n "$hcc_pod_at_stop" ]; then
+ hcc_restarts_at_stop="$(hcc_restart_count "$hcc_pod_at_stop" || true)"
fi
+log "Pausing the churn proxy (flag file; proxy and HCC pods stay up)"
+pause_hcc_churn_proxy || die "could not pause the churn proxy"
+post_stop_t0="$(date +%s)"
-# Snapshot the buffer only AFTER the stability window: with first_ready≈6s the
-# readiness loop alone has seen ~1 cut; the window adds STABILITY_WINDOW_SEC of
-# further churn, which is what makes CHURN_MIN_CUTS satisfiable in fix mode.
-# In livelock mode the readiness loop already consumed READINESS_BUDGET_SEC.
+# ── Post-churn convergence: the absorbing-state detector ──
+# The fail-closed contract re-certifies from watch recovery alone — no content
+# change is injected here, on purpose: convergence must not need one. A true
+# livelock cannot pass this phase; a correct transient regime cannot fail it.
+sample_ready_series "$POST_STOP_READY_SEC" post-reach 1
+post_ready_epoch="$(first_200_epoch post-reach)"
+if [ -n "$post_ready_epoch" ]; then
+ post_converged=1
+ post_ready_secs=$((post_ready_epoch - post_stop_t0))
+ sample_ready_series "$STABILITY_WINDOW_SEC" post-hold 0
+fi
+
+# Snapshot the buffer only AFTER the observation and post-churn phases: with
+# first_ready≈6s the readiness search alone has seen ~1 cut; CHURN_OBSERVE_SEC
+# of further churn is what makes CHURN_MIN_CUTS and WATCH_CUT_MIN_LINES
+# satisfiable in fix mode. In livelock mode the readiness search already
+# consumed READINESS_BUDGET_SEC of churn.
require_hcc_recovery_log_stream ||
warn "HCC log stream ended before the verdict; counting the frozen buffer (fail-closed: an undercount can only turn this gate RED)"
stop_hcc_recovery_log_stream
recount_churn_evidence
+read -r churn_total churn_200 churn_503 churn_maxstreak churn_transitions <<<"$(series_metrics churn)"
+read -r hold_total _ hold_503 _ _ <<<"$(series_metrics post-hold)"
write_evidence_artifact
# ── Verdict ──
@@ -308,6 +438,8 @@ if [ "$EXPECT_LIVELOCK" = 1 ]; then
fail "churn cut the Context watch only ${cuts}x — reproducer is VACUOUS"
[ "$gen_pattern" -ge 1 ] && ok "observed divergent-generation admission pattern ${gen_pattern}x" ||
fail "no divergent-generation pattern — bug mechanism NOT exercised"
+ [ "$churn_total" -gt 0 ] && ok "sampled /ready ${churn_total}x under churn" ||
+ fail "ZERO /ready samples under churn — the sampler never ran, and a zero-sample run must never pass"
if [ -z "$first_ready" ]; then
# Cross-check the exec probe against the kubelet's own view: a broken
# ready_status (wrong container/port) reports 503 forever and would fake
@@ -318,20 +450,67 @@ if [ "$EXPECT_LIVELOCK" = 1 ]; then
else
fail "exec probe reported 503 but the Deployment shows ${ready_replicas} ready replica(s) — probe is broken, reproducer VACUOUS"
fi
+ [ "$churn_maxstreak" -ge "$LIVELOCK_MIN_STREAK_SEC" ] &&
+ ok "the 503 streak never closed: ${churn_maxstreak}s continuous (>= ${LIVELOCK_MIN_STREAK_SEC}s; recovery budget is ${RECOVERY_BUDGET_SEC}s)" ||
+ fail "longest 503 streak was only ${churn_maxstreak}s — not a livelock signature"
else
fail "expected livelock but /ready reached 200 at ${first_ready}s — reproducer does not exercise the bug"
fi
+ # Post-churn behaviour is DIAGNOSTIC in this mode: the generation pin only
+ # livelocks WHILE the generation keeps bumping, so once the churn is paused
+ # the pre-fix code may legitimately certify. Asserting either outcome would
+ # turn an implementation detail of the pre-fix build into a gate verdict.
+ log "post-churn (diagnostic only): converged=${post_converged} ready-after=${post_ready_secs:-never}s hold-503s=${hold_503}"
else
[ "$cuts" -ge "$CHURN_MIN_CUTS" ] &&
- ok "churn was real: ${cuts} Context-watch cuts (${total_watch_cut_lines} watch-ended lines) across readiness+stability" ||
+ ok "churn was real: ${cuts} Context-watch cuts (${total_watch_cut_lines} watch-ended lines) across readiness+observation" ||
fail "fix validated WITHOUT churn (${cuts} Context-watch cuts) — result is worthless"
+ [ "$total_watch_cut_lines" -ge "$WATCH_CUT_MIN_LINES" ] &&
+ ok "every informer was cut repeatedly: ${total_watch_cut_lines} watch-ended lines (>= ${WATCH_CUT_MIN_LINES} = 2 cycles x 3 watches)" ||
+ fail "only ${total_watch_cut_lines} watch-ended lines (< ${WATCH_CUT_MIN_LINES}) — churn did not cut all three informers at least twice"
+ [ "$churn_total" -gt 0 ] && ok "sampled /ready ${churn_total}x under churn" ||
+ fail "ZERO /ready samples under churn — the sampler never ran, and a zero-sample run must never pass"
if [ -n "$first_ready" ]; then
ok "fix converged to Ready under sustained Premature close (time-to-Ready=${first_ready}s)"
- [ "$stable" = 1 ] && ok "readiness stable for ${STABILITY_WINDOW_SEC}s under continuous churn" ||
- fail "readiness oscillated 200->503 under churn — fix not stable"
else
fail "fix did NOT reach Ready under churn within ${READINESS_BUDGET_SEC}s — livelock regression (see ${LOG_ARTIFACT})"
fi
+ # Anti-vacuity: the proxy must actually have bitten the readiness path. A
+ # transient 503 under churn is the fail-closed CONTRACT working, not a bug;
+ # an observation window with zero 503s means the cuts never reached the
+ # watches and the bounded-recovery claim below would be vacuously true.
+ [ "$churn_503" -ge 1 ] &&
+ ok "churn bit the readiness path: ${churn_503} sample(s) at 503 (transient fail-closed is the contract)" ||
+ fail "zero 503 samples under churn — the proxy never bit the watches; bounded-recovery evidence is VACUOUS"
+ [ "$churn_transitions" -ge "$MIN_CHURN_RECOVERIES" ] &&
+ ok "repeated in-churn recovery: ${churn_transitions} distinct 503->200 transitions (>= ${MIN_CHURN_RECOVERIES})" ||
+ fail "only ${churn_transitions} 503->200 transition(s) under churn — a livelock is not refuted by fewer than ${MIN_CHURN_RECOVERIES} recoveries"
+ [ "$churn_maxstreak" -le "$RECOVERY_BUDGET_SEC" ] &&
+ ok "every in-churn 503 outage closed within budget: max streak ${churn_maxstreak}s <= ${RECOVERY_BUDGET_SEC}s" ||
+ fail "a 503 streak lasted ${churn_maxstreak}s (> ${RECOVERY_BUDGET_SEC}s recovery budget) — recovery is not bounded under churn"
+ # Post-churn: absorbing-state detector (the decisive livelock/transient split).
+ if [ "$post_converged" = 1 ]; then
+ ok "post-churn: /ready re-certified in ${post_ready_secs}s with NO content change injected"
+ [ "$hold_total" -gt 0 ] && [ "$hold_503" -eq 0 ] &&
+ ok "post-churn hold: ${hold_total} samples over ${STABILITY_WINDOW_SEC}s, zero 503 — no absorbing state" ||
+ fail "post-churn hold saw ${hold_503} 503(s) across ${hold_total} sample(s) — readiness did not stabilize once the churn stopped"
+ else
+ fail "post-churn: /ready did NOT converge within ${POST_STOP_READY_SEC}s after the churn was paused — absorbing state (livelock signature)"
+ fi
+ # In-situ guarantee: the convergence above must belong to the SAME process
+ # that lived through the churn. A restarted or replaced pod proves fresh-boot
+ # readiness, which the bootstrap gate already covers — not in-situ recovery.
+ hcc_pod_now="$(running_hcc_pod || true)"
+ hcc_restarts_now=""
+ if [ -n "$hcc_pod_now" ]; then
+ hcc_restarts_now="$(hcc_restart_count "$hcc_pod_now" || true)"
+ fi
+ if [ -n "$hcc_pod_at_stop" ] && [ "$hcc_pod_now" = "$hcc_pod_at_stop" ] &&
+ [ -n "$hcc_restarts_at_stop" ] && [ "$hcc_restarts_now" = "$hcc_restarts_at_stop" ]; then
+ ok "post-churn recovery was in-situ: pod ${hcc_pod_at_stop} unchanged, restartCount ${hcc_restarts_now}"
+ else
+ fail "HCC pod changed or restarted across the post-churn phase (${hcc_pod_at_stop:-none}/restarts=${hcc_restarts_at_stop:-?} -> ${hcc_pod_now:-none}/restarts=${hcc_restarts_now:-?}) — convergence could be a fresh boot, not in-situ recovery"
+ fi
fi
log "Evidence artifact: ${LOG_ARTIFACT}"
# cleanup() (EXIT trap) restores HCC, deletes the fleet, and runs print_results.
diff --git a/scripts/tests/test-hcc-watch-churn-readiness-gate.sh b/scripts/tests/test-hcc-watch-churn-readiness-gate.sh
index 09fd2004f..2ef41f306 100755
--- a/scripts/tests/test-hcc-watch-churn-readiness-gate.sh
+++ b/scripts/tests/test-hcc-watch-churn-readiness-gate.sh
@@ -28,12 +28,42 @@ grep -Fq 'authority changed before .* admission' "$GATE" &&
pass "livelock regime keys on the real generation-divergence log" ||
fail "livelock regime does not check the generation-divergence signal"
-# 3. Fix regime demands sustained stability, not a transient 200.
-grep -Fq 'readiness oscillated 200->503' "$GATE" &&
- pass "fix regime asserts sustained readiness under continuous churn" ||
- fail "fix regime accepts a transient 200 between cuts"
+# 3. Fix regime bounds in-churn recovery instead of banning transient 503s.
+# The fail-closed contract drops /ready to 503 TRANSIENTLY while a cut watch
+# re-syncs, so a zero-tolerance "no 503 after first 200" window fails the
+# CORRECT regime; the honest property is that every 503 streak closes within
+# the recovery budget.
+grep -Eq 'churn_maxstreak.*-le.*RECOVERY_BUDGET_SEC' "$GATE" &&
+ pass "fix regime bounds every in-churn 503 streak by RECOVERY_BUDGET_SEC" ||
+ fail "fix regime lost the bounded-recovery streak assertion"
-# 4. Branch-scoped + fault-injection ack + single-writer + restorable, like siblings.
+# 4. Post-churn absorbing-state detector: after the churn is paused the SAME
+# HCC pod must re-certify without a content change and hold with zero 503s.
+grep -Fq 'pause_hcc_churn_proxy' "$GATE" &&
+ grep -Fq 'did NOT converge' "$GATE" &&
+ grep -Eq 'hold_503.*-eq.*0' "$GATE" &&
+ pass "fix regime keeps the post-churn convergence + zero-503 hold detector" ||
+ fail "fix regime lost the post-churn absorbing-state detector"
+
+# 5. Sampler anti-vacuity: the proxy must bite (>= 1 in-churn 503), recovery
+# must repeat (>= MIN_CHURN_RECOVERIES 503->200 transitions), and a run
+# with zero samples must never pass (zero-tests-is-never-success).
+grep -Eq 'churn_503.*-ge.*1' "$GATE" &&
+ grep -Eq 'churn_transitions.*-ge.*MIN_CHURN_RECOVERIES' "$GATE" &&
+ grep -Eq 'churn_total.*-gt.*0' "$GATE" &&
+ pass "sampler anti-vacuity (503 bite, repeated recovery, nonzero samples) enforced" ||
+ fail "sampler can pass vacuously (no 503s, a single recovery, or zero samples)"
+
+# 6. In-situ guarantee: post-churn convergence must come from the same pod
+# with an unchanged restart count (fresh boot is a different gate's claim),
+# and the churn pause must be the restart-free flag mechanism.
+grep -Fq 'in-situ' "$GATE" &&
+ grep -Fq 'restartCount' "$GATE" &&
+ grep -Fq 'churn-ctl/paused' "$FIXTURE" &&
+ pass "post-churn convergence is pinned to the same non-restarted HCC pod" ||
+ fail "a fresh-boot recovery could impersonate in-situ convergence"
+
+# 7. Branch-scoped + fault-injection ack + single-writer + restorable, like siblings.
grep -Fq 'is_branch_scoped_e2e_context' "$GATE" &&
grep -Fq 'E2E_HCC_WATCH_FAULT_INJECTION' "$GATE" &&
grep -Fq 'require_branch_owned_hcc_gate' "$GATE" &&
@@ -42,18 +72,18 @@ grep -Fq 'is_branch_scoped_e2e_context' "$GATE" &&
pass "gate keeps branch-scope, ack, single-writer and restore contract" ||
fail "gate drops a branch-scope / ack / single-writer / restore guard"
-# 5. Churn cadence is parameterized with an explicit default.
+# 8. Churn cadence is parameterized with an explicit default.
grep -Eq 'CHURN_PERIOD_MS:-[0-9]{3,5}' "$GATE" &&
pass "churn cadence is parameterized with an explicit default" ||
fail "churn cadence is not parameterized"
-# 6. The node label is bound to the explicit profile (no non-null wildcard).
+# 9. The node label is bound to the explicit profile (no non-null wildcard).
grep -Fq 'minikube.k8s.io/name"] == $c' "$GATE" &&
! grep -Fq 'minikube.k8s.io/name"] != null' "$GATE" &&
pass "gate binds the minikube node label to the explicit profile" ||
fail "gate can accept a node owned by another profile"
-# 7. The single-writer invariant is enforced (exactly one replica).
+# 10. The single-writer invariant is enforced (exactly one replica).
grep -Fq 'expected exactly one HCC replica' "$GATE" &&
pass "gate refuses to run unless HCC is single-replica" ||
fail "gate does not enforce the single-writer invariant"