Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(hcc): preserve owner cleanup during policy lifecycle checks

Alfredo Lopezd5a713e6647a
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
index 392289a2a..6c105a641 100644
--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh
@@ -183,7 +183,11 @@ np604_cleanup() {
NP604_WATCH_PID=''
[ "$NP604_CREATED" = 1 ] || return 0
np604_kctl delete mcpserver,secret -n "$MCP_NS" \
- -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null
+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null || return 1
+ # Consume the deletion events while the owning controller is still running.
+ # Removing the Context only after stopping HCC would leave its allows to a
+ # namespace-wide orphan sweep, whose safety cap correctly refuses this fleet.
+ wait_until 120 'NP604 owner-driven cleanup before HCC stop' np604_resources_absent
}
np604_resources_absent() {
diff --git a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
index cfeaa6345..423043cf5 100755
--- a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
+++ b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh
@@ -17,4 +17,9 @@ export E2E_EXPECTED_PRE_GATE_GATE=minikube-t2
# witness. Other synthetic McpServers would add unrelated crashing workloads.
export FLEET_CONTEXTS=1 FLEET_MCPSERVERS=0 FLEET_HOSTS=1
export CHURN_MIN_CUTS=3 CHURN_OBSERVE_SEC=45
+# A healthy small fleet can recover between 1 Hz probes. This lane proves
+# three real watch reconnections, stable policies and business recovery; it
+# does not claim a minimum number of sampled outages. The stress gate's
+# default sampled-recovery requirement remains unchanged for its own callers.
+export MIN_CHURN_RECOVERIES=0
exec bash "$SCRIPT_DIR/e2e-hcc-watch-churn-readiness.sh"
diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
index 61d231c67..8789640e4 100755
--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh
@@ -108,6 +108,10 @@ WATCH_CUT_MIN_LINES="${WATCH_CUT_MIN_LINES:-6}"
# Minimum distinct 503->200 transitions under churn: one recovery could be a
# fluke; repeated recovery refutes the livelock.
MIN_CHURN_RECOVERIES="${MIN_CHURN_RECOVERIES:-2}"
+[[ "$MIN_CHURN_RECOVERIES" =~ ^[0-9]+$ ]] || {
+ echo 'MIN_CHURN_RECOVERIES must be a non-negative integer' >&2
+ exit 2
+}
FLEET_CONTEXTS="${FLEET_CONTEXTS:-24}"
FLEET_MCPSERVERS="${FLEET_MCPSERVERS:-115}"
FLEET_HOSTS="${FLEET_HOSTS:-8}"
@@ -281,6 +285,19 @@ Restore the HCC deployment (remove redirect env + hostAliases, restore replicas)
EOF
}
+assert_hcc_sampled_recoveries() {
+ if [ "$MIN_CHURN_RECOVERIES" -eq 0 ]; then
+ printf 'NOT_APPLICABLE: minimum sampled outage count; actual watch cuts and final convergence remain required\n'
+ return 0
+ fi
+ [ "$churn_503" -ge 1 ] &&
+ ok "churn bit the readiness path: ${churn_503} sample(s) at 503 (transient fail-closed is the contract)" ||
+ fail "zero 503 samples under churn — the proxy never bit the watches; bounded-recovery evidence is VACUOUS"
+ [ "$churn_transitions" -ge "$MIN_CHURN_RECOVERIES" ] &&
+ ok "repeated in-churn recovery: ${churn_transitions} distinct 503->200 transitions (>= ${MIN_CHURN_RECOVERIES})" ||
+ fail "only ${churn_transitions} 503->200 transition(s) under churn — a livelock is not refuted by fewer than ${MIN_CHURN_RECOVERIES} recoveries"
+}
+
cleanup() {
local status=$? cleanup_failed=0 restore_ok=1
trap - EXIT
@@ -494,12 +511,7 @@ fi
# transient 503 under churn is the fail-closed CONTRACT working, not a bug;
# an observation window with zero 503s means the cuts never reached the
# watches and the bounded-recovery claim below would be vacuously true.
-[ "$churn_503" -ge 1 ] &&
- ok "churn bit the readiness path: ${churn_503} sample(s) at 503 (transient fail-closed is the contract)" ||
- fail "zero 503 samples under churn — the proxy never bit the watches; bounded-recovery evidence is VACUOUS"
-[ "$churn_transitions" -ge "$MIN_CHURN_RECOVERIES" ] &&
- ok "repeated in-churn recovery: ${churn_transitions} distinct 503->200 transitions (>= ${MIN_CHURN_RECOVERIES})" ||
- fail "only ${churn_transitions} 503->200 transition(s) under churn — a livelock is not refuted by fewer than ${MIN_CHURN_RECOVERIES} recoveries"
+assert_hcc_sampled_recoveries
[ "$churn_maxstreak" -le "$RECOVERY_BUDGET_SEC" ] &&
ok "every in-churn 503 outage closed within budget: max streak ${churn_maxstreak}s <= ${RECOVERY_BUDGET_SEC}s" ||
fail "a 503 streak lasted ${churn_maxstreak}s (> ${RECOVERY_BUDGET_SEC}s recovery budget) — recovery is not bounded under churn"
diff --git a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
index 379b22561..cc215e336 100755
--- a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
+++ b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh
@@ -86,3 +86,46 @@ fi
printf '%s\n' 'MODIFIED mcp-server/ctx-fixture-context-fixture-control' >> "$tmp/policy-events.txt"
np604_observer_witness
echo 'PASS: observer requires initial LIST and a distinct streaming MODIFIED witness'
+
+sample_assertion="$(sed -n '/^assert_hcc_sampled_recoveries() {/,/^}/p' "$ROOT/scripts/e2e/e2e-hcc-watch-churn-readiness.sh")"
+[[ -n "$sample_assertion" ]]
+eval "$sample_assertion"
+ok() { echo PASS >> "$tmp/sample-results"; }
+fail() { echo FAIL >> "$tmp/sample-results"; }
+churn_503=2 churn_transitions=1 MIN_CHURN_RECOVERIES=2
+assert_hcc_sampled_recoveries
+grep -q FAIL "$tmp/sample-results"
+: > "$tmp/sample-results"
+churn_transitions=2
+assert_hcc_sampled_recoveries
+[[ "$(grep -c PASS "$tmp/sample-results")" = 2 ]]
+: > "$tmp/sample-results"
+MIN_CHURN_RECOVERIES=0 churn_503=0 churn_transitions=0
+assert_hcc_sampled_recoveries
+[[ ! -s "$tmp/sample-results" ]] # Disabled sampling must not produce vacuous PASS.
+echo 'PASS: stress sampling remains required; focused mode makes no sampled-outage claim'
+
+(
+ HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1
+ HCC_DEPLOY=host-context-controller HCC_NS=control-plane
+ FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0
+ E2E_HCC_POLICY_LIFECYCLE=1 NP604_CREATED=1 NP604_WATCH_PID=''
+ NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control
+ PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe
+ HCC_LOG_BUFFER="$tmp/buffer" READY_SERIES="$tmp/series"
+ kctl() { printf '%s\n' "$*" >> "$tmp/order"; }
+ stop_hcc_recovery_log_stream() { :; }
+ hcc_pods_absent() { :; }
+ wait_until() { shift 2; "$@"; }
+ restore_hcc_after_churn() { :; }
+ finalize_hcc_watch_gate_lock() { :; }
+ print_results() { :; }
+ eval "$cleanup_source"
+ cleanup
+)
+awk '
+ /get deployment,service,networkpolicy/ && !owner_check {owner_check=NR}
+ /scale deployment host-context-controller.*--replicas=0/ && !stop {stop=NR}
+ END {exit !(owner_check && stop && owner_check < stop)}
+' "$tmp/order"
+echo 'PASS: actual cleanup checks owner-driven fixture removal before stopping HCC'