Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(hcc): preserve owner cleanup during policy lifecycle checks
Alfredo Lopezd5a713e6647a
diff --git a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.shindex 392289a2a..6c105a641 100644--- a/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh+++ b/scripts/e2e/_lib/hcc-networkpolicy-lifecycle.sh@@ -183,7 +183,11 @@ np604_cleanup() {NP604_WATCH_PID=''[ "$NP604_CREATED" = 1 ] || return 0np604_kctl delete mcpserver,secret -n "$MCP_NS" \- -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null+ -l "e2e.clerum.io/suite=hcc-np604,e2e.clerum.io/run=${RUN_ID}" --ignore-not-found >/dev/null || return 1+ # Consume the deletion events while the owning controller is still running.+ # Removing the Context only after stopping HCC would leave its allows to a+ # namespace-wide orphan sweep, whose safety cap correctly refuses this fleet.+ wait_until 120 'NP604 owner-driven cleanup before HCC stop' np604_resources_absent}np604_resources_absent() {diff --git a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.shindex cfeaa6345..423043cf5 100755--- a/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh+++ b/scripts/e2e/e2e-hcc-networkpolicy-lifecycle.sh@@ -17,4 +17,9 @@ export E2E_EXPECTED_PRE_GATE_GATE=minikube-t2# witness. Other synthetic McpServers would add unrelated crashing workloads.export FLEET_CONTEXTS=1 FLEET_MCPSERVERS=0 FLEET_HOSTS=1export CHURN_MIN_CUTS=3 CHURN_OBSERVE_SEC=45+# A healthy small fleet can recover between 1 Hz probes. This lane proves+# three real watch reconnections, stable policies and business recovery; it+# does not claim a minimum number of sampled outages. The stress gate's+# default sampled-recovery requirement remains unchanged for its own callers.+export MIN_CHURN_RECOVERIES=0exec bash "$SCRIPT_DIR/e2e-hcc-watch-churn-readiness.sh"diff --git a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh b/scripts/e2e/e2e-hcc-watch-churn-readiness.shindex 61d231c67..8789640e4 100755--- a/scripts/e2e/e2e-hcc-watch-churn-readiness.sh+++ b/scripts/e2e/e2e-hcc-watch-churn-readiness.sh@@ -108,6 +108,10 @@ WATCH_CUT_MIN_LINES="${WATCH_CUT_MIN_LINES:-6}"# Minimum distinct 503->200 transitions under churn: one recovery could be a# fluke; repeated recovery refutes the livelock.MIN_CHURN_RECOVERIES="${MIN_CHURN_RECOVERIES:-2}"+[[ "$MIN_CHURN_RECOVERIES" =~ ^[0-9]+$ ]] || {+ echo 'MIN_CHURN_RECOVERIES must be a non-negative integer' >&2+ exit 2+}FLEET_CONTEXTS="${FLEET_CONTEXTS:-24}"FLEET_MCPSERVERS="${FLEET_MCPSERVERS:-115}"FLEET_HOSTS="${FLEET_HOSTS:-8}"@@ -281,6 +285,19 @@ Restore the HCC deployment (remove redirect env + hostAliases, restore replicas)EOF}+assert_hcc_sampled_recoveries() {+ if [ "$MIN_CHURN_RECOVERIES" -eq 0 ]; then+ printf 'NOT_APPLICABLE: minimum sampled outage count; actual watch cuts and final convergence remain required\n'+ return 0+ fi+ [ "$churn_503" -ge 1 ] &&+ ok "churn bit the readiness path: ${churn_503} sample(s) at 503 (transient fail-closed is the contract)" ||+ fail "zero 503 samples under churn — the proxy never bit the watches; bounded-recovery evidence is VACUOUS"+ [ "$churn_transitions" -ge "$MIN_CHURN_RECOVERIES" ] &&+ ok "repeated in-churn recovery: ${churn_transitions} distinct 503->200 transitions (>= ${MIN_CHURN_RECOVERIES})" ||+ fail "only ${churn_transitions} 503->200 transition(s) under churn — a livelock is not refuted by fewer than ${MIN_CHURN_RECOVERIES} recoveries"+}+cleanup() {local status=$? cleanup_failed=0 restore_ok=1trap - EXIT@@ -494,12 +511,7 @@ fi# transient 503 under churn is the fail-closed CONTRACT working, not a bug;# an observation window with zero 503s means the cuts never reached the# watches and the bounded-recovery claim below would be vacuously true.-[ "$churn_503" -ge 1 ] &&- ok "churn bit the readiness path: ${churn_503} sample(s) at 503 (transient fail-closed is the contract)" ||- fail "zero 503 samples under churn — the proxy never bit the watches; bounded-recovery evidence is VACUOUS"-[ "$churn_transitions" -ge "$MIN_CHURN_RECOVERIES" ] &&- ok "repeated in-churn recovery: ${churn_transitions} distinct 503->200 transitions (>= ${MIN_CHURN_RECOVERIES})" ||- fail "only ${churn_transitions} 503->200 transition(s) under churn — a livelock is not refuted by fewer than ${MIN_CHURN_RECOVERIES} recoveries"+assert_hcc_sampled_recoveries[ "$churn_maxstreak" -le "$RECOVERY_BUDGET_SEC" ] &&ok "every in-churn 503 outage closed within budget: max streak ${churn_maxstreak}s <= ${RECOVERY_BUDGET_SEC}s" ||fail "a 503 streak lasted ${churn_maxstreak}s (> ${RECOVERY_BUDGET_SEC}s recovery budget) — recovery is not bounded under churn"diff --git a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh b/scripts/tests/test-hcc-watch-churn-empty-fleet.shindex 379b22561..cc215e336 100755--- a/scripts/tests/test-hcc-watch-churn-empty-fleet.sh+++ b/scripts/tests/test-hcc-watch-churn-empty-fleet.sh@@ -86,3 +86,46 @@ fiprintf '%s\n' 'MODIFIED mcp-server/ctx-fixture-context-fixture-control' >> "$tmp/policy-events.txt"np604_observer_witnessecho 'PASS: observer requires initial LIST and a distinct streaming MODIFIED witness'++sample_assertion="$(sed -n '/^assert_hcc_sampled_recoveries() {/,/^}/p' "$ROOT/scripts/e2e/e2e-hcc-watch-churn-readiness.sh")"+[[ -n "$sample_assertion" ]]+eval "$sample_assertion"+ok() { echo PASS >> "$tmp/sample-results"; }+fail() { echo FAIL >> "$tmp/sample-results"; }+churn_503=2 churn_transitions=1 MIN_CHURN_RECOVERIES=2+assert_hcc_sampled_recoveries+grep -q FAIL "$tmp/sample-results"+: > "$tmp/sample-results"+churn_transitions=2+assert_hcc_sampled_recoveries+[[ "$(grep -c PASS "$tmp/sample-results")" = 2 ]]+: > "$tmp/sample-results"+MIN_CHURN_RECOVERIES=0 churn_503=0 churn_transitions=0+assert_hcc_sampled_recoveries+[[ ! -s "$tmp/sample-results" ]] # Disabled sampling must not produce vacuous PASS.+echo 'PASS: stress sampling remains required; focused mode makes no sampled-outage claim'++(+ HCC_PATCHED=1 HCC_SCALED_DOWN=1 ORIGINAL_REPLICAS=1+ HCC_DEPLOY=host-context-controller HCC_NS=control-plane+ FLEET_CREATED=0 PROXY_CREATED=0 PROBE_CREATED=0+ E2E_HCC_POLICY_LIFECYCLE=1 NP604_CREATED=1 NP604_WATCH_PID=''+ NP604_SERVER=fixture-affected NP604_CONTROL=fixture-control+ PROXY_EGRESS_NP=fixture-egress HCC_PROXY_NP=fixture-hcc PROBE_EGRESS_NP=fixture-probe+ HCC_LOG_BUFFER="$tmp/buffer" READY_SERIES="$tmp/series"+ kctl() { printf '%s\n' "$*" >> "$tmp/order"; }+ stop_hcc_recovery_log_stream() { :; }+ hcc_pods_absent() { :; }+ wait_until() { shift 2; "$@"; }+ restore_hcc_after_churn() { :; }+ finalize_hcc_watch_gate_lock() { :; }+ print_results() { :; }+ eval "$cleanup_source"+ cleanup+)+awk '+ /get deployment,service,networkpolicy/ && !owner_check {owner_check=NR}+ /scale deployment host-context-controller.*--replicas=0/ && !stop {stop=NR}+ END {exit !(owner_check && stop && owner_check < stop)}+' "$tmp/order"+echo 'PASS: actual cleanup checks owner-driven fixture removal before stopping HCC'