Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(#391): close HCC rollout-undo coverage holes in PR 416

Alfredo Lopez7e647c1982c5
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 35b2d3d3a..b0ed09c7b 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -400,6 +400,9 @@ jobs:
- name: Test HCC rollout readiness gate contract
run: bash scripts/tests/test-hcc-rollout-readiness-gate.sh
+ - name: Test HCC rollout readiness guards (executable, cluster-free)
+ run: bash scripts/tests/test-hcc-rollout-readiness-guards.sh
+
- name: Test HCC API gateway readiness separation
run: bash scripts/tests/test-hcc-api-gateway-readiness.sh
diff --git a/Makefile b/Makefile
index 1cc8deaa8..11408f3ce 100644
--- a/Makefile
+++ b/Makefile
@@ -961,6 +961,15 @@ test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Cont
@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-mcp-context-readiness.sh
+.PHONY: test-e2e-hcc-rollout-readiness
+test-e2e-hcc-rollout-readiness: ## Measure the HCC Recreate rollout window (D1/c4). EXPECT_STUCK=1 reproduces the D1b outage; EXPECT_RECOVERY=1 proves the evenfire#391 rollout-undo path. The two flags are EXCLUSIVE; default both 0 = healthy measurement.
+ @echo "Running HCC rollout readiness gate..."
+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }
+ E2E_HCC_ROLLOUT_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" \
+ MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) \
+ EXPECT_STUCK=$(EXPECT_STUCK) EXPECT_RECOVERY=$(EXPECT_RECOVERY) \
+ bash scripts/e2e/e2e-hcc-rollout-readiness.sh
+
.PHONY: test-e2e-stateless-multinode
test-e2e-stateless-multinode: ## Run stateless multi-node lane (opt-in: STATELESS_MULTINODE_GATE=1; needs >=2 schedulable nodes; exit 3 = cross-node UNVERIFIED)
@echo "Running stateless multi-node lane..."
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 84cf93bac..8725e9991 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.229",
+ "version": "0.1.230",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.229",
+ "version": "0.1.230",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index efe9ae9da..777adbc22 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.229",
+ "version": "0.1.230",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/deploymentStrategy.test.ts b/host-context-controller/src/deploymentStrategy.test.ts
index 0779328a6..ff51e3259 100644
--- a/host-context-controller/src/deploymentStrategy.test.ts
+++ b/host-context-controller/src/deploymentStrategy.test.ts
@@ -17,12 +17,16 @@ import path from 'node:path'
* only way to honour the documented single-writer assumption during a rollout.
*/
describe('host-context-controller Deployment strategy', () => {
- const docs = loadAll(
- readFileSync(
- path.join(__dirname, '../../deploy/base/control-plane/host-context-controller.yaml'),
- 'utf8'
- )
- ) as Array<{ kind?: string; metadata?: { name?: string }; spec?: Record<string, unknown> }>
+ const yamlPath = path.join(
+ __dirname,
+ '../../deploy/base/control-plane/host-context-controller.yaml'
+ )
+ const yamlSource = readFileSync(yamlPath, 'utf8')
+ const docs = loadAll(yamlSource) as Array<{
+ kind?: string
+ metadata?: { name?: string }
+ spec?: Record<string, unknown>
+ }>
const deployment = docs.find(
d => d?.kind === 'Deployment' && d?.metadata?.name === 'host-context-controller'
@@ -65,4 +69,59 @@ describe('host-context-controller Deployment strategy', () => {
// fleet.
expect(deployment?.spec?.progressDeadlineSeconds).toBe(1200)
})
+
+ /**
+ * evenfire#391 detection-vs-recovery split. The D1b premise is a SET, not
+ * three independent knobs: Recreate (terminate-before-create, zero fallback
+ * replicas) + replicas:1 (single writer) + progressDeadlineSeconds:1200
+ * (detection). Change any one of them and the #391 recovery contract — the
+ * evenfire-infra deploy pipeline performing `rollout undo --to-revision`,
+ * never Kubernetes and never HCC code — is reasoning about a deployment
+ * that no longer exists. These pins fail if the set is broken up or if the
+ * documented ownership/detection-only contract is silently deleted.
+ */
+ describe('evenfire#391 detection-vs-recovery split', () => {
+ it('keeps the D1b premise set together: Recreate + replicas:1 + deadline 1200', () => {
+ // Asserted as one test on purpose: a partial edit (e.g. bumping replicas
+ // while keeping Recreate, or flipping the strategy while keeping the
+ // 1200s deadline) must fail HERE as a broken premise set, not pass three
+ // unrelated single-field tests.
+ const strategy = deployment?.spec?.strategy as
+ | { type?: string; rollingUpdate?: unknown }
+ | undefined
+ expect({
+ replicas: deployment?.spec?.replicas,
+ strategyType: strategy?.type,
+ rollingUpdate: strategy?.rollingUpdate,
+ progressDeadlineSeconds: deployment?.spec?.progressDeadlineSeconds,
+ }).toEqual({
+ replicas: 1,
+ strategyType: 'Recreate',
+ rollingUpdate: undefined,
+ progressDeadlineSeconds: 1200,
+ })
+ })
+
+ it('documents that the progress deadline is DETECTION ONLY and never rolls back', () => {
+ // The deadline flips Progressing=False and fails `kubectl rollout
+ // status`; it does NOT undo anything. If this documented contract is
+ // deleted from the manifest, a future reader will assume Kubernetes
+ // self-heals a botched Recreate rollout — the exact D1b failure mode.
+ expect(yamlSource).toContain('DETECTION ONLY')
+ expect(yamlSource).toContain('it does not roll back')
+ expect(yamlSource).toContain('Kubernetes will not self-heal')
+ })
+
+ it('pins recovery ownership to the evenfire-infra rollout undo (evenfire#391)', () => {
+ // Recovery is owned by the evenfire-infra deploy workflows via
+ // `rollout undo --to-revision` for this Deployment only (evenfire#391).
+ // HCC must NOT grow its own undo runtime, and the manifest must keep
+ // naming the real owner so a silent deletion of the ownership comment
+ // fails loudly here.
+ expect(yamlSource).toContain('evenfire-infra')
+ expect(yamlSource).toContain('rollout undo --to-revision')
+ expect(yamlSource).toContain('evenfire#391')
+ expect(yamlSource).toContain('The deploy job still fails after a successful undo')
+ })
+ })
})
diff --git a/scripts/e2e/e2e-hcc-rollout-readiness.sh b/scripts/e2e/e2e-hcc-rollout-readiness.sh
index b6f5e699a..9261f7082 100755
--- a/scripts/e2e/e2e-hcc-rollout-readiness.sh
+++ b/scripts/e2e/e2e-hcc-rollout-readiness.sh
@@ -19,11 +19,14 @@
#
# Modes:
# EXPECT_RECOVERY=1 — exclusive with EXPECT_STUCK. Botch the image, prove the
-# D1b 503 outage, then run last-good revision restore as the TEST ACTION
-# (not cleanup). Recovery must recertify /ready 200, restore the original
-# image and last-good revision, and replace the botched pod. An undo no-op
-# is FAIL. After a proven undo, IMAGE_BROKEN=0 so cleanup does not set the
-# image again. This is the evenfire#391 evidence path.
+# D1b 503 outage (same sustained-outage pins as EXPECT_STUCK: >=1 503,
+# zero 503->200, maxstreak > healthy budget), then run last-good revision
+# restore as the TEST ACTION (not cleanup). Recovery must recertify /ready
+# 200, restore the original image and last-good revision, and replace the
+# botched pod. An undo no-op is FAIL. IMAGE_BROKEN stays 1 unless every
+# post-undo proof passed (e2e_fail==0), so a failed image/revision/uid/hold
+# still lets cleanup restore the original image. This is the evenfire#391
+# evidence path.
# EXPECT_STUCK=0 (default) — healthy rollout via `kubectl rollout restart`.
# rollout restart mutates the pod template (restartedAt annotation), which
# drives the SAME Deployment machinery — terminate old, then create new —
@@ -463,6 +466,9 @@ else
[ "$stuck_transitions" -eq 0 ] &&
ok "no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal" ||
fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"
+ [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&
+ ok "SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip" ||
+ fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"
live_revision_before_undo="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
[ "$live_revision_before_undo" != "$LAST_GOOD_REVISION" ] ||
@@ -475,7 +481,6 @@ else
fail "last-known-good revision ${LAST_GOOD_REVISION} did not become Ready"
wait_until 90 "HCC /ready after undo" hcc_ready_now ||
fail "HCC /ready did not return 200 after last-good restore"
- IMAGE_BROKEN=0
recovered_image="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
recovered_revision="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
@@ -499,6 +504,10 @@ else
[ "$hold_total" -gt 0 ] && [ "$hold_503" -eq 0 ] &&
ok "post-undo hold: ${hold_total} samples over ${STABILITY_WINDOW_SEC}s, zero 503" ||
fail "post-undo hold saw ${hold_503} 503(s) across ${hold_total} sample(s)"
+ # fail() records and continues, so only clear the broken-image flag when
+ # every post-undo proof above actually passed. Otherwise cleanup must still
+ # restore the original image.
+ [ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0
fi
diff --git a/scripts/tests/test-hcc-rollout-readiness-gate.sh b/scripts/tests/test-hcc-rollout-readiness-gate.sh
index b7f7ded3e..251f6fe5b 100755
--- a/scripts/tests/test-hcc-rollout-readiness-gate.sh
+++ b/scripts/tests/test-hcc-rollout-readiness-gate.sh
@@ -123,17 +123,56 @@ grep -Fq 'create_synthetic_fleet' "$GATE" &&
# 12. evenfire#391 recovery mode: exclusive with EXPECT_STUCK, undo is the test
-# action (not cleanup), last-good revision is captured, IMAGE_BROKEN is
-# cleared after a proven undo, and a no-op undo fails.
+# action (not cleanup), last-good revision is captured, the pre-undo outage
+# is the same sustained D1b pin as EXPECT_STUCK, IMAGE_BROKEN is cleared
+# only AFTER every post-undo proof, and a no-op undo fails.
+recovery_block="$(awk '/FASE D \(recovery, evenfire#391\)/,0' "$GATE")"
+image_clear_line="$(grep -n 'IMAGE_BROKEN=0' <<<"$recovery_block" | head -1 | cut -d: -f1)"
+hold_assert_line="$(grep -n 'post-undo hold' <<<"$recovery_block" | head -1 | cut -d: -f1)"
grep -Fq 'EXPECT_RECOVERY' "$GATE" &&
grep -Fq 'EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive' "$GATE" &&
grep -Fq 'TEST ACTION' "$GATE" &&
grep -Fq -- '--to-revision' "$GATE" &&
grep -Fq 'LAST_GOOD_REVISION' "$GATE" &&
- grep -Fq 'IMAGE_BROKEN=0' "$GATE" &&
+ grep -Eq 'stuck_maxstreak.*-gt.*ROLLOUT_DOWNTIME_BUDGET_SEC' <<<"$recovery_block" &&
+ grep -Fq '[ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0' <<<"$recovery_block" &&
+ [ -n "$image_clear_line" ] && [ -n "$hold_assert_line" ] &&
+ [ "$image_clear_line" -gt "$hold_assert_line" ] &&
grep -Fq 'undo was a no-op' "$GATE" &&
! grep -q 'EXPECT_RECOVERY=1' <<<"$(awk '/^cleanup\(\)/,/^}/' "$GATE")" &&
pass "EXPECT_RECOVERY undoes to last-good outside cleanup and fails a no-op" ||
- fail "EXPECT_RECOVERY is missing, shares cleanup, or can pass on a no-op undo"
+ fail "EXPECT_RECOVERY is missing, shares cleanup, skips sustained outage, or clears IMAGE_BROKEN before proofs"
+
+# 13. Makefile entry point: the gate is reachable through the same shaped
+# target as the sibling HCC gates — fault-injection ack + pre-gate-sync
+# expectation + explicit profile=context pin, with the exclusive
+# EXPECT_STUCK/EXPECT_RECOVERY modes surfaced (and documented as
+# exclusive) rather than hardcoded.
+MAKEFILE="${ROOT}/Makefile"
+make_target_block="$(awk '/^test-e2e-hcc-rollout-readiness:/,/^$/' "$MAKEFILE")"
+grep -Fq '.PHONY: test-e2e-hcc-rollout-readiness' "$MAKEFILE" &&
+ grep -Fq 'E2E_HCC_ROLLOUT_FAULT_INJECTION=1' <<<"$make_target_block" &&
+ grep -Fq 'E2E_EXPECTED_PRE_GATE_GATE' <<<"$make_target_block" &&
+ grep -Fq 'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT)' <<<"$make_target_block" &&
+ grep -Fq 'KUBECONTEXT=$(E2E_KUBECONTEXT)' <<<"$make_target_block" &&
+ grep -Fq 'EXPECT_STUCK=$(EXPECT_STUCK)' <<<"$make_target_block" &&
+ grep -Fq 'EXPECT_RECOVERY=$(EXPECT_RECOVERY)' <<<"$make_target_block" &&
+ grep -Fq 'scripts/e2e/e2e-hcc-rollout-readiness.sh' <<<"$make_target_block" &&
+ grep -Fq 'EXCLUSIVE' <<<"$make_target_block" &&
+ pass "Makefile target test-e2e-hcc-rollout-readiness keeps ack + profile=context + modes + gate invocation" ||
+ fail "Makefile target test-e2e-hcc-rollout-readiness is missing or lost its ack/profile/mode/gate contract"
+
+# 14. Executable guard suite: CI must RUN the cluster-free guard tests (not
+# only this grep contract), and that suite must actually EXECUTE the gate
+# — a guards script degraded back to grep would be this file twice.
+GUARDS="${ROOT}/scripts/tests/test-hcc-rollout-readiness-guards.sh"
+CI_WORKFLOW="${ROOT}/.github/workflows/ci-public.yml"
+[ -x "$GUARDS" ] &&
+ bash -n "$GUARDS" &&
+ grep -Fq '"$BASH_BIN" "$GATE"' "$GUARDS" &&
+ grep -Fq 'EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive.' "$GUARDS" &&
+ grep -Fq 'scripts/tests/test-hcc-rollout-readiness-guards.sh' "$CI_WORKFLOW" &&
+ pass "executable guard suite exists, executes the gate, and is wired into CI" ||
+ fail "guard suite missing/non-executable, no longer executes the gate, or dropped from CI"
exit "$FAIL"
diff --git a/scripts/tests/test-hcc-rollout-readiness-guards.sh b/scripts/tests/test-hcc-rollout-readiness-guards.sh
new file mode 100755
index 000000000..3564e43c3
--- /dev/null
+++ b/scripts/tests/test-hcc-rollout-readiness-guards.sh
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# Executable, cluster-free guard tests for scripts/e2e/e2e-hcc-rollout-readiness.sh
+# (the evenfire#391 / PR #382 D1b gate).
+#
+# The sibling contract test (test-hcc-rollout-readiness-gate.sh) is grep-based:
+# it proves the guard TEXT exists in the gate, not that the guards RUN. This
+# suite actually EXECUTES the gate under a fully isolated environment (env -i)
+# and asserts each fail-closed guard fires: exit != 0 AND the guard's own
+# stable die/echo text on stderr/stdout. A refactor that keeps the strings but
+# reorders or dead-codes the guards fails HERE.
+#
+# Every case below exits inside the guard block (gate lines before the first
+# `kctl get nodes`) or against a PATH-stubbed fake kubectl — no case ever
+# contacts a real cluster or the developer's kubeconfig.
+set -u
+FAIL=0
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+GATE="${ROOT}/scripts/e2e/e2e-hcc-rollout-readiness.sh"
+BASH_BIN="$(command -v bash)"
+pass() { echo "PASS: $1"; }
+fail() {
+ echo "FAIL: $1"
+ FAIL=1
+}
+
+command -v jq >/dev/null 2>&1 || {
+ echo "FAIL: this harness itself requires jq on PATH (the gate's happy path needs it)" >&2
+ exit 1
+}
+
+# Branch-shaped fake context: passes is_branch_scoped_e2e_context
+# (^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$) so each case reaches the guard
+# under test instead of dying on the context-shape guard.
+BRANCH_CTX="clerum-feat-issue-391-deadbeef"
+
+WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/hcc-rollout-guards.XXXXXX")"
+trap 'rm -rf "$WORKDIR"' EXIT
+# Point KUBECONFIG at an empty throwaway file in every case: even a bug that
+# slipped past PATH isolation must never read or mutate the developer's
+# kubeconfig.
+FAKE_KUBECONFIG="${WORKDIR}/empty-kubeconfig"
+: >"$FAKE_KUBECONFIG"
+
+# run_gate_expecting <description> <needle> [VAR=value ...]
+# Runs the gate under `env -i` (nothing leaks from the developer shell) with
+# exactly the provided variables, expecting: non-zero exit, the guard's stable
+# needle in the combined output, and NO evidence the gate got past its guard
+# block into cluster mutation.
+run_gate_expecting() {
+ local desc="$1" needle="$2"
+ shift 2
+ local out rc
+ out="$(env -i HOME="$HOME" TMPDIR="${WORKDIR}" KUBECONFIG="$FAKE_KUBECONFIG" "$@" \
+ "$BASH_BIN" "$GATE" 2>&1)"
+ rc=$?
+ if [ "$rc" -eq 0 ]; then
+ fail "${desc} — gate exited 0 instead of refusing"
+ return
+ fi
+ if ! grep -Fq "$needle" <<<"$out"; then
+ fail "${desc} — exit ${rc} but guard text '${needle}' missing; got: $(head -c 400 <<<"$out")"
+ return
+ fi
+ # Positive shape: the refusal must come from the guard block, before any
+ # cluster mutation phase starts.
+ if grep -Fq 'Creating synthetic fleet' <<<"$out" || grep -Fq 'Baseline: sampling' <<<"$out"; then
+ fail "${desc} — guard fired but the gate still progressed past the guard block"
+ return
+ fi
+ pass "$desc"
+}
+
+# 1. Missing KUBECONTEXT / E2E_K8S_CONTEXT: the very first guard must refuse.
+run_gate_expecting \
+ "refuses when neither KUBECONTEXT nor E2E_K8S_CONTEXT is set" \
+ "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." \
+ PATH="$PATH"
+
+# 2a. Non-branch context: the shared `minikube` context is not branch-scoped.
+run_gate_expecting \
+ "refuses fault injection on the shared 'minikube' context" \
+ "Refusing rollout fault injection on non-branch context 'minikube'." \
+ PATH="$PATH" KUBECONTEXT=minikube
+
+# 2b. Prod-shaped context: is_branch_scoped_e2e_context hard-rejects *prod*
+# even when the rest of the name is branch-shaped.
+run_gate_expecting \
+ "refuses fault injection on a *prod* context" \
+ "Refusing rollout fault injection on non-branch context 'clerum-prod-deadbeef'." \
+ PATH="$PATH" KUBECONTEXT=clerum-prod-deadbeef
+
+# 3. Missing fault-injection acknowledgement: a branch-scoped context alone is
+# not consent to roll (and possibly break) the HCC deployment.
+run_gate_expecting \
+ "refuses without E2E_HCC_ROLLOUT_FAULT_INJECTION=1" \
+ "Set E2E_HCC_ROLLOUT_FAULT_INJECTION=1 to acknowledge that this gate rolls" \
+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX"
+
+# 4. EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive modes: the D1b outage
+# reproduction and the evenfire#391 undo evidence must never be conflated
+# into one run.
+run_gate_expecting \
+ "refuses EXPECT_STUCK=1 together with EXPECT_RECOVERY=1" \
+ "EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive." \
+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX" E2E_HCC_ROLLOUT_FAULT_INJECTION=1 \
+ EXPECT_STUCK=1 EXPECT_RECOVERY=1
+
+# 5a. EXPECT_STUCK must be 0 or 1 — anything else is a typo, not a mode.
+run_gate_expecting \
+ "refuses an invalid EXPECT_STUCK value" \
+ "EXPECT_STUCK must be 0 (measure a healthy rollout window) or 1" \
+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX" E2E_HCC_ROLLOUT_FAULT_INJECTION=1 \
+ EXPECT_STUCK=2
+
+# 5b. EXPECT_RECOVERY must be 0 or 1.
+run_gate_expecting \
+ "refuses an invalid EXPECT_RECOVERY value" \
+ "EXPECT_RECOVERY must be 0 or 1." \
+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX" E2E_HCC_ROLLOUT_FAULT_INJECTION=1 \
+ EXPECT_STUCK=0 EXPECT_RECOVERY=yes
+
+# 6. jq missing: build a minimal PATH containing only the external tools the
+# gate needs BEFORE its jq guard (dirname for SCRIPT_DIR, grep for
+# is_branch_scoped_e2e_context) and deliberately no jq. The gate must die
+# on its own jq guard, not on an accidental command-not-found later.
+NOJQ_BIN="${WORKDIR}/nojq-bin"
+mkdir -p "$NOJQ_BIN"
+for tool in dirname grep; do
+ ln -s "$(command -v "$tool")" "${NOJQ_BIN}/${tool}"
+done
+run_gate_expecting \
+ "refuses when jq is not on PATH" \
+ "jq is required" \
+ PATH="$NOJQ_BIN" KUBECONTEXT="$BRANCH_CTX"
+
+# 7. Wrong minikube node: a PATH-stubbed fake kubectl reports a node labelled
+# with a DIFFERENT profile. The stub records every invocation, refuses any
+# mutating verb, and only answers `get nodes` on the explicit --context.
+# That way a guard reorder that mutates the cluster BEFORE the node-label
+# check cannot hide behind a stub that always prints JSON and exits 0.
+FAKE_KUBECTL_BIN="${WORKDIR}/fake-kubectl-bin"
+FAKE_KUBECTL_LOG="${WORKDIR}/fake-kubectl.log"
+mkdir -p "$FAKE_KUBECTL_BIN"
+cat >"${FAKE_KUBECTL_BIN}/kubectl" <<EOF
+#!/usr/bin/env bash
+printf '%s\n' "\$*" >>"${FAKE_KUBECTL_LOG}"
+ctx=""
+args=()
+while [ "\$#" -gt 0 ]; do
+ case "\$1" in
+ --context) ctx="\$2"; shift 2 ;;
+ --context=*) ctx="\${1#--context=}"; shift ;;
+ *) args+=("\$1"); shift ;;
+ esac
+done
+verb="\${args[0]:-}"
+case "\$verb" in
+ set|delete|apply|patch|rollout|scale|create|replace|annotate|label|exec|drain|edit|expose|autoscale|cordon|uncordon|taint)
+ echo "fake-kubectl: refusing mutating verb \$verb" >&2
+ exit 2
+ ;;
+esac
+if [ "\$ctx" != "${BRANCH_CTX}" ] || [ "\$verb" != "get" ] || [ "\${args[1]:-}" != "nodes" ]; then
+ echo "fake-kubectl: unexpected invocation ctx=\$ctx argv=\${args[*]}" >&2
+ exit 2
+fi
+echo '{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"clerum-other-branch-cafebabe"}}}]}'
+EOF
+chmod +x "${FAKE_KUBECTL_BIN}/kubectl"
+run_gate_expecting \
+ "refuses when the node label belongs to another minikube profile" \
+ "Refusing rollout fault injection: target is not this profile's minikube node." \
+ PATH="${FAKE_KUBECTL_BIN}:$PATH" KUBECONTEXT="$BRANCH_CTX" \
+ E2E_HCC_ROLLOUT_FAULT_INJECTION=1 EXPECT_STUCK=0 EXPECT_RECOVERY=0
+if [ ! -s "$FAKE_KUBECTL_LOG" ]; then
+ fail "fake kubectl recorded no invocations — the node-label guard may not have run"
+elif ! grep -Fq -- "--context ${BRANCH_CTX} get nodes" "$FAKE_KUBECTL_LOG"; then
+ fail "fake kubectl was not asked for get nodes on the explicit context; got: $(tr '\n' '|' <"$FAKE_KUBECTL_LOG")"
+elif grep -Eq '(^| )(set|delete|apply|patch|rollout|scale|create|replace|annotate|label)( |$)' "$FAKE_KUBECTL_LOG"; then
+ fail "fake kubectl recorded a mutating verb before the node-label guard; got: $(tr '\n' '|' <"$FAKE_KUBECTL_LOG")"
+else
+ pass "fake kubectl saw only get nodes --context ${BRANCH_CTX} (no mutation)"
+fi
+
+exit "$FAIL"