Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(#391): close HCC rollout-undo coverage holes in PR 416
Alfredo Lopez7e647c1982c5
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 35b2d3d3a..b0ed09c7b 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -400,6 +400,9 @@ jobs:- name: Test HCC rollout readiness gate contractrun: bash scripts/tests/test-hcc-rollout-readiness-gate.sh+ - name: Test HCC rollout readiness guards (executable, cluster-free)+ run: bash scripts/tests/test-hcc-rollout-readiness-guards.sh+- name: Test HCC API gateway readiness separationrun: bash scripts/tests/test-hcc-api-gateway-readiness.shdiff --git a/Makefile b/Makefileindex 1cc8deaa8..11408f3ce 100644--- a/Makefile+++ b/Makefile@@ -961,6 +961,15 @@ test-e2e-hcc-mcp-context-readiness: ## Prove HCC readiness during exact MCP/Cont@test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }E2E_HCC_MCP_READINESS_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-mcp-context-readiness.sh+.PHONY: test-e2e-hcc-rollout-readiness+test-e2e-hcc-rollout-readiness: ## Measure the HCC Recreate rollout window (D1/c4). EXPECT_STUCK=1 reproduces the D1b outage; EXPECT_RECOVERY=1 proves the evenfire#391 rollout-undo path. The two flags are EXCLUSIVE; default both 0 = healthy measurement.+ @echo "Running HCC rollout readiness gate..."+ @test -n "$(E2E_EXPECTED_PRE_GATE_GATE)" || { echo "Set E2E_EXPECTED_PRE_GATE_GATE to the gate recorded by the branch-owned pre-gate sync" >&2; exit 1; }+ E2E_HCC_ROLLOUT_FAULT_INJECTION=1 E2E_EXPECTED_PRE_GATE_GATE="$(E2E_EXPECTED_PRE_GATE_GATE)" \+ MINIKUBE_PROFILE=$(E2E_KUBECONTEXT) KUBECONTEXT=$(E2E_KUBECONTEXT) \+ EXPECT_STUCK=$(EXPECT_STUCK) EXPECT_RECOVERY=$(EXPECT_RECOVERY) \+ bash scripts/e2e/e2e-hcc-rollout-readiness.sh+.PHONY: test-e2e-stateless-multinodetest-e2e-stateless-multinode: ## Run stateless multi-node lane (opt-in: STATELESS_MULTINODE_GATE=1; needs >=2 schedulable nodes; exit 3 = cross-node UNVERIFIED)@echo "Running stateless multi-node lane..."diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 84cf93bac..8725e9991 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.229",+ "version": "0.1.230","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.229",+ "version": "0.1.230","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex efe9ae9da..777adbc22 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.229",+ "version": "0.1.230","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/deploymentStrategy.test.ts b/host-context-controller/src/deploymentStrategy.test.tsindex 0779328a6..ff51e3259 100644--- a/host-context-controller/src/deploymentStrategy.test.ts+++ b/host-context-controller/src/deploymentStrategy.test.ts@@ -17,12 +17,16 @@ import path from 'node:path'* only way to honour the documented single-writer assumption during a rollout.*/describe('host-context-controller Deployment strategy', () => {- const docs = loadAll(- readFileSync(- path.join(__dirname, '../../deploy/base/control-plane/host-context-controller.yaml'),- 'utf8'- )- ) as Array<{ kind?: string; metadata?: { name?: string }; spec?: Record<string, unknown> }>+ const yamlPath = path.join(+ __dirname,+ '../../deploy/base/control-plane/host-context-controller.yaml'+ )+ const yamlSource = readFileSync(yamlPath, 'utf8')+ const docs = loadAll(yamlSource) as Array<{+ kind?: string+ metadata?: { name?: string }+ spec?: Record<string, unknown>+ }>const deployment = docs.find(d => d?.kind === 'Deployment' && d?.metadata?.name === 'host-context-controller'@@ -65,4 +69,59 @@ describe('host-context-controller Deployment strategy', () => {// fleet.expect(deployment?.spec?.progressDeadlineSeconds).toBe(1200)})++ /**+ * evenfire#391 detection-vs-recovery split. The D1b premise is a SET, not+ * three independent knobs: Recreate (terminate-before-create, zero fallback+ * replicas) + replicas:1 (single writer) + progressDeadlineSeconds:1200+ * (detection). Change any one of them and the #391 recovery contract — the+ * evenfire-infra deploy pipeline performing `rollout undo --to-revision`,+ * never Kubernetes and never HCC code — is reasoning about a deployment+ * that no longer exists. These pins fail if the set is broken up or if the+ * documented ownership/detection-only contract is silently deleted.+ */+ describe('evenfire#391 detection-vs-recovery split', () => {+ it('keeps the D1b premise set together: Recreate + replicas:1 + deadline 1200', () => {+ // Asserted as one test on purpose: a partial edit (e.g. bumping replicas+ // while keeping Recreate, or flipping the strategy while keeping the+ // 1200s deadline) must fail HERE as a broken premise set, not pass three+ // unrelated single-field tests.+ const strategy = deployment?.spec?.strategy as+ | { type?: string; rollingUpdate?: unknown }+ | undefined+ expect({+ replicas: deployment?.spec?.replicas,+ strategyType: strategy?.type,+ rollingUpdate: strategy?.rollingUpdate,+ progressDeadlineSeconds: deployment?.spec?.progressDeadlineSeconds,+ }).toEqual({+ replicas: 1,+ strategyType: 'Recreate',+ rollingUpdate: undefined,+ progressDeadlineSeconds: 1200,+ })+ })++ it('documents that the progress deadline is DETECTION ONLY and never rolls back', () => {+ // The deadline flips Progressing=False and fails `kubectl rollout+ // status`; it does NOT undo anything. If this documented contract is+ // deleted from the manifest, a future reader will assume Kubernetes+ // self-heals a botched Recreate rollout — the exact D1b failure mode.+ expect(yamlSource).toContain('DETECTION ONLY')+ expect(yamlSource).toContain('it does not roll back')+ expect(yamlSource).toContain('Kubernetes will not self-heal')+ })++ it('pins recovery ownership to the evenfire-infra rollout undo (evenfire#391)', () => {+ // Recovery is owned by the evenfire-infra deploy workflows via+ // `rollout undo --to-revision` for this Deployment only (evenfire#391).+ // HCC must NOT grow its own undo runtime, and the manifest must keep+ // naming the real owner so a silent deletion of the ownership comment+ // fails loudly here.+ expect(yamlSource).toContain('evenfire-infra')+ expect(yamlSource).toContain('rollout undo --to-revision')+ expect(yamlSource).toContain('evenfire#391')+ expect(yamlSource).toContain('The deploy job still fails after a successful undo')+ })+ })})diff --git a/scripts/e2e/e2e-hcc-rollout-readiness.sh b/scripts/e2e/e2e-hcc-rollout-readiness.shindex b6f5e699a..9261f7082 100755--- a/scripts/e2e/e2e-hcc-rollout-readiness.sh+++ b/scripts/e2e/e2e-hcc-rollout-readiness.sh@@ -19,11 +19,14 @@## Modes:# EXPECT_RECOVERY=1 — exclusive with EXPECT_STUCK. Botch the image, prove the-# D1b 503 outage, then run last-good revision restore as the TEST ACTION-# (not cleanup). Recovery must recertify /ready 200, restore the original-# image and last-good revision, and replace the botched pod. An undo no-op-# is FAIL. After a proven undo, IMAGE_BROKEN=0 so cleanup does not set the-# image again. This is the evenfire#391 evidence path.+# D1b 503 outage (same sustained-outage pins as EXPECT_STUCK: >=1 503,+# zero 503->200, maxstreak > healthy budget), then run last-good revision+# restore as the TEST ACTION (not cleanup). Recovery must recertify /ready+# 200, restore the original image and last-good revision, and replace the+# botched pod. An undo no-op is FAIL. IMAGE_BROKEN stays 1 unless every+# post-undo proof passed (e2e_fail==0), so a failed image/revision/uid/hold+# still lets cleanup restore the original image. This is the evenfire#391+# evidence path.# EXPECT_STUCK=0 (default) — healthy rollout via `kubectl rollout restart`.# rollout restart mutates the pod template (restartedAt annotation), which# drives the SAME Deployment machinery — terminate old, then create new —@@ -463,6 +466,9 @@ else[ "$stuck_transitions" -eq 0 ] &&ok "no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal" ||fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"+ [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&+ ok "SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip" ||+ fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"live_revision_before_undo="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"[ "$live_revision_before_undo" != "$LAST_GOOD_REVISION" ] ||@@ -475,7 +481,6 @@ elsefail "last-known-good revision ${LAST_GOOD_REVISION} did not become Ready"wait_until 90 "HCC /ready after undo" hcc_ready_now ||fail "HCC /ready did not return 200 after last-good restore"- IMAGE_BROKEN=0recovered_image="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"recovered_revision="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"@@ -499,6 +504,10 @@ else[ "$hold_total" -gt 0 ] && [ "$hold_503" -eq 0 ] &&ok "post-undo hold: ${hold_total} samples over ${STABILITY_WINDOW_SEC}s, zero 503" ||fail "post-undo hold saw ${hold_503} 503(s) across ${hold_total} sample(s)"+ # fail() records and continues, so only clear the broken-image flag when+ # every post-undo proof above actually passed. Otherwise cleanup must still+ # restore the original image.+ [ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0fidiff --git a/scripts/tests/test-hcc-rollout-readiness-gate.sh b/scripts/tests/test-hcc-rollout-readiness-gate.shindex b7f7ded3e..251f6fe5b 100755--- a/scripts/tests/test-hcc-rollout-readiness-gate.sh+++ b/scripts/tests/test-hcc-rollout-readiness-gate.sh@@ -123,17 +123,56 @@ grep -Fq 'create_synthetic_fleet' "$GATE" &&# 12. evenfire#391 recovery mode: exclusive with EXPECT_STUCK, undo is the test-# action (not cleanup), last-good revision is captured, IMAGE_BROKEN is-# cleared after a proven undo, and a no-op undo fails.+# action (not cleanup), last-good revision is captured, the pre-undo outage+# is the same sustained D1b pin as EXPECT_STUCK, IMAGE_BROKEN is cleared+# only AFTER every post-undo proof, and a no-op undo fails.+recovery_block="$(awk '/FASE D \(recovery, evenfire#391\)/,0' "$GATE")"+image_clear_line="$(grep -n 'IMAGE_BROKEN=0' <<<"$recovery_block" | head -1 | cut -d: -f1)"+hold_assert_line="$(grep -n 'post-undo hold' <<<"$recovery_block" | head -1 | cut -d: -f1)"grep -Fq 'EXPECT_RECOVERY' "$GATE" &&grep -Fq 'EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive' "$GATE" &&grep -Fq 'TEST ACTION' "$GATE" &&grep -Fq -- '--to-revision' "$GATE" &&grep -Fq 'LAST_GOOD_REVISION' "$GATE" &&- grep -Fq 'IMAGE_BROKEN=0' "$GATE" &&+ grep -Eq 'stuck_maxstreak.*-gt.*ROLLOUT_DOWNTIME_BUDGET_SEC' <<<"$recovery_block" &&+ grep -Fq '[ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0' <<<"$recovery_block" &&+ [ -n "$image_clear_line" ] && [ -n "$hold_assert_line" ] &&+ [ "$image_clear_line" -gt "$hold_assert_line" ] &&grep -Fq 'undo was a no-op' "$GATE" &&! grep -q 'EXPECT_RECOVERY=1' <<<"$(awk '/^cleanup\(\)/,/^}/' "$GATE")" &&pass "EXPECT_RECOVERY undoes to last-good outside cleanup and fails a no-op" ||- fail "EXPECT_RECOVERY is missing, shares cleanup, or can pass on a no-op undo"+ fail "EXPECT_RECOVERY is missing, shares cleanup, skips sustained outage, or clears IMAGE_BROKEN before proofs"++# 13. Makefile entry point: the gate is reachable through the same shaped+# target as the sibling HCC gates — fault-injection ack + pre-gate-sync+# expectation + explicit profile=context pin, with the exclusive+# EXPECT_STUCK/EXPECT_RECOVERY modes surfaced (and documented as+# exclusive) rather than hardcoded.+MAKEFILE="${ROOT}/Makefile"+make_target_block="$(awk '/^test-e2e-hcc-rollout-readiness:/,/^$/' "$MAKEFILE")"+grep -Fq '.PHONY: test-e2e-hcc-rollout-readiness' "$MAKEFILE" &&+ grep -Fq 'E2E_HCC_ROLLOUT_FAULT_INJECTION=1' <<<"$make_target_block" &&+ grep -Fq 'E2E_EXPECTED_PRE_GATE_GATE' <<<"$make_target_block" &&+ grep -Fq 'MINIKUBE_PROFILE=$(E2E_KUBECONTEXT)' <<<"$make_target_block" &&+ grep -Fq 'KUBECONTEXT=$(E2E_KUBECONTEXT)' <<<"$make_target_block" &&+ grep -Fq 'EXPECT_STUCK=$(EXPECT_STUCK)' <<<"$make_target_block" &&+ grep -Fq 'EXPECT_RECOVERY=$(EXPECT_RECOVERY)' <<<"$make_target_block" &&+ grep -Fq 'scripts/e2e/e2e-hcc-rollout-readiness.sh' <<<"$make_target_block" &&+ grep -Fq 'EXCLUSIVE' <<<"$make_target_block" &&+ pass "Makefile target test-e2e-hcc-rollout-readiness keeps ack + profile=context + modes + gate invocation" ||+ fail "Makefile target test-e2e-hcc-rollout-readiness is missing or lost its ack/profile/mode/gate contract"++# 14. Executable guard suite: CI must RUN the cluster-free guard tests (not+# only this grep contract), and that suite must actually EXECUTE the gate+# — a guards script degraded back to grep would be this file twice.+GUARDS="${ROOT}/scripts/tests/test-hcc-rollout-readiness-guards.sh"+CI_WORKFLOW="${ROOT}/.github/workflows/ci-public.yml"+[ -x "$GUARDS" ] &&+ bash -n "$GUARDS" &&+ grep -Fq '"$BASH_BIN" "$GATE"' "$GUARDS" &&+ grep -Fq 'EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive.' "$GUARDS" &&+ grep -Fq 'scripts/tests/test-hcc-rollout-readiness-guards.sh' "$CI_WORKFLOW" &&+ pass "executable guard suite exists, executes the gate, and is wired into CI" ||+ fail "guard suite missing/non-executable, no longer executes the gate, or dropped from CI"exit "$FAIL"diff --git a/scripts/tests/test-hcc-rollout-readiness-guards.sh b/scripts/tests/test-hcc-rollout-readiness-guards.shnew file mode 100755index 000000000..3564e43c3--- /dev/null+++ b/scripts/tests/test-hcc-rollout-readiness-guards.sh@@ -0,0 +1,185 @@+#!/usr/bin/env bash+# Executable, cluster-free guard tests for scripts/e2e/e2e-hcc-rollout-readiness.sh+# (the evenfire#391 / PR #382 D1b gate).+#+# The sibling contract test (test-hcc-rollout-readiness-gate.sh) is grep-based:+# it proves the guard TEXT exists in the gate, not that the guards RUN. This+# suite actually EXECUTES the gate under a fully isolated environment (env -i)+# and asserts each fail-closed guard fires: exit != 0 AND the guard's own+# stable die/echo text on stderr/stdout. A refactor that keeps the strings but+# reorders or dead-codes the guards fails HERE.+#+# Every case below exits inside the guard block (gate lines before the first+# `kctl get nodes`) or against a PATH-stubbed fake kubectl — no case ever+# contacts a real cluster or the developer's kubeconfig.+set -u+FAIL=0+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+GATE="${ROOT}/scripts/e2e/e2e-hcc-rollout-readiness.sh"+BASH_BIN="$(command -v bash)"+pass() { echo "PASS: $1"; }+fail() {+ echo "FAIL: $1"+ FAIL=1+}++command -v jq >/dev/null 2>&1 || {+ echo "FAIL: this harness itself requires jq on PATH (the gate's happy path needs it)" >&2+ exit 1+}++# Branch-shaped fake context: passes is_branch_scoped_e2e_context+# (^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$) so each case reaches the guard+# under test instead of dying on the context-shape guard.+BRANCH_CTX="clerum-feat-issue-391-deadbeef"++WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/hcc-rollout-guards.XXXXXX")"+trap 'rm -rf "$WORKDIR"' EXIT+# Point KUBECONFIG at an empty throwaway file in every case: even a bug that+# slipped past PATH isolation must never read or mutate the developer's+# kubeconfig.+FAKE_KUBECONFIG="${WORKDIR}/empty-kubeconfig"+: >"$FAKE_KUBECONFIG"++# run_gate_expecting <description> <needle> [VAR=value ...]+# Runs the gate under `env -i` (nothing leaks from the developer shell) with+# exactly the provided variables, expecting: non-zero exit, the guard's stable+# needle in the combined output, and NO evidence the gate got past its guard+# block into cluster mutation.+run_gate_expecting() {+ local desc="$1" needle="$2"+ shift 2+ local out rc+ out="$(env -i HOME="$HOME" TMPDIR="${WORKDIR}" KUBECONFIG="$FAKE_KUBECONFIG" "$@" \+ "$BASH_BIN" "$GATE" 2>&1)"+ rc=$?+ if [ "$rc" -eq 0 ]; then+ fail "${desc} — gate exited 0 instead of refusing"+ return+ fi+ if ! grep -Fq "$needle" <<<"$out"; then+ fail "${desc} — exit ${rc} but guard text '${needle}' missing; got: $(head -c 400 <<<"$out")"+ return+ fi+ # Positive shape: the refusal must come from the guard block, before any+ # cluster mutation phase starts.+ if grep -Fq 'Creating synthetic fleet' <<<"$out" || grep -Fq 'Baseline: sampling' <<<"$out"; then+ fail "${desc} — guard fired but the gate still progressed past the guard block"+ return+ fi+ pass "$desc"+}++# 1. Missing KUBECONTEXT / E2E_K8S_CONTEXT: the very first guard must refuse.+run_gate_expecting \+ "refuses when neither KUBECONTEXT nor E2E_K8S_CONTEXT is set" \+ "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." \+ PATH="$PATH"++# 2a. Non-branch context: the shared `minikube` context is not branch-scoped.+run_gate_expecting \+ "refuses fault injection on the shared 'minikube' context" \+ "Refusing rollout fault injection on non-branch context 'minikube'." \+ PATH="$PATH" KUBECONTEXT=minikube++# 2b. Prod-shaped context: is_branch_scoped_e2e_context hard-rejects *prod*+# even when the rest of the name is branch-shaped.+run_gate_expecting \+ "refuses fault injection on a *prod* context" \+ "Refusing rollout fault injection on non-branch context 'clerum-prod-deadbeef'." \+ PATH="$PATH" KUBECONTEXT=clerum-prod-deadbeef++# 3. Missing fault-injection acknowledgement: a branch-scoped context alone is+# not consent to roll (and possibly break) the HCC deployment.+run_gate_expecting \+ "refuses without E2E_HCC_ROLLOUT_FAULT_INJECTION=1" \+ "Set E2E_HCC_ROLLOUT_FAULT_INJECTION=1 to acknowledge that this gate rolls" \+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX"++# 4. EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive modes: the D1b outage+# reproduction and the evenfire#391 undo evidence must never be conflated+# into one run.+run_gate_expecting \+ "refuses EXPECT_STUCK=1 together with EXPECT_RECOVERY=1" \+ "EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive." \+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX" E2E_HCC_ROLLOUT_FAULT_INJECTION=1 \+ EXPECT_STUCK=1 EXPECT_RECOVERY=1++# 5a. EXPECT_STUCK must be 0 or 1 — anything else is a typo, not a mode.+run_gate_expecting \+ "refuses an invalid EXPECT_STUCK value" \+ "EXPECT_STUCK must be 0 (measure a healthy rollout window) or 1" \+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX" E2E_HCC_ROLLOUT_FAULT_INJECTION=1 \+ EXPECT_STUCK=2++# 5b. EXPECT_RECOVERY must be 0 or 1.+run_gate_expecting \+ "refuses an invalid EXPECT_RECOVERY value" \+ "EXPECT_RECOVERY must be 0 or 1." \+ PATH="$PATH" KUBECONTEXT="$BRANCH_CTX" E2E_HCC_ROLLOUT_FAULT_INJECTION=1 \+ EXPECT_STUCK=0 EXPECT_RECOVERY=yes++# 6. jq missing: build a minimal PATH containing only the external tools the+# gate needs BEFORE its jq guard (dirname for SCRIPT_DIR, grep for+# is_branch_scoped_e2e_context) and deliberately no jq. The gate must die+# on its own jq guard, not on an accidental command-not-found later.+NOJQ_BIN="${WORKDIR}/nojq-bin"+mkdir -p "$NOJQ_BIN"+for tool in dirname grep; do+ ln -s "$(command -v "$tool")" "${NOJQ_BIN}/${tool}"+done+run_gate_expecting \+ "refuses when jq is not on PATH" \+ "jq is required" \+ PATH="$NOJQ_BIN" KUBECONTEXT="$BRANCH_CTX"++# 7. Wrong minikube node: a PATH-stubbed fake kubectl reports a node labelled+# with a DIFFERENT profile. The stub records every invocation, refuses any+# mutating verb, and only answers `get nodes` on the explicit --context.+# That way a guard reorder that mutates the cluster BEFORE the node-label+# check cannot hide behind a stub that always prints JSON and exits 0.+FAKE_KUBECTL_BIN="${WORKDIR}/fake-kubectl-bin"+FAKE_KUBECTL_LOG="${WORKDIR}/fake-kubectl.log"+mkdir -p "$FAKE_KUBECTL_BIN"+cat >"${FAKE_KUBECTL_BIN}/kubectl" <<EOF+#!/usr/bin/env bash+printf '%s\n' "\$*" >>"${FAKE_KUBECTL_LOG}"+ctx=""+args=()+while [ "\$#" -gt 0 ]; do+ case "\$1" in+ --context) ctx="\$2"; shift 2 ;;+ --context=*) ctx="\${1#--context=}"; shift ;;+ *) args+=("\$1"); shift ;;+ esac+done+verb="\${args[0]:-}"+case "\$verb" in+ set|delete|apply|patch|rollout|scale|create|replace|annotate|label|exec|drain|edit|expose|autoscale|cordon|uncordon|taint)+ echo "fake-kubectl: refusing mutating verb \$verb" >&2+ exit 2+ ;;+esac+if [ "\$ctx" != "${BRANCH_CTX}" ] || [ "\$verb" != "get" ] || [ "\${args[1]:-}" != "nodes" ]; then+ echo "fake-kubectl: unexpected invocation ctx=\$ctx argv=\${args[*]}" >&2+ exit 2+fi+echo '{"items":[{"metadata":{"labels":{"minikube.k8s.io/name":"clerum-other-branch-cafebabe"}}}]}'+EOF+chmod +x "${FAKE_KUBECTL_BIN}/kubectl"+run_gate_expecting \+ "refuses when the node label belongs to another minikube profile" \+ "Refusing rollout fault injection: target is not this profile's minikube node." \+ PATH="${FAKE_KUBECTL_BIN}:$PATH" KUBECONTEXT="$BRANCH_CTX" \+ E2E_HCC_ROLLOUT_FAULT_INJECTION=1 EXPECT_STUCK=0 EXPECT_RECOVERY=0+if [ ! -s "$FAKE_KUBECTL_LOG" ]; then+ fail "fake kubectl recorded no invocations — the node-label guard may not have run"+elif ! grep -Fq -- "--context ${BRANCH_CTX} get nodes" "$FAKE_KUBECTL_LOG"; then+ fail "fake kubectl was not asked for get nodes on the explicit context; got: $(tr '\n' '|' <"$FAKE_KUBECTL_LOG")"+elif grep -Eq '(^| )(set|delete|apply|patch|rollout|scale|create|replace|annotate|label)( |$)' "$FAKE_KUBECTL_LOG"; then+ fail "fake kubectl recorded a mutating verb before the node-label guard; got: $(tr '\n' '|' <"$FAKE_KUBECTL_LOG")"+else+ pass "fake kubectl saw only get nodes --context ${BRANCH_CTX} (no mutation)"+fi++exit "$FAIL"