Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix: detect real kubectl port-forward argv in T2

Alfredo Lopez04e1a2a8fa7e
diff --git a/.cursor/rules/minikube-t2-harness.mdc b/.cursor/rules/minikube-t2-harness.mdc
index 3a640e674..5634fbfea 100644
--- a/.cursor/rules/minikube-t2-harness.mdc
+++ b/.cursor/rules/minikube-t2-harness.mdc
@@ -34,8 +34,8 @@ Invariants that must never regress:
`pre-gate-sync` with `--skip-port-forwards`.
- `t2_process_check` accepts only real `kubectl` (`comm=kubectl`) port-forward
PIDs recorded in the profile cache or legacy `/tmp/pf-<profile>-*.pid`.
- The awk argv0 filter anchors on the `ps -ef` TIME token in both macOS
- (`N:MM[.ss]`) and Linux (`HH:MM:SS`) forms.
+ awk is a loose pre-filter: `kubectl` as argv0/path token and a later
+ standalone `port-forward` token (flags may sit between them).
- Stable failure codes (`BOOTSTRAP_REQUIRED`, `HEAD_MARKER_MISMATCH`,
`PORT_FORWARD_CONFLICT`, `PROFILE_BUSY`, ...) keep their meanings.
diff --git a/.cursor/skills/minikube-t0-t1-t2/reference.md b/.cursor/skills/minikube-t0-t1-t2/reference.md
index 646827216..b6154f3ec 100644
--- a/.cursor/skills/minikube-t0-t1-t2/reference.md
+++ b/.cursor/skills/minikube-t0-t1-t2/reference.md
@@ -26,9 +26,9 @@ Companion to `SKILL.md`. Source of truth: `scripts/minikube/t2.sh`,
argv** when avoidable. `t2_process_check` accepts only real `kubectl`
processes (`comm=kubectl`) whose PIDs are recorded in the profile cache or
legacy `/tmp/pf-<profile>-*.pid`; a lookalike wrapper trips
- `PORT_FORWARD_CONFLICT`. The argv0 pre-filter keys off the `ps -ef` TIME
- field immediately before `kubectl` and must match both macOS `N:MM[.ss]`
- and Linux `HH:MM:SS`.
+ `PORT_FORWARD_CONFLICT`. The awk pre-filter matches `kubectl` as an argv0
+ or path token plus a later standalone `port-forward` token; wrappers that
+ only mention those words are rejected by `comm`.
- **Do not commit** `.local-notes/`, lockfiles produced by an incidental
`npm ci`, generated ports/profile metadata, or anything secret-like. Run
`make minikube-t2-public-boundary` first.
diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.md
index 9ccc2323e..05ba7a3a8 100644
--- a/docs/testing/minikube-t2-runbook.md
+++ b/docs/testing/minikube-t2-runbook.md
@@ -107,8 +107,10 @@ Never remove a lock with a live owner, and never remove the whole lock root.
The runner checks required namespaces, Services, Secret names, ConfigMap names,
the `control-postgres` PVC/deployment, all required deployment readiness, the
image manifest/source, and real `kubectl port-forward` processes for this
-profile. The conflict check keys argv0 off the `ps -ef` TIME field and accepts
-both macOS `N:MM[.ss]` and Linux `HH:MM:SS`. Allowed port-forward PIDs live in
+profile. The conflict check is a loose argv pre-filter (`kubectl` as argv0 or
+path token plus a later standalone `port-forward` token; flags may sit
+between them) plus a `comm=kubectl` hardening and a PID allowlist.
+Allowed port-forward PIDs live in
`$HOME/.cache/clerum/minikube-profiles/<profile>/pids/*.pid` (and the
legacy `/tmp/pf-<profile>-*.pid` files written by `pf-all-stack.sh`).
`make minikube-t2` invokes `pre-gate-sync` with `--skip-port-forwards` so the
diff --git a/scripts/e2e/e2e-hcc-rollout-readiness.sh b/scripts/e2e/e2e-hcc-rollout-readiness.sh
index 9261f7082..a3f3a66a8 100755
--- a/scripts/e2e/e2e-hcc-rollout-readiness.sh
+++ b/scripts/e2e/e2e-hcc-rollout-readiness.sh
@@ -13,9 +13,10 @@
# measures it against the wall clock, over a synthetic clerum-dev-scale fleet
# (the initial LISTs and the revocation pass are part of the window), and pins
# the D1b failure mode empirically: `progressDeadlineSeconds: 1200` is
-# DETECTION only — a rollout whose new pod can never certify leaves the HCC
-# control plane down until a manual `kubectl rollout undo`, because Recreate
-# keeps zero old replicas to fall back on.
+# DETECTION only — a botched Recreate rollout leaves the HCC down until
+# evenfire-infra deploy workflows run automatic `rollout undo --to-revision`.
+# This e2e proves that operator action; cleanup `set image` is only the
+# leftover restore fallback.
#
# Modes:
# EXPECT_RECOVERY=1 — exclusive with EXPECT_STUCK. Botch the image, prove the
@@ -44,9 +45,10 @@
# total outage, empirically. The sustained 503 streak — longer than the
# healthy budget and still open at observation end — is the EXPECTED
# evidence. The observation is bounded (STUCK_OBSERVE_SEC) and cleanup
-# performs the manual-operator recovery (restore the original image, wait
+# performs the fallback image restore (restore the original image, wait
# for re-certification), so the gate always terminates loudly instead of
-# hanging alongside the outage it reproduces.
+# hanging alongside the outage it reproduces. Production recovery is the
+# infra-owned automatic `rollout undo --to-revision` (evenfire#391).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/e2e/e2e-lib.sh
@@ -221,6 +223,76 @@ botched_image_ref() {
printf '%s:e2e-botched-%s' "$base" "$RUN_ID"
}
+# Shared D1b outage pins for EXPECT_STUCK and EXPECT_RECOVERY. flavor=stuck
+# keeps the longer D1b strings; flavor=recovery keeps the pre-undo variants.
+# A failed pod-UID listing must not count as "old pod terminated".
+assert_botched_outage() {
+ local flavor="$1" old_uid_query
+ case "$flavor" in
+ stuck | recovery) ;;
+ *) die "assert_botched_outage: flavor must be stuck or recovery, got '${flavor}'" ;;
+ esac
+
+ deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"
+ old_uid_live=-1
+ if old_uid_query="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
+ -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}')"; then
+ old_uid_live="$(printf '%s\n' "$old_uid_query" | grep -cxF "$OLD_POD_UID" || true)"
+ else
+ fail "could not list HCC pod UIDs — refusing to treat a query failure as old-pod termination"
+ fi
+ stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"
+ replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"
+
+ [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||
+ fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"
+ if [ "$flavor" = recovery ]; then
+ [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&
+ ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||
+ fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE}"
+ [ "$old_uid_live" = 0 ] &&
+ ok "Recreate terminated the old HEALTHY pod before any viable replacement existed" ||
+ fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout"
+ if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then
+ ok "replacement pod is pinned by the injected image failure: ${replacement_row}"
+ else
+ fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none})"
+ fi
+ [ "$stuck_503" -ge 1 ] &&
+ ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||
+ fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"
+ [ "$stuck_transitions" -eq 0 ] &&
+ ok "no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal" ||
+ fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"
+ [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&
+ ok "SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip" ||
+ fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"
+ else
+ [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&
+ ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||
+ fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE} — the outage (if any) has a different cause"
+ [ "$old_uid_live" = 0 ] &&
+ ok "Recreate terminated the old HEALTHY pod before any viable replacement existed — zero fallback replicas (the D1b mechanism)" ||
+ fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout — the terminate-before-create premise did not hold"
+ if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then
+ ok "replacement pod is pinned by the injected image failure: ${replacement_row}"
+ else
+ fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none}) — the outage is not attributable to the injected bad rollout"
+ fi
+ [ "$stuck_503" -ge 1 ] &&
+ ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||
+ fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"
+ [ "$stuck_transitions" -eq 0 ] &&
+ ok "no 503->200 transition ever closed the outage — Kubernetes did NOT self-heal the botched Recreate rollout" ||
+ fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) — the D1b premise that recovery requires an explicit rollback is refuted; investigate before trusting this gate"
+ [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&
+ ok "SUSTAINED outage measured: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) and still open at observation end — recovery requires an explicit rollback (infra-owned rollout undo --to-revision in production; cleanup's set image restore here) (D1b reproduced)" ||
+ fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — the botched rollout did not produce the D1b total outage"
+ fi
+}
+
print_repair_instructions() {
cat >&2 <<EOF
HCC rollout gate cleanup could not restore a verified healthy state.
@@ -274,8 +346,10 @@ cleanup() {
trap - EXIT
set +e
if [ "$IMAGE_BROKEN" = 1 ]; then
- # This IS the manual operator action D1b forces: roll back to the last
- # good image, because Kubernetes will not do it on its own.
+ # Leftover restore fallback only. Pipeline/infra owns automatic
+ # `rollout undo --to-revision`; this e2e proves that operator action.
+ # Cleanup `set image` restores the original image if IMAGE_BROKEN is
+ # still set after that proof.
kctl set image deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
host-context-controller="$HCC_IMAGE" >/dev/null 2>&1 || restore_ok=0
fi
@@ -399,38 +473,8 @@ elif [ "$EXPECT_STUCK" = 1 ]; then
IMAGE_BROKEN=1
sample_ready_series "$STUCK_OBSERVE_SEC" stuck 0
read -r stuck_total stuck_200 stuck_503 stuck_maxstreak stuck_transitions <<<"$(series_metrics stuck)"
- deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"
- old_uid_live="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
- -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null | grep -cxF "$OLD_POD_UID" || true)"
- stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"
- replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"
+ assert_botched_outage stuck
write_evidence_artifact
-
- # ── Verdict (botched rollout): the sustained outage is the EXPECTED result ──
- [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||
- fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"
- [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&
- ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||
- fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE} — the outage (if any) has a different cause"
- [ "$old_uid_live" = 0 ] &&
- ok "Recreate terminated the old HEALTHY pod before any viable replacement existed — zero fallback replicas (the D1b mechanism)" ||
- fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout — the terminate-before-create premise did not hold"
- if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then
- ok "replacement pod is pinned by the injected image failure: ${replacement_row}"
- else
- fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none}) — the outage is not attributable to the injected bad rollout"
- fi
- [ "$stuck_503" -ge 1 ] &&
- ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||
- fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"
- [ "$stuck_transitions" -eq 0 ] &&
- ok "no 503->200 transition ever closed the outage — Kubernetes did NOT self-heal the botched Recreate rollout" ||
- fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) — the D1b manual-undo premise is refuted; investigate before trusting this gate"
- [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&
- ok "SUSTAINED outage measured: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) and still open at observation end — recovery requires the manual image rollback cleanup performs (D1b reproduced)" ||
- fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — the botched rollout did not produce the D1b total outage"
else
# ── FASE D (recovery, evenfire#391): botch, prove the outage, then undo ──
BOTCHED_IMAGE="$(botched_image_ref)"
@@ -441,35 +485,10 @@ else
IMAGE_BROKEN=1
sample_ready_series "$STUCK_OBSERVE_SEC" stuck 0
read -r stuck_total stuck_200 stuck_503 stuck_maxstreak stuck_transitions <<<"$(series_metrics stuck)"
- deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"
- old_uid_live="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null | grep -cxF "$OLD_POD_UID" || true)"
- stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"
- replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"
+ # D1b outage pins, including stuck_maxstreak -gt ROLLOUT_DOWNTIME_BUDGET_SEC.
+ assert_botched_outage recovery
BOTCHED_POD_UID="$(awk -F '\t' '{ print $2; exit }' <<<"$replacement_row")"
- [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||
- fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"
- [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&
- ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||
- fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE}"
- [ "$old_uid_live" = 0 ] &&
- ok "Recreate terminated the old HEALTHY pod before any viable replacement existed" ||
- fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout"
- if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then
- ok "replacement pod is pinned by the injected image failure: ${replacement_row}"
- else
- fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none})"
- fi
- [ "$stuck_503" -ge 1 ] &&
- ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||
- fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"
- [ "$stuck_transitions" -eq 0 ] &&
- ok "no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal" ||
- fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"
- [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&
- ok "SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip" ||
- fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"
-
live_revision_before_undo="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
[ "$live_revision_before_undo" != "$LAST_GOOD_REVISION" ] ||
fail "live revision ${live_revision_before_undo} still equals last-good ${LAST_GOOD_REVISION} — undo would be a no-op"
diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.sh
index 40b58ac3d..bfccdfe42 100755
--- a/scripts/minikube/t2-common.sh
+++ b/scripts/minikube/t2-common.sh
@@ -512,9 +512,14 @@ t2_process_check() {
local safe_profile
safe_profile="$(printf '%s' "$T2_PROFILE" | tr -c 'A-Za-z0-9_.-' '_')"
# Only real kubectl port-forward processes. A wrapper whose argv merely
- # mentions those words is not a port-forward.
+ # mentions those words is not a port-forward (rejected by comm=kubectl).
# Default IFS so UID/PID/PPID split. `IFS=` left pid empty and skipped every line.
- # awk keys argv0 off the ps -ef TIME column: macOS N:MM[.ss], Linux HH:MM:SS.
+ # awk is a loose pre-filter: kubectl as argv0/path token AND a later
+ # standalone port-forward token. Flags may sit between those tokens.
+ # Known limitation: a port-forward whose argv carries no profile/context
+ # token (e.g. pf-control-stack's context-less `kubectl -n NS port-forward`)
+ # cannot be attributed to a cluster from ps output alone and is skipped —
+ # flagging it would also flag unrelated clusters' forwards.
while read -r uid pid ppid rest; do
[ -n "$pid" ] || continue
command_line="$uid $pid $ppid $rest"
@@ -544,7 +549,7 @@ t2_process_check() {
T2_NEXT_COMMAND='stop the unrelated profile port-forward or select the owner worktree; do not share it'
t2_fail PORT_FORWARD_CONFLICT 'a port-forward for this profile is owned by another process'
fi
- done < <(ps -ef 2>/dev/null | awk '/[[:space:]][0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?[[:space:]]+([^[:space:]]*\/)?kubectl[[:space:]]+port-forward([[:space:]]|$)/ {print}' || true)
+ done < <(ps -ef 2>/dev/null | awk '/([^[:space:]]*\/)?kubectl([[:space:]]|$)/ && /[[:space:]]port-forward([[:space:]]|$)/ {print}' || true)
}
t2_classify_transition() {
diff --git a/scripts/tests/test-hcc-rollout-readiness-gate.sh b/scripts/tests/test-hcc-rollout-readiness-gate.sh
index 251f6fe5b..4478693ed 100755
--- a/scripts/tests/test-hcc-rollout-readiness-gate.sh
+++ b/scripts/tests/test-hcc-rollout-readiness-gate.sh
@@ -69,7 +69,7 @@ grep -Eq 'baseline_503.*-eq.*0' "$GATE" &&
grep -Fq 'EXPECT_STUCK' "$GATE" &&
grep -Fq 'STUCK_OBSERVE_SEC' "$GATE" &&
grep -Eq 'stuck_transitions.*-eq.*0' "$GATE" &&
- grep -Eq 'stuck_maxstreak.*-gt.*ROLLOUT_DOWNTIME_BUDGET_SEC' "$GATE" &&
+ grep -Fq '"$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC"' "$GATE" &&
grep -Fq 'ErrImagePull|ImagePullBackOff' "$GATE" &&
pass "D1b mode: bounded, pinned to ImagePull, sustained outage with zero recoveries" ||
fail "the botched-rollout mode lost its outage evidence or can hang unbounded"
@@ -127,6 +127,11 @@ grep -Fq 'create_synthetic_fleet' "$GATE" &&
# is the same sustained D1b pin as EXPECT_STUCK, IMAGE_BROKEN is cleared
# only AFTER every post-undo proof, and a no-op undo fails.
recovery_block="$(awk '/FASE D \(recovery, evenfire#391\)/,0' "$GATE")"
+# The sustained-outage comparison lives in the shared assert_botched_outage
+# helper; pin the LITERAL test expression in the helper body (a comment
+# mentioning the variables must never satisfy this check) plus the recovery
+# block's invocation of that helper.
+outage_helper="$(awk '/^assert_botched_outage\(\) \{/,/^\}/' "$GATE")"
image_clear_line="$(grep -n 'IMAGE_BROKEN=0' <<<"$recovery_block" | head -1 | cut -d: -f1)"
hold_assert_line="$(grep -n 'post-undo hold' <<<"$recovery_block" | head -1 | cut -d: -f1)"
grep -Fq 'EXPECT_RECOVERY' "$GATE" &&
@@ -134,7 +139,8 @@ grep -Fq 'EXPECT_RECOVERY' "$GATE" &&
grep -Fq 'TEST ACTION' "$GATE" &&
grep -Fq -- '--to-revision' "$GATE" &&
grep -Fq 'LAST_GOOD_REVISION' "$GATE" &&
- grep -Eq 'stuck_maxstreak.*-gt.*ROLLOUT_DOWNTIME_BUDGET_SEC' <<<"$recovery_block" &&
+ grep -Fq 'assert_botched_outage recovery' <<<"$recovery_block" &&
+ grep -Fq '"$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC"' <<<"$outage_helper" &&
grep -Fq '[ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0' <<<"$recovery_block" &&
[ -n "$image_clear_line" ] && [ -n "$hold_assert_line" ] &&
[ "$image_clear_line" -gt "$hold_assert_line" ] &&
diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.sh
index 98556e4a9..12fa9d7e7 100755
--- a/scripts/tests/test-minikube-t2-contract.sh
+++ b/scripts/tests/test-minikube-t2-contract.sh
@@ -32,10 +32,16 @@ if grep -Eq 'while IFS= read -r uid pid ppid rest' "$COMMON"; then
exit 1
fi
grep -Fq 'while read -r uid pid ppid rest' "$COMMON"
-if ! grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON"; then
- echo 'FAIL: t2_process_check awk TIME token must accept Linux HH:MM:SS' >&2
+if grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON"; then
+ echo 'FAIL: t2_process_check awk still anchors on a TIME-column regex' >&2
exit 1
fi
+if grep -Fq 'kubectl[[:space:]]+port-forward' "$COMMON"; then
+ echo 'FAIL: t2_process_check awk still requires kubectl/port-forward adjacency' >&2
+ exit 1
+fi
+grep -Fq '([^[:space:]]*\/)?kubectl([[:space:]]|$)' "$COMMON"
+grep -Fq '[[:space:]]port-forward([[:space:]]|$)' "$COMMON"
grep -Fq 'CONTROL_API_REAL_PG_CONTEXT' "$T1"
grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "$T1"
grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$T1"
diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.sh
index 134efc692..5ab4c42dc 100755
--- a/scripts/tests/test-minikube-t2-scenarios.sh
+++ b/scripts/tests/test-minikube-t2-scenarios.sh
@@ -256,25 +256,91 @@ grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"
grep -Fq 'PORT_FORWARD_CONFLICT' "$COMMON"
grep -Fq 'while read -r uid pid ppid rest' "$COMMON" || fail 't2_process_check must split ps -ef fields'
grep -Fq 'IFS= read -r uid pid ppid rest' "$COMMON" && fail 'IFS= read disables PID split and silences PORT_FORWARD_CONFLICT'
-grep -Fq '([^[:space:]]*\/)?kubectl[[:space:]]+port-forward' "$COMMON" || fail 't2_process_check awk must match argv0 kubectl, not a wrapper mention'
-grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON" || fail 't2_process_check awk TIME token must accept Linux HH:MM:SS as well as macOS N:MM[.ss]'
-bare_pf='user 12345 1 0 10:00 ttys000 0:00 kubectl port-forward --context=clerum-feat-scenario svc/x 8080:80'
-path_pf='user 124 1 0 10:00 ttys000 0:00 /usr/local/bin/kubectl port-forward --context=clerum-feat-scenario svc/y 9090:80'
-macos_frac_pf='user 12346 1 0 10:00 ttys000 0:00.03 kubectl port-forward --context=clerum-feat-scenario svc/x 8080:80'
-linux_pf='user 126 1 0 10:00 pts/0 00:00:00 kubectl port-forward --context=clerum-feat-scenario svc/z 7070:80'
-linux_path_pf='user 127 1 0 10:00 pts/0 00:00:00 /usr/bin/kubectl port-forward --context=clerum-feat-scenario svc/z 7070:80'
-wrapper_pf='user 125 1 0 10:00 ttys000 0:00 bash -c echo kubectl port-forward'
-linux_wrapper_pf='user 128 1 0 10:00 pts/0 00:00:00 bash -c echo kubectl port-forward'
-awk_pf='/[[:space:]][0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?[[:space:]]+([^[:space:]]*\/)?kubectl[[:space:]]+port-forward([[:space:]]|$)/ {print}'
-printf '%s\n' "$bare_pf" "$path_pf" "$macos_frac_pf" "$linux_pf" "$linux_path_pf" "$wrapper_pf" "$linux_wrapper_pf" | awk "$awk_pf" >"$tmp/pf-awk.out"
-grep -Fq 'kubectl port-forward --context=clerum-feat-scenario svc/x' "$tmp/pf-awk.out" || fail 'awk missed a bare kubectl port-forward'
-grep -Fq '/usr/local/bin/kubectl port-forward' "$tmp/pf-awk.out" || fail 'awk missed a path kubectl port-forward'
-grep -Fq '0:00.03 kubectl port-forward' "$tmp/pf-awk.out" || fail 'awk missed a macOS fractional TIME kubectl port-forward'
-grep -Fq '00:00:00 kubectl port-forward --context=clerum-feat-scenario svc/z' "$tmp/pf-awk.out" || fail 'awk missed a Linux HH:MM:SS kubectl port-forward'
-grep -Fq '/usr/bin/kubectl port-forward' "$tmp/pf-awk.out" || fail 'awk missed a Linux path kubectl port-forward'
-grep -Fq 'bash -c echo kubectl port-forward' "$tmp/pf-awk.out" && fail 'awk matched a wrapper that only mentions kubectl'
-read -r uid pid ppid rest <<<"$bare_pf"
-[ "$pid" = 12345 ] || fail "ps field split left pid='$pid' instead of 12345"
+grep -Fq '([^[:space:]]*\/)?kubectl([[:space:]]|$)' "$COMMON" || fail 't2_process_check awk must match a kubectl argv0/path token'
+grep -Fq '[[:space:]]port-forward([[:space:]]|$)' "$COMMON" || fail 't2_process_check awk must match a standalone port-forward token'
+grep -Fq 'kubectl[[:space:]]+port-forward' "$COMMON" && fail 't2_process_check awk still requires kubectl/port-forward adjacency'
+grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON" && fail 't2_process_check awk still anchors on a TIME-column regex'
+
+# Real launcher argv (flags between kubectl and port-forward). TIME prefixes
+# are only ps -ef wrappers. The dead adjacency regex must miss these lines.
+pf_profile="clerum-t2-pf-fixture"
+pf_all='user 4242 1 0 10:00 ttys000 0:00 kubectl --context='"${pf_profile}"' -n control-plane port-forward --address=127.0.0.1 svc/control-ui 3000:3000'
+pf_ctl='user 4243 1 0 10:00 ttys000 0:00.03 kubectl -n control-plane port-forward svc/control-api 8090:8090'
+pf_e2e='user 4244 1 0 10:00 pts/0 00:00:00 kubectl --context '"${pf_profile}"' port-forward svc/control-ui -n control-plane 3000:3000'
+bare_pf='user 4245 1 0 10:00 ttys000 0:00 kubectl port-forward --context='"${pf_profile}"' svc/x 8080:80'
+path_pf='user 4246 1 0 10:00 pts/0 00:00:00 /usr/local/bin/kubectl --context='"${pf_profile}"' -n control-plane port-forward --address=127.0.0.1 svc/control-api 8090:8090'
+wrapper_pf='user 4247 1 0 10:00 ttys000 0:00 bash -c echo kubectl port-forward --context='"${pf_profile}"
+awk_pf="$(sed -n '/^t2_process_check()/,/^}/p' "$COMMON" | sed -n "s/.*awk '\\(.*\\)' .*/\\1/p")"
+[ -n "$awk_pf" ] || fail 'could not extract t2_process_check awk program'
+printf '%s\n' "$awk_pf" | grep -Fq 'kubectl[[:space:]]+port-forward' && fail 'extracted awk still requires kubectl/port-forward adjacency'
+dead_awk='/[[:space:]][0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?[[:space:]]+([^[:space:]]*\/)?kubectl[[:space:]]+port-forward([[:space:]]|$)/ {print}'
+printf '%s\n' "$pf_all" "$pf_ctl" "$pf_e2e" | awk "$dead_awk" >"$tmp/pf-dead-awk.out"
+[ ! -s "$tmp/pf-dead-awk.out" ] || fail 'real-launcher fixtures still match the dead adjacency regex — fixtures are wrong'
+printf '%s\n' "$pf_all" "$pf_ctl" "$pf_e2e" "$bare_pf" "$path_pf" "$wrapper_pf" | awk "$awk_pf" >"$tmp/pf-awk.out"
+grep -Fq -- "--context=${pf_profile} -n control-plane port-forward --address=127.0.0.1" "$tmp/pf-awk.out" || fail 'awk missed pf-all-stack real launcher'
+grep -Fq 'kubectl -n control-plane port-forward svc/control-api' "$tmp/pf-awk.out" || fail 'awk missed pf-control-stack real launcher'
+grep -Fq -- "--context ${pf_profile} port-forward svc/control-ui" "$tmp/pf-awk.out" || fail 'awk missed run-e2e real launcher'
+grep -Fq 'kubectl port-forward --context=' "$tmp/pf-awk.out" || fail 'awk missed a bare kubectl port-forward'
+grep -Fq '/usr/local/bin/kubectl --context=' "$tmp/pf-awk.out" || fail 'awk missed a path kubectl with flags before port-forward'
+read -r uid pid ppid rest <<<"$pf_all"
+[ "$pid" = 4242 ] || fail "ps field split left pid='$pid' instead of 4242"
+
+pf_root="$tmp/pf-check-profiles"
+mkdir -p "$pf_root/$pf_profile/pids"
+pf_bin="$tmp/pf-ps-bin"
+mkdir -p "$pf_bin"
+cat >"$pf_bin/ps" <<'EOF'
+#!/usr/bin/env bash
+if [ "${1:-}" = "-ef" ]; then
+ cat "${T2_PS_EF_FILE:?}"
+ exit 0
+fi
+pid=""
+while [ "$#" -gt 0 ]; do
+ case "$1" in
+ -p)
+ pid="$2"
+ shift 2
+ ;;
+ -p*)
+ pid="${1#-p}"
+ shift
+ ;;
+ *)
+ shift
+ ;;
+ esac
+done
+case "$pid" in
+ 4242|4243|4244|4245|4246) printf '%s\n' kubectl ;;
+ 4247) printf '%s\n' bash ;;
+ *) printf '\n' ;;
+esac
+EOF
+chmod +x "$pf_bin/ps"
+
+run_process_check() {
+ local ef_file="$1" out="$2"
+ T2_PS_EF_FILE="$ef_file" PATH="$pf_bin:$PATH" \
+ T2_PROFILE="$pf_profile" T2_CONTEXT="$pf_profile" \
+ MINIKUBE_PROFILE="$pf_profile" \
+ T2_PROFILE_ROOT="$pf_root" \
+ T2_PROJECT_DIR="$ROOT" \
+ T2_LOCK_ROOT="$tmp/locks" \
+ bash -c 'source "$1"; t2_process_check' bash "$COMMON" >"$out" 2>&1
+}
+
+printf '%s\n' "$pf_all" >"$tmp/ps-ef-real.out"
+if run_process_check "$tmp/ps-ef-real.out" "$tmp/pf-real.err"; then
+ fail 't2_process_check did not fail a foreign real launcher (PORT_FORWARD_CONFLICT)'
+fi
+grep -Fq 'PORT_FORWARD_CONFLICT' "$tmp/pf-real.err" || fail 'foreign real launcher did not report PORT_FORWARD_CONFLICT'
+
+printf '%s\n' "$wrapper_pf" >"$tmp/ps-ef-wrap.out"
+if ! run_process_check "$tmp/ps-ef-wrap.out" "$tmp/pf-wrap.err"; then
+ fail 't2_process_check failed on a wrapper that only mentions kubectl (comm=bash)'
+fi
+
grep -Fq 'REUSE_DB=true' "$ROOT/scripts/minikube/t2.sh"
grep -Fq 'CONTROL_DB_RESET_PVC_UID' "$ROOT/scripts/minikube/t2.sh"