Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: detect real kubectl port-forward argv in T2
Alfredo Lopez873e371f24f2
diff --git a/.cursor/rules/minikube-t2-harness.mdc b/.cursor/rules/minikube-t2-harness.mdcindex 3a640e674..5634fbfea 100644--- a/.cursor/rules/minikube-t2-harness.mdc+++ b/.cursor/rules/minikube-t2-harness.mdc@@ -34,8 +34,8 @@ Invariants that must never regress:`pre-gate-sync` with `--skip-port-forwards`.- `t2_process_check` accepts only real `kubectl` (`comm=kubectl`) port-forwardPIDs recorded in the profile cache or legacy `/tmp/pf-<profile>-*.pid`.- The awk argv0 filter anchors on the `ps -ef` TIME token in both macOS- (`N:MM[.ss]`) and Linux (`HH:MM:SS`) forms.+ awk is a loose pre-filter: `kubectl` as argv0/path token and a later+ standalone `port-forward` token (flags may sit between them).- Stable failure codes (`BOOTSTRAP_REQUIRED`, `HEAD_MARKER_MISMATCH`,`PORT_FORWARD_CONFLICT`, `PROFILE_BUSY`, ...) keep their meanings.diff --git a/.cursor/skills/minikube-t0-t1-t2/reference.md b/.cursor/skills/minikube-t0-t1-t2/reference.mdindex 646827216..b6154f3ec 100644--- a/.cursor/skills/minikube-t0-t1-t2/reference.md+++ b/.cursor/skills/minikube-t0-t1-t2/reference.md@@ -26,9 +26,9 @@ Companion to `SKILL.md`. Source of truth: `scripts/minikube/t2.sh`,argv** when avoidable. `t2_process_check` accepts only real `kubectl`processes (`comm=kubectl`) whose PIDs are recorded in the profile cache orlegacy `/tmp/pf-<profile>-*.pid`; a lookalike wrapper trips- `PORT_FORWARD_CONFLICT`. The argv0 pre-filter keys off the `ps -ef` TIME- field immediately before `kubectl` and must match both macOS `N:MM[.ss]`- and Linux `HH:MM:SS`.+ `PORT_FORWARD_CONFLICT`. The awk pre-filter matches `kubectl` as an argv0+ or path token plus a later standalone `port-forward` token; wrappers that+ only mention those words are rejected by `comm`.- **Do not commit** `.local-notes/`, lockfiles produced by an incidental`npm ci`, generated ports/profile metadata, or anything secret-like. Run`make minikube-t2-public-boundary` first.diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.mdindex 0a241f40e..ae45d4f6b 100644--- a/docs/testing/minikube-t2-runbook.md+++ b/docs/testing/minikube-t2-runbook.md@@ -107,8 +107,10 @@ Never remove a lock with a live owner, and never remove the whole lock root.The runner checks required namespaces, Services, Secret names, ConfigMap names,the `control-postgres` PVC/deployment, all required deployment readiness, theimage manifest/source, and real `kubectl port-forward` processes for this-profile. The conflict check keys argv0 off the `ps -ef` TIME field and accepts-both macOS `N:MM[.ss]` and Linux `HH:MM:SS`. Allowed port-forward PIDs live in+profile. The conflict check is a loose argv pre-filter (`kubectl` as argv0 or+path token plus a later standalone `port-forward` token; flags may sit+between them) plus a `comm=kubectl` hardening and a PID allowlist.+Allowed port-forward PIDs live in`$HOME/.cache/clerum/minikube-profiles/<profile>/pids/*.pid` (and thelegacy `/tmp/pf-<profile>-*.pid` files written by `pf-all-stack.sh`).`make minikube-t2` invokes `pre-gate-sync` with `--skip-port-forwards` so thediff --git a/scripts/e2e/e2e-hcc-rollout-readiness.sh b/scripts/e2e/e2e-hcc-rollout-readiness.shindex 9261f7082..a3f3a66a8 100755--- a/scripts/e2e/e2e-hcc-rollout-readiness.sh+++ b/scripts/e2e/e2e-hcc-rollout-readiness.sh@@ -13,9 +13,10 @@# measures it against the wall clock, over a synthetic clerum-dev-scale fleet# (the initial LISTs and the revocation pass are part of the window), and pins# the D1b failure mode empirically: `progressDeadlineSeconds: 1200` is-# DETECTION only — a rollout whose new pod can never certify leaves the HCC-# control plane down until a manual `kubectl rollout undo`, because Recreate-# keeps zero old replicas to fall back on.+# DETECTION only — a botched Recreate rollout leaves the HCC down until+# evenfire-infra deploy workflows run automatic `rollout undo --to-revision`.+# This e2e proves that operator action; cleanup `set image` is only the+# leftover restore fallback.## Modes:# EXPECT_RECOVERY=1 — exclusive with EXPECT_STUCK. Botch the image, prove the@@ -44,9 +45,10 @@# total outage, empirically. The sustained 503 streak — longer than the# healthy budget and still open at observation end — is the EXPECTED# evidence. The observation is bounded (STUCK_OBSERVE_SEC) and cleanup-# performs the manual-operator recovery (restore the original image, wait+# performs the fallback image restore (restore the original image, wait# for re-certification), so the gate always terminates loudly instead of-# hanging alongside the outage it reproduces.+# hanging alongside the outage it reproduces. Production recovery is the+# infra-owned automatic `rollout undo --to-revision` (evenfire#391).set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"# shellcheck source=scripts/e2e/e2e-lib.sh@@ -221,6 +223,76 @@ botched_image_ref() {printf '%s:e2e-botched-%s' "$base" "$RUN_ID"}+# Shared D1b outage pins for EXPECT_STUCK and EXPECT_RECOVERY. flavor=stuck+# keeps the longer D1b strings; flavor=recovery keeps the pre-undo variants.+# A failed pod-UID listing must not count as "old pod terminated".+assert_botched_outage() {+ local flavor="$1" old_uid_query+ case "$flavor" in+ stuck | recovery) ;;+ *) die "assert_botched_outage: flavor must be stuck or recovery, got '${flavor}'" ;;+ esac++ deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"+ old_uid_live=-1+ if old_uid_query="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \+ -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}')"; then+ old_uid_live="$(printf '%s\n' "$old_uid_query" | grep -cxF "$OLD_POD_UID" || true)"+ else+ fail "could not list HCC pod UIDs — refusing to treat a query failure as old-pod termination"+ fi+ stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \+ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"+ replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"++ [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||+ fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"+ if [ "$flavor" = recovery ]; then+ [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&+ ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||+ fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE}"+ [ "$old_uid_live" = 0 ] &&+ ok "Recreate terminated the old HEALTHY pod before any viable replacement existed" ||+ fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout"+ if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then+ ok "replacement pod is pinned by the injected image failure: ${replacement_row}"+ else+ fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none})"+ fi+ [ "$stuck_503" -ge 1 ] &&+ ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||+ fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"+ [ "$stuck_transitions" -eq 0 ] &&+ ok "no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal" ||+ fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"+ [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&+ ok "SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip" ||+ fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"+ else+ [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&+ ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||+ fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE} — the outage (if any) has a different cause"+ [ "$old_uid_live" = 0 ] &&+ ok "Recreate terminated the old HEALTHY pod before any viable replacement existed — zero fallback replicas (the D1b mechanism)" ||+ fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout — the terminate-before-create premise did not hold"+ if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then+ ok "replacement pod is pinned by the injected image failure: ${replacement_row}"+ else+ fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none}) — the outage is not attributable to the injected bad rollout"+ fi+ [ "$stuck_503" -ge 1 ] &&+ ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||+ fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"+ [ "$stuck_transitions" -eq 0 ] &&+ ok "no 503->200 transition ever closed the outage — Kubernetes did NOT self-heal the botched Recreate rollout" ||+ fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) — the D1b premise that recovery requires an explicit rollback is refuted; investigate before trusting this gate"+ [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&+ ok "SUSTAINED outage measured: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) and still open at observation end — recovery requires an explicit rollback (infra-owned rollout undo --to-revision in production; cleanup's set image restore here) (D1b reproduced)" ||+ fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — the botched rollout did not produce the D1b total outage"+ fi+}+print_repair_instructions() {cat >&2 <<EOFHCC rollout gate cleanup could not restore a verified healthy state.@@ -274,8 +346,10 @@ cleanup() {trap - EXITset +eif [ "$IMAGE_BROKEN" = 1 ]; then- # This IS the manual operator action D1b forces: roll back to the last- # good image, because Kubernetes will not do it on its own.+ # Leftover restore fallback only. Pipeline/infra owns automatic+ # `rollout undo --to-revision`; this e2e proves that operator action.+ # Cleanup `set image` restores the original image if IMAGE_BROKEN is+ # still set after that proof.kctl set image deployment/"$HCC_DEPLOY" -n "$HCC_NS" \host-context-controller="$HCC_IMAGE" >/dev/null 2>&1 || restore_ok=0fi@@ -399,38 +473,8 @@ elif [ "$EXPECT_STUCK" = 1 ]; thenIMAGE_BROKEN=1sample_ready_series "$STUCK_OBSERVE_SEC" stuck 0read -r stuck_total stuck_200 stuck_503 stuck_maxstreak stuck_transitions <<<"$(series_metrics stuck)"- deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \- -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"- old_uid_live="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \- -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null | grep -cxF "$OLD_POD_UID" || true)"- stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \- -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"- replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"+ assert_botched_outage stuckwrite_evidence_artifact-- # ── Verdict (botched rollout): the sustained outage is the EXPECTED result ──- [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||- fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"- [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&- ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||- fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE} — the outage (if any) has a different cause"- [ "$old_uid_live" = 0 ] &&- ok "Recreate terminated the old HEALTHY pod before any viable replacement existed — zero fallback replicas (the D1b mechanism)" ||- fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout — the terminate-before-create premise did not hold"- if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then- ok "replacement pod is pinned by the injected image failure: ${replacement_row}"- else- fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none}) — the outage is not attributable to the injected bad rollout"- fi- [ "$stuck_503" -ge 1 ] &&- ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||- fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"- [ "$stuck_transitions" -eq 0 ] &&- ok "no 503->200 transition ever closed the outage — Kubernetes did NOT self-heal the botched Recreate rollout" ||- fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) — the D1b manual-undo premise is refuted; investigate before trusting this gate"- [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&- ok "SUSTAINED outage measured: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) and still open at observation end — recovery requires the manual image rollback cleanup performs (D1b reproduced)" ||- fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — the botched rollout did not produce the D1b total outage"else# ── FASE D (recovery, evenfire#391): botch, prove the outage, then undo ──BOTCHED_IMAGE="$(botched_image_ref)"@@ -441,35 +485,10 @@ elseIMAGE_BROKEN=1sample_ready_series "$STUCK_OBSERVE_SEC" stuck 0read -r stuck_total stuck_200 stuck_503 stuck_maxstreak stuck_transitions <<<"$(series_metrics stuck)"- deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"- old_uid_live="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null | grep -cxF "$OLD_POD_UID" || true)"- stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[*].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"- replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" '$2 != old && $2 != "" { print; exit }' <<<"$stuck_pod_rows")"+ # D1b outage pins, including stuck_maxstreak -gt ROLLOUT_DOWNTIME_BUDGET_SEC.+ assert_botched_outage recoveryBOTCHED_POD_UID="$(awk -F '\t' '{ print $2; exit }' <<<"$replacement_row")"- [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||- fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"- [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&- ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||- fail "deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE}"- [ "$old_uid_live" = 0 ] &&- ok "Recreate terminated the old HEALTHY pod before any viable replacement existed" ||- fail "old pod ${OLD_POD_NAME} is still alive under a Recreate rollout"- if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then- ok "replacement pod is pinned by the injected image failure: ${replacement_row}"- else- fail "no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none})"- fi- [ "$stuck_503" -ge 1 ] &&- ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||- fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"- [ "$stuck_transitions" -eq 0 ] &&- ok "no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal" ||- fail "the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"- [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&- ok "SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip" ||- fail "outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"-live_revision_before_undo="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"[ "$live_revision_before_undo" != "$LAST_GOOD_REVISION" ] ||fail "live revision ${live_revision_before_undo} still equals last-good ${LAST_GOOD_REVISION} — undo would be a no-op"diff --git a/scripts/minikube/t2-common.sh b/scripts/minikube/t2-common.shindex ec8460bd4..5dbdac6ae 100755--- a/scripts/minikube/t2-common.sh+++ b/scripts/minikube/t2-common.sh@@ -507,9 +507,14 @@ t2_process_check() {local safe_profilesafe_profile="$(printf '%s' "$T2_PROFILE" | tr -c 'A-Za-z0-9_.-' '_')"# Only real kubectl port-forward processes. A wrapper whose argv merely- # mentions those words is not a port-forward.+ # mentions those words is not a port-forward (rejected by comm=kubectl).# Default IFS so UID/PID/PPID split. `IFS=` left pid empty and skipped every line.- # awk keys argv0 off the ps -ef TIME column: macOS N:MM[.ss], Linux HH:MM:SS.+ # awk is a loose pre-filter: kubectl as argv0/path token AND a later+ # standalone port-forward token. Flags may sit between those tokens.+ # Known limitation: a port-forward whose argv carries no profile/context+ # token (e.g. pf-control-stack's context-less `kubectl -n NS port-forward`)+ # cannot be attributed to a cluster from ps output alone and is skipped —+ # flagging it would also flag unrelated clusters' forwards.while read -r uid pid ppid rest; do[ -n "$pid" ] || continuecommand_line="$uid $pid $ppid $rest"@@ -531,7 +536,7 @@ t2_process_check() {T2_NEXT_COMMAND='stop the unrelated profile port-forward or select the owner worktree; do not share it't2_fail PORT_FORWARD_CONFLICT 'a port-forward for this profile is owned by another process'fi- done < <(ps -ef 2>/dev/null | awk '/[[:space:]][0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?[[:space:]]+([^[:space:]]*\/)?kubectl[[:space:]]+port-forward([[:space:]]|$)/ {print}' || true)+ done < <(ps -ef 2>/dev/null | awk '/([^[:space:]]*\/)?kubectl([[:space:]]|$)/ && /[[:space:]]port-forward([[:space:]]|$)/ {print}' || true)}t2_classify_transition() {diff --git a/scripts/tests/test-hcc-rollout-readiness-gate.sh b/scripts/tests/test-hcc-rollout-readiness-gate.shindex 251f6fe5b..4478693ed 100755--- a/scripts/tests/test-hcc-rollout-readiness-gate.sh+++ b/scripts/tests/test-hcc-rollout-readiness-gate.sh@@ -69,7 +69,7 @@ grep -Eq 'baseline_503.*-eq.*0' "$GATE" &&grep -Fq 'EXPECT_STUCK' "$GATE" &&grep -Fq 'STUCK_OBSERVE_SEC' "$GATE" &&grep -Eq 'stuck_transitions.*-eq.*0' "$GATE" &&- grep -Eq 'stuck_maxstreak.*-gt.*ROLLOUT_DOWNTIME_BUDGET_SEC' "$GATE" &&+ grep -Fq '"$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC"' "$GATE" &&grep -Fq 'ErrImagePull|ImagePullBackOff' "$GATE" &&pass "D1b mode: bounded, pinned to ImagePull, sustained outage with zero recoveries" ||fail "the botched-rollout mode lost its outage evidence or can hang unbounded"@@ -127,6 +127,11 @@ grep -Fq 'create_synthetic_fleet' "$GATE" &&# is the same sustained D1b pin as EXPECT_STUCK, IMAGE_BROKEN is cleared# only AFTER every post-undo proof, and a no-op undo fails.recovery_block="$(awk '/FASE D \(recovery, evenfire#391\)/,0' "$GATE")"+# The sustained-outage comparison lives in the shared assert_botched_outage+# helper; pin the LITERAL test expression in the helper body (a comment+# mentioning the variables must never satisfy this check) plus the recovery+# block's invocation of that helper.+outage_helper="$(awk '/^assert_botched_outage\(\) \{/,/^\}/' "$GATE")"image_clear_line="$(grep -n 'IMAGE_BROKEN=0' <<<"$recovery_block" | head -1 | cut -d: -f1)"hold_assert_line="$(grep -n 'post-undo hold' <<<"$recovery_block" | head -1 | cut -d: -f1)"grep -Fq 'EXPECT_RECOVERY' "$GATE" &&@@ -134,7 +139,8 @@ grep -Fq 'EXPECT_RECOVERY' "$GATE" &&grep -Fq 'TEST ACTION' "$GATE" &&grep -Fq -- '--to-revision' "$GATE" &&grep -Fq 'LAST_GOOD_REVISION' "$GATE" &&- grep -Eq 'stuck_maxstreak.*-gt.*ROLLOUT_DOWNTIME_BUDGET_SEC' <<<"$recovery_block" &&+ grep -Fq 'assert_botched_outage recovery' <<<"$recovery_block" &&+ grep -Fq '"$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC"' <<<"$outage_helper" &&grep -Fq '[ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0' <<<"$recovery_block" &&[ -n "$image_clear_line" ] && [ -n "$hold_assert_line" ] &&[ "$image_clear_line" -gt "$hold_assert_line" ] &&diff --git a/scripts/tests/test-minikube-t2-contract.sh b/scripts/tests/test-minikube-t2-contract.shindex 6d022af71..51f4fdbf5 100755--- a/scripts/tests/test-minikube-t2-contract.sh+++ b/scripts/tests/test-minikube-t2-contract.sh@@ -31,10 +31,16 @@ if grep -Eq 'while IFS= read -r uid pid ppid rest' "$COMMON"; thenexit 1figrep -Fq 'while read -r uid pid ppid rest' "$COMMON"-if ! grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON"; then- echo 'FAIL: t2_process_check awk TIME token must accept Linux HH:MM:SS' >&2+if grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON"; then+ echo 'FAIL: t2_process_check awk still anchors on a TIME-column regex' >&2exit 1fi+if grep -Fq 'kubectl[[:space:]]+port-forward' "$COMMON"; then+ echo 'FAIL: t2_process_check awk still requires kubectl/port-forward adjacency' >&2+ exit 1+fi+grep -Fq '([^[:space:]]*\/)?kubectl([[:space:]]|$)' "$COMMON"+grep -Fq '[[:space:]]port-forward([[:space:]]|$)' "$COMMON"grep -Fq 'CONTROL_API_REAL_PG_CONTEXT' "$T1"grep -Fq 'CONTROL_API_REAL_PG_ADMIN_URL=' "$T1"grep -Fq 'CONTROL_API_REAL_PG_REQUIRED=1' "$T1"diff --git a/scripts/tests/test-minikube-t2-scenarios.sh b/scripts/tests/test-minikube-t2-scenarios.shindex fe4f80a53..1615582a1 100755--- a/scripts/tests/test-minikube-t2-scenarios.sh+++ b/scripts/tests/test-minikube-t2-scenarios.sh@@ -250,25 +250,91 @@ grep -Fq 'trap t2_lock_release EXIT INT TERM' "$COMMON"grep -Fq 'PORT_FORWARD_CONFLICT' "$COMMON"grep -Fq 'while read -r uid pid ppid rest' "$COMMON" || fail 't2_process_check must split ps -ef fields'grep -Fq 'IFS= read -r uid pid ppid rest' "$COMMON" && fail 'IFS= read disables PID split and silences PORT_FORWARD_CONFLICT'-grep -Fq '([^[:space:]]*\/)?kubectl[[:space:]]+port-forward' "$COMMON" || fail 't2_process_check awk must match argv0 kubectl, not a wrapper mention'-grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON" || fail 't2_process_check awk TIME token must accept Linux HH:MM:SS as well as macOS N:MM[.ss]'-bare_pf='user 12345 1 0 10:00 ttys000 0:00 kubectl port-forward --context=clerum-feat-scenario svc/x 8080:80'-path_pf='user 124 1 0 10:00 ttys000 0:00 /usr/local/bin/kubectl port-forward --context=clerum-feat-scenario svc/y 9090:80'-macos_frac_pf='user 12346 1 0 10:00 ttys000 0:00.03 kubectl port-forward --context=clerum-feat-scenario svc/x 8080:80'-linux_pf='user 126 1 0 10:00 pts/0 00:00:00 kubectl port-forward --context=clerum-feat-scenario svc/z 7070:80'-linux_path_pf='user 127 1 0 10:00 pts/0 00:00:00 /usr/bin/kubectl port-forward --context=clerum-feat-scenario svc/z 7070:80'-wrapper_pf='user 125 1 0 10:00 ttys000 0:00 bash -c echo kubectl port-forward'-linux_wrapper_pf='user 128 1 0 10:00 pts/0 00:00:00 bash -c echo kubectl port-forward'-awk_pf='/[[:space:]][0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?[[:space:]]+([^[:space:]]*\/)?kubectl[[:space:]]+port-forward([[:space:]]|$)/ {print}'-printf '%s\n' "$bare_pf" "$path_pf" "$macos_frac_pf" "$linux_pf" "$linux_path_pf" "$wrapper_pf" "$linux_wrapper_pf" | awk "$awk_pf" >"$tmp/pf-awk.out"-grep -Fq 'kubectl port-forward --context=clerum-feat-scenario svc/x' "$tmp/pf-awk.out" || fail 'awk missed a bare kubectl port-forward'-grep -Fq '/usr/local/bin/kubectl port-forward' "$tmp/pf-awk.out" || fail 'awk missed a path kubectl port-forward'-grep -Fq '0:00.03 kubectl port-forward' "$tmp/pf-awk.out" || fail 'awk missed a macOS fractional TIME kubectl port-forward'-grep -Fq '00:00:00 kubectl port-forward --context=clerum-feat-scenario svc/z' "$tmp/pf-awk.out" || fail 'awk missed a Linux HH:MM:SS kubectl port-forward'-grep -Fq '/usr/bin/kubectl port-forward' "$tmp/pf-awk.out" || fail 'awk missed a Linux path kubectl port-forward'-grep -Fq 'bash -c echo kubectl port-forward' "$tmp/pf-awk.out" && fail 'awk matched a wrapper that only mentions kubectl'-read -r uid pid ppid rest <<<"$bare_pf"-[ "$pid" = 12345 ] || fail "ps field split left pid='$pid' instead of 12345"+grep -Fq '([^[:space:]]*\/)?kubectl([[:space:]]|$)' "$COMMON" || fail 't2_process_check awk must match a kubectl argv0/path token'+grep -Fq '[[:space:]]port-forward([[:space:]]|$)' "$COMMON" || fail 't2_process_check awk must match a standalone port-forward token'+grep -Fq 'kubectl[[:space:]]+port-forward' "$COMMON" && fail 't2_process_check awk still requires kubectl/port-forward adjacency'+grep -Fq '[0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?' "$COMMON" && fail 't2_process_check awk still anchors on a TIME-column regex'++# Real launcher argv (flags between kubectl and port-forward). TIME prefixes+# are only ps -ef wrappers. The dead adjacency regex must miss these lines.+pf_profile="clerum-t2-pf-fixture"+pf_all='user 4242 1 0 10:00 ttys000 0:00 kubectl --context='"${pf_profile}"' -n control-plane port-forward --address=127.0.0.1 svc/control-ui 3000:3000'+pf_ctl='user 4243 1 0 10:00 ttys000 0:00.03 kubectl -n control-plane port-forward svc/control-api 8090:8090'+pf_e2e='user 4244 1 0 10:00 pts/0 00:00:00 kubectl --context '"${pf_profile}"' port-forward svc/control-ui -n control-plane 3000:3000'+bare_pf='user 4245 1 0 10:00 ttys000 0:00 kubectl port-forward --context='"${pf_profile}"' svc/x 8080:80'+path_pf='user 4246 1 0 10:00 pts/0 00:00:00 /usr/local/bin/kubectl --context='"${pf_profile}"' -n control-plane port-forward --address=127.0.0.1 svc/control-api 8090:8090'+wrapper_pf='user 4247 1 0 10:00 ttys000 0:00 bash -c echo kubectl port-forward --context='"${pf_profile}"+awk_pf="$(sed -n '/^t2_process_check()/,/^}/p' "$COMMON" | sed -n "s/.*awk '\\(.*\\)' .*/\\1/p")"+[ -n "$awk_pf" ] || fail 'could not extract t2_process_check awk program'+printf '%s\n' "$awk_pf" | grep -Fq 'kubectl[[:space:]]+port-forward' && fail 'extracted awk still requires kubectl/port-forward adjacency'+dead_awk='/[[:space:]][0-9]+:[0-9]+(:[0-9]+)?(\.[0-9]+)?[[:space:]]+([^[:space:]]*\/)?kubectl[[:space:]]+port-forward([[:space:]]|$)/ {print}'+printf '%s\n' "$pf_all" "$pf_ctl" "$pf_e2e" | awk "$dead_awk" >"$tmp/pf-dead-awk.out"+[ ! -s "$tmp/pf-dead-awk.out" ] || fail 'real-launcher fixtures still match the dead adjacency regex — fixtures are wrong'+printf '%s\n' "$pf_all" "$pf_ctl" "$pf_e2e" "$bare_pf" "$path_pf" "$wrapper_pf" | awk "$awk_pf" >"$tmp/pf-awk.out"+grep -Fq -- "--context=${pf_profile} -n control-plane port-forward --address=127.0.0.1" "$tmp/pf-awk.out" || fail 'awk missed pf-all-stack real launcher'+grep -Fq 'kubectl -n control-plane port-forward svc/control-api' "$tmp/pf-awk.out" || fail 'awk missed pf-control-stack real launcher'+grep -Fq -- "--context ${pf_profile} port-forward svc/control-ui" "$tmp/pf-awk.out" || fail 'awk missed run-e2e real launcher'+grep -Fq 'kubectl port-forward --context=' "$tmp/pf-awk.out" || fail 'awk missed a bare kubectl port-forward'+grep -Fq '/usr/local/bin/kubectl --context=' "$tmp/pf-awk.out" || fail 'awk missed a path kubectl with flags before port-forward'+read -r uid pid ppid rest <<<"$pf_all"+[ "$pid" = 4242 ] || fail "ps field split left pid='$pid' instead of 4242"++pf_root="$tmp/pf-check-profiles"+mkdir -p "$pf_root/$pf_profile/pids"+pf_bin="$tmp/pf-ps-bin"+mkdir -p "$pf_bin"+cat >"$pf_bin/ps" <<'EOF'+#!/usr/bin/env bash+if [ "${1:-}" = "-ef" ]; then+ cat "${T2_PS_EF_FILE:?}"+ exit 0+fi+pid=""+while [ "$#" -gt 0 ]; do+ case "$1" in+ -p)+ pid="$2"+ shift 2+ ;;+ -p*)+ pid="${1#-p}"+ shift+ ;;+ *)+ shift+ ;;+ esac+done+case "$pid" in+ 4242|4243|4244|4245|4246) printf '%s\n' kubectl ;;+ 4247) printf '%s\n' bash ;;+ *) printf '\n' ;;+esac+EOF+chmod +x "$pf_bin/ps"++run_process_check() {+ local ef_file="$1" out="$2"+ T2_PS_EF_FILE="$ef_file" PATH="$pf_bin:$PATH" \+ T2_PROFILE="$pf_profile" T2_CONTEXT="$pf_profile" \+ MINIKUBE_PROFILE="$pf_profile" \+ T2_PROFILE_ROOT="$pf_root" \+ T2_PROJECT_DIR="$ROOT" \+ T2_LOCK_ROOT="$tmp/locks" \+ bash -c 'source "$1"; t2_process_check' bash "$COMMON" >"$out" 2>&1+}++printf '%s\n' "$pf_all" >"$tmp/ps-ef-real.out"+if run_process_check "$tmp/ps-ef-real.out" "$tmp/pf-real.err"; then+ fail 't2_process_check did not fail a foreign real launcher (PORT_FORWARD_CONFLICT)'+fi+grep -Fq 'PORT_FORWARD_CONFLICT' "$tmp/pf-real.err" || fail 'foreign real launcher did not report PORT_FORWARD_CONFLICT'++printf '%s\n' "$wrapper_pf" >"$tmp/ps-ef-wrap.out"+if ! run_process_check "$tmp/ps-ef-wrap.out" "$tmp/pf-wrap.err"; then+ fail 't2_process_check failed on a wrapper that only mentions kubectl (comm=bash)'+fi+grep -Fq 'REUSE_DB=true' "$ROOT/scripts/minikube/t2.sh"grep -Fq 'CONTROL_DB_RESET_PVC_UID' "$ROOT/scripts/minikube/t2.sh"