Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
31.7 KB
Raw Blame
#!/usr/bin/env bash
# HCC availability across a `strategy: Recreate` + `replicas: 1` ROLLOUT — the
# node-upgrade / config-rollout scenario behind decision D1/c4 (PR #382).
#
# SCOPE — deploy/base/control-plane/host-context-controller.yaml pins the HCC
# to Recreate with a single replica, so EVERY rollout (image bump, config
# change, or the recreate that follows a GKE node-upgrade drain) TERMINATES the
# old pod before the replacement starts. The availability gap is structural:
#   termination + reschedule + container start + initial LISTs over the whole
#   inventory + the NetworkPolicy revocation pass + readiness probe
#   (initialDelaySeconds: 10)
# It was estimated at ~30-90s in the D1/c4 audit but never MEASURED. This gate
# measures it against the wall clock, over a synthetic clerum-dev-scale fleet
# (the initial LISTs and the revocation pass are part of the window), and pins
# the D1b failure mode empirically: `progressDeadlineSeconds: 1200` is
# DETECTION only — a botched Recreate rollout leaves the HCC down until
# evenfire-infra deploy workflows run automatic `rollout undo --to-revision`.
# This e2e proves that operator action; cleanup `set image` is only the
# leftover restore fallback.
#
# Modes:
#   EXPECT_RECOVERY=1 — exclusive with EXPECT_STUCK. Botch the image, prove the
#     D1b 503 outage (same sustained-outage pins as EXPECT_STUCK: >=1 503,
#     zero 503->200, maxstreak > healthy budget), then run last-good revision
#     restore as the TEST ACTION (not cleanup). Recovery must recertify /ready
#     200, restore the original image and last-good revision, and replace the
#     botched pod. An undo no-op is FAIL. IMAGE_BROKEN stays 1 unless every
#     post-undo proof passed (e2e_fail==0), so a failed image/revision/uid/hold
#     still lets cleanup restore the original image. This is the evenfire#391
#     evidence path.
#   EXPECT_STUCK=0 (default) — healthy rollout via `kubectl rollout restart`.
#     rollout restart mutates the pod template (restartedAt annotation), which
#     drives the SAME Deployment machinery — terminate old, then create new —
#     that an image/config rollout or a node-upgrade recreate follows under
#     Recreate. A bare `kubectl delete pod` was rejected as the simulation: it
#     bypasses the Deployment controller entirely (no new pod-template
#     revision, no progress conditions) and could not host the botched-image
#     variant. VERDICT: the 503 window (wall-clock max 503 streak across the
#     rollout phase) MUST close within ROLLOUT_DOWNTIME_BUDGET_SEC; the
#     measured number is the deliverable, recorded in the evidence artifact.
#   EXPECT_STUCK=1 — botched rollout (D1b): roll to a same-repository image tag
#     that cannot exist (e2e-botched-<run>). Recreate terminates the old
#     (healthy) pod, the replacement pins in ImagePullBackOff, and /ready
#     flatlines at 503 with ZERO 503->200 transitions: the stuck rollout IS a
#     total outage, empirically. The sustained 503 streak — longer than the
#     healthy budget and still open at observation end — is the EXPECTED
#     evidence. The observation is bounded (STUCK_OBSERVE_SEC) and cleanup
#     performs the fallback image restore (restore the original image, wait
#     for re-certification), so the gate always terminates loudly instead of
#     hanging alongside the outage it reproduces. Production recovery is the
#     infra-owned automatic `rollout undo --to-revision` (evenfire#391).
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/e2e/e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-lock.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-recovery-fixture.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-watch-churn-fixture.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/hcc-ready-series.sh"

# ── Fail-closed guards (identical contract to the sibling HCC gates) ──
[ -n "$E2E_KUBECONTEXT" ] || {
  echo "KUBECONTEXT/E2E_K8S_CONTEXT must select a branch-scoped minikube context." >&2
  exit 1
}
is_branch_scoped_e2e_context "$E2E_KUBECONTEXT" || {
  echo "Refusing rollout fault injection on non-branch context '${E2E_KUBECONTEXT}'." >&2
  exit 1
}
require_safe_kube_context
command -v jq >/dev/null 2>&1 || {
  echo "jq is required" >&2
  exit 1
}
[ "${E2E_HCC_ROLLOUT_FAULT_INJECTION:-0}" = 1 ] || {
  echo "Set E2E_HCC_ROLLOUT_FAULT_INJECTION=1 to acknowledge that this gate rolls" >&2
  echo "(and, with EXPECT_STUCK=1 or EXPECT_RECOVERY=1, deliberately breaks) the HCC deployment." >&2
  exit 1
}
case "${EXPECT_STUCK:-0}" in
  0 | 1) ;;
  *)
    echo "EXPECT_STUCK must be 0 (measure a healthy rollout window) or 1 (reproduce the D1b botched-rollout outage)." >&2
    exit 1
    ;;
esac
EXPECT_STUCK="${EXPECT_STUCK:-0}"
EXPECT_RECOVERY="${EXPECT_RECOVERY:-0}"
case "${EXPECT_RECOVERY}" in
  0 | 1) ;;
  *)
    echo "EXPECT_RECOVERY must be 0 or 1." >&2
    exit 1
    ;;
esac
if [ "$EXPECT_STUCK" = 1 ] && [ "$EXPECT_RECOVERY" = 1 ]; then
  echo "EXPECT_STUCK=1 and EXPECT_RECOVERY=1 are exclusive." >&2
  exit 1
fi
kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \
  'any(.items[]; .metadata.labels["minikube.k8s.io/name"] == $c)' >/dev/null ||
  {
    echo "Refusing rollout fault injection: target is not this profile's minikube node." >&2
    exit 1
  }

# ── Tunables ──
HCC_NS="${HCC_NS:-control-plane}"
HCC_DEPLOY="${HCC_DEPLOY:-host-context-controller}"
MCP_NS="${MCP_SERVER_NS:-mcp-server}"
HOST_NS="${MCP_HOST_NS:-mcp-host}"
# Wall-clock budget for the healthy-rollout 503 window. Deliberately generous:
# the point of EXPECT_STUCK=0 is to MEASURE the window, not to fail it — the
# observed number lands in the evidence artifact and in the verdict line.
ROLLOUT_DOWNTIME_BUDGET_SEC="${ROLLOUT_DOWNTIME_BUDGET_SEC:-120}"
# Total observation budget for the healthy rollout: pod termination (default
# grace 30s) + the downtime window + margin.
ROLLOUT_OBSERVE_BUDGET_SEC="${ROLLOUT_OBSERVE_BUDGET_SEC:-$((ROLLOUT_DOWNTIME_BUDGET_SEC + 120))}"
# Bounded observation of the botched rollout: must exceed the healthy budget so
# "the outage outlived the budget and never closed" is a provable claim.
STUCK_OBSERVE_SEC="${STUCK_OBSERVE_SEC:-$((ROLLOUT_DOWNTIME_BUDGET_SEC + 90))}"
BASELINE_SAMPLE_SEC="${BASELINE_SAMPLE_SEC:-10}"
STABILITY_WINDOW_SEC="${STABILITY_WINDOW_SEC:-30}"
BOOTSTRAP_READY_BUDGET_SEC="${BOOTSTRAP_READY_BUDGET_SEC:-360}"
# Synthetic clerum-dev-scale fleet (same shape as the watch-churn gate): the
# replacement pod's initial LISTs and revocation pass must run over a realistic
# inventory, or the measured window is a bare-cluster lower bound.
WITH_SYNTHETIC_FLEET="${WITH_SYNTHETIC_FLEET:-1}"
FLEET_CONTEXTS="${FLEET_CONTEXTS:-24}"
FLEET_MCPSERVERS="${FLEET_MCPSERVERS:-115}"
FLEET_HOSTS="${FLEET_HOSTS:-8}"
[ "$STUCK_OBSERVE_SEC" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] || {
  echo "STUCK_OBSERVE_SEC (${STUCK_OBSERVE_SEC}) must exceed ROLLOUT_DOWNTIME_BUDGET_SEC (${ROLLOUT_DOWNTIME_BUDGET_SEC}):" >&2
  echo "the sustained-outage claim needs an observation longer than the budget it must exceed." >&2
  exit 1
}
case "$WITH_SYNTHETIC_FLEET" in
  0 | 1) ;;
  *)
    echo "WITH_SYNTHETIC_FLEET must be 0 or 1." >&2
    exit 1
    ;;
esac

RUN_ID="$(date +%s)-$$"
FLEET_PREFIX="$(truncate_rfc1123 "e2e-hcc-rollout-${RUN_ID}")"
FLEET_SECRET="$(truncate_rfc1123 "${FLEET_PREFIX}-llm")"
LOG_ARTIFACT="$(mktemp "${TMPDIR:-/tmp}/hcc-rollout-readiness.XXXXXX")"
# 1Hz /ready sample series: "<epoch> <status> <phase>" per line (baseline /
# rollout / stuck / post-hold). All window math derives from this file.
READY_SERIES="$(mktemp "${TMPDIR:-/tmp}/hcc-rollout-series.XXXXXX")"
ORIGINAL_REPLICAS=""
HCC_IMAGE=""
BOTCHED_IMAGE=""
FLEET_CREATED=0
HCC_SCALED_DOWN=0
ROLLOUT_TRIGGERED=0
IMAGE_BROKEN=0
ROLLOUT_CONVERGED=0
last_status=""
# Verdict-evidence counters; assigned by the series readers before any read.
baseline_total=0 baseline_200=0 baseline_503=0
roll_total=0 roll_200=0 roll_503=0 roll_maxstreak=0 roll_transitions=0
stuck_total=0 stuck_200=0 stuck_503=0 stuck_maxstreak=0 stuck_transitions=0
hold_total=0 hold_503=0
OLD_POD_NAME="" OLD_POD_UID="" NEW_POD_NAME="" NEW_POD_UID="" NEW_POD_RESTARTS=""
rollout_t0="" rollout_recovered_at=""
# shellcheck disable=SC2034
HCC_GATE_LOCK_ACQUIRED=0 HCC_GATE_LOCK_NAME="" HCC_GATE_LOCK_UID="" HCC_GATE_FINALIZATION_FAILURE=""

die() {
  fail "$*"
  exit 1
}

hcc_ready_now() {
  local pod
  pod="$(running_hcc_pod)" && [ -n "$pod" ] && [ "$(ready_status "$pod")" = 200 ]
}

# The rollout-phase sampler: records "<epoch> <status> <phase>" at ~1Hz like
# sample_ready_series, but converges only when a pod with a DIFFERENT uid
# serves 200 — a 200 from the not-yet-terminated OLD pod must not end the
# observation, and the same pod re-certifying would be the churn gate's claim,
# not a rollout. Sets NEW_POD_NAME / NEW_POD_UID on convergence.
sample_until_new_pod_ready() {
  local duration=$1 phase=$2 old_uid=$3
  local s_deadline t0 now pod status uid
  s_deadline=$(($(date +%s) + duration))
  while t0="$(date +%s)"; [ "$t0" -lt "$s_deadline" ]; do
    status=503 uid="" pod=""
    if pod="$(running_hcc_pod)" && [ -n "$pod" ]; then
      status="$(ready_status "$pod")"
      uid="$(kctl get pod "$pod" -n "$HCC_NS" -o jsonpath='{.metadata.uid}' 2>/dev/null || true)"
    fi
    printf '%s %s %s\n' "$t0" "$status" "$phase" >>"$READY_SERIES"
    last_status="$status"
    if [ "$status" = 200 ] && [ -n "$uid" ] && [ "$uid" != "$old_uid" ]; then
      NEW_POD_NAME="$pod"
      NEW_POD_UID="$uid"
      return 0
    fi
    now="$(date +%s)"
    [ $((now - t0)) -ge 1 ] || sleep 1
  done
  return 1
}

# Same-repository tag that cannot exist: guaranteed ErrImagePull/
# ImagePullBackOff with zero foreign-image pulls — the least destructive and
# most faithful "bad rollout" (the Deployment machinery is exercised exactly as
# a real bad image bump would exercise it).
botched_image_ref() {
  local base="${HCC_IMAGE%%@*}" repo_tail
  repo_tail="${base##*/}"
  case "$repo_tail" in
    *:*) base="${base%:*}" ;;
  esac
  printf '%s:e2e-botched-%s' "$base" "$RUN_ID"
}

# Shared D1b outage pins for EXPECT_STUCK and EXPECT_RECOVERY. flavor=stuck
# keeps the longer D1b strings; flavor=recovery keeps the pre-undo variants.
# A failed pod-UID listing must not count as "old pod terminated".
assert_botched_outage() {
  local flavor="$1" old_uid_query
  local image_fail old_pod_ok old_pod_fail replacement_fail
  local transitions_ok transitions_fail streak_ok streak_fail
  case "$flavor" in
    stuck | recovery) ;;
    *) die "assert_botched_outage: flavor must be stuck or recovery, got '${flavor}'" ;;
  esac

  deployed_image_now="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
    -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}' 2>/dev/null || true)"
  old_uid_live=-1
  if old_uid_query="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
    -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}')"; then
    old_uid_live="$(printf '%s\n' "$old_uid_query" | grep -cxF "$OLD_POD_UID" || true)"
  else
    fail "could not list HCC pod UIDs — refusing to treat a query failure as old-pod termination"
  fi
  deployment_revision="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
    -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"
  stuck_pod_rows="$(kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\t"}{.metadata.deletionTimestamp}{"\t"}{.metadata.ownerReferences[?(@.kind=="ReplicaSet")].name}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].image}{"\t"}{.status.containerStatuses[?(@.name=="host-context-controller")].state.waiting.reason}{"\n"}{end}' 2>/dev/null || true)"
  replacement_row="$(awk -F '\t' -v old="$OLD_POD_UID" -v image="$BOTCHED_IMAGE" \
    '$2 != old && $2 != "" && $4 == "" && $3 != "Failed" && $3 != "Succeeded" && $5 != "" && $6 == image && $7 ~ /ErrImagePull|ImagePullBackOff/ { print; exit }' <<<"$stuck_pod_rows")"
  replacement_owner="$(cut -f5 <<<"$replacement_row")"
  replacement_owner_revision=""
  if [ -n "$replacement_owner" ]; then
    replacement_owner_revision="$(kctl get rs "$replacement_owner" -n "$HCC_NS" \
      -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}' 2>/dev/null || true)"
  fi

  case "$flavor" in
    recovery)
      image_fail="deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE}"
      old_pod_ok="Recreate terminated the old HEALTHY pod before any viable replacement existed"
      old_pod_fail="old pod ${OLD_POD_NAME} is still alive under a Recreate rollout"
      replacement_fail="no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none})"
      transitions_ok="no 503->200 transition closed the outage before undo — Kubernetes did NOT self-heal"
      transitions_fail="the outage closed on its own (${stuck_transitions} 503->200 transition(s)) before undo"
      streak_ok="SUSTAINED outage before undo: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) — undo recovers D1b, not a blip"
      streak_fail="outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — undo would recover a blip, not D1b"
      ;;
    stuck)
      image_fail="deployment image is '${deployed_image_now:-unset}', not the injected ${BOTCHED_IMAGE} — the outage (if any) has a different cause"
      old_pod_ok="Recreate terminated the old HEALTHY pod before any viable replacement existed — zero fallback replicas (the D1b mechanism)"
      old_pod_fail="old pod ${OLD_POD_NAME} is still alive under a Recreate rollout — the terminate-before-create premise did not hold"
      replacement_fail="no replacement pod pinned in ErrImagePull|ImagePullBackOff (rows: ${stuck_pod_rows:-none}) — the outage is not attributable to the injected bad rollout"
      transitions_ok="no 503->200 transition ever closed the outage — Kubernetes did NOT self-heal the botched Recreate rollout"
      transitions_fail="the outage closed on its own (${stuck_transitions} 503->200 transition(s)) — the D1b premise that recovery requires an explicit rollback is refuted; investigate before trusting this gate"
      streak_ok="SUSTAINED outage measured: ${stuck_maxstreak}s continuous 503 (> ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget) and still open at observation end — recovery requires an explicit rollback (infra-owned rollout undo --to-revision in production; cleanup's set image restore here) (D1b reproduced)"
      streak_fail="outage lasted only ${stuck_maxstreak}s and did not exceed the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s healthy budget — the botched rollout did not produce the D1b total outage"
      ;;
  esac

  [ "$stuck_total" -gt 0 ] && ok "sampled /ready ${stuck_total}x across the botched rollout" ||
    fail "ZERO /ready samples across the botched rollout — the sampler never ran, and a zero-sample run must never pass"
  [ "$deployed_image_now" = "$BOTCHED_IMAGE" ] &&
    ok "the botched image is what the Deployment is actually rolling (${BOTCHED_IMAGE})" ||
    fail "$image_fail"
  [ "$old_uid_live" = 0 ] && ok "$old_pod_ok" || fail "$old_pod_fail"
  if [ -n "$replacement_row" ] && grep -Eq 'ErrImagePull|ImagePullBackOff' <<<"$replacement_row"; then
    if [ -n "$deployment_revision" ] && [ "$replacement_owner_revision" = "$deployment_revision" ]; then
      ok "replacement pod is a live current-revision pod pinned by the injected image failure: ${replacement_row}"
    else
      fail "replacement pod owner revision ${replacement_owner_revision:-unset} does not match Deployment revision ${deployment_revision:-unset}"
    fi
  else
    fail "$replacement_fail"
  fi
  [ "$stuck_503" -ge 1 ] &&
    ok "the botched rollout bit the readiness path: ${stuck_503} sample(s) at 503" ||
    fail "zero 503 samples under a botched Recreate rollout — the outage claim is VACUOUS"
  [ "$stuck_transitions" -eq 0 ] && ok "$transitions_ok" || fail "$transitions_fail"
  [ "$stuck_maxstreak" -gt "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] && ok "$streak_ok" || fail "$streak_fail"
}

print_repair_instructions() {
  cat >&2 <<EOF
HCC rollout gate cleanup could not restore a verified healthy state.
Context: ${E2E_KUBECONTEXT}
HCC: ${HCC_NS}/${HCC_DEPLOY}
Original image: ${HCC_IMAGE:-unknown}
Synthetic fleet label: e2e.clerum.io/suite=hcc-watch-churn (run ${RUN_ID})

Inspect before changing anything:
  kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get deployment ${HCC_DEPLOY} -o yaml
  kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} get pods -l app=${HCC_DEPLOY} -o wide

Restore the HCC deployment (original image, original replicas, wait for readiness):
  kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} set image deployment/${HCC_DEPLOY} \\
    host-context-controller=${HCC_IMAGE:-<original image>}
  kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} scale deployment/${HCC_DEPLOY} --replicas=${ORIGINAL_REPLICAS:-1}
  kubectl --context=${E2E_KUBECONTEXT} -n ${HCC_NS} rollout status deployment/${HCC_DEPLOY} --timeout=240s

Delete any leftover synthetic fleet:
  kubectl --context=${E2E_KUBECONTEXT} delete mcpserver,context,host -A -l e2e.clerum.io/suite=hcc-watch-churn
EOF
}

write_evidence_artifact() {
  local rollout_transition="not-observed"
  if [ -n "$rollout_recovered_at" ] && [ -n "$rollout_t0" ]; then
    rollout_transition="$((rollout_recovered_at - rollout_t0))s"
  fi

  # Diagnostics ONLY — every verdict input is computed before this runs, so
  # every command here is deliberately best-effort (same rationale as the
  # sibling churn gate: an empty grep under `set -euo pipefail` must never
  # abort the gate before its verdict).
  {
    echo "=== mode: EXPECT_STUCK=${EXPECT_STUCK} EXPECT_RECOVERY=${EXPECT_RECOVERY}  fleet=${WITH_SYNTHETIC_FLEET} (${FLEET_CONTEXTS}/${FLEET_MCPSERVERS}/${FLEET_HOSTS}) ==="
    echo "=== HCC deploy ==="
    kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o wide || true
    echo "=== deployment conditions (progressDeadlineSeconds is DETECTION only) ==="
    kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
      -o jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' || true
    echo ""
    echo "=== pods ==="
    kctl get pods -n "$HCC_NS" -l "app=${HCC_DEPLOY}" -o wide || true
    echo "=== baseline: samples=${baseline_total} 200=${baseline_200} 503=${baseline_503} ==="
    if [ "$EXPECT_RECOVERY" = 1 ]; then
      echo "=== recovered pod identity: old ${OLD_POD_NAME:-none} (${OLD_POD_UID:-not-observed}) -> recovered ${NEW_POD_NAME:-none} (${NEW_POD_UID:-not-observed}) restarts=${NEW_POD_RESTARTS:-not-observed} ==="
      echo "=== recovery revisions: last-good=${LAST_GOOD_REVISION:-not-captured} botched=${live_revision_before_undo:-not-observed} recovered=${recovered_revision:-not-observed} ==="
      echo "=== pre-undo outage: samples=${stuck_total} 200=${stuck_200} 503=${stuck_503} max-503-streak=${stuck_maxstreak}s 503->200=${stuck_transitions} observe=${STUCK_OBSERVE_SEC}s ==="
      echo "=== post-undo hold: samples=${hold_total} 503=${hold_503} ==="
    elif [ "$EXPECT_STUCK" = 1 ]; then
      echo "=== botched replacement: ${replacement_row:-not-observed} ==="
      echo "=== stuck outage: samples=${stuck_total} 200=${stuck_200} 503=${stuck_503} max-503-streak=${stuck_maxstreak}s 503->200=${stuck_transitions} observe=${STUCK_OBSERVE_SEC}s ==="
    else
      echo "=== pod identity: old ${OLD_POD_NAME:-none} (${OLD_POD_UID:-not-observed}) -> new ${NEW_POD_NAME:-none} (${NEW_POD_UID:-not-observed}) restarts=${NEW_POD_RESTARTS:-not-observed} ==="
      echo "=== rollout: samples=${roll_total} 200=${roll_200} 503=${roll_503} MEASURED-window=${roll_maxstreak}s 503->200=${roll_transitions} budget=${ROLLOUT_DOWNTIME_BUDGET_SEC}s total-transition=${rollout_transition} ==="
      echo "=== post-rollout hold: samples=${hold_total} 503=${hold_503} ==="
    fi
    echo "=== /ready series (epoch status phase) ==="
    cat "$READY_SERIES" || true
  } >"$LOG_ARTIFACT" 2>&1 || true
}

cleanup() {
  local status=$? cleanup_failed=0 restore_ok=1
  trap - EXIT
  set +e
  if [ "$IMAGE_BROKEN" = 1 ]; then
    # Leftover restore fallback only. Pipeline/infra owns automatic
    # `rollout undo --to-revision`; this e2e proves that operator action.
    # Cleanup `set image` restores the original image if IMAGE_BROKEN is
    # still set after that proof.
    kctl set image deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
      host-context-controller="$HCC_IMAGE" >/dev/null 2>&1 || restore_ok=0
  fi
  if [ "$HCC_SCALED_DOWN" = 1 ]; then
    kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas="${ORIGINAL_REPLICAS:-1}" >/dev/null 2>&1 || restore_ok=0
  fi
  if [ "$ROLLOUT_TRIGGERED" = 1 ] || [ "$IMAGE_BROKEN" = 1 ] || [ "$HCC_SCALED_DOWN" = 1 ]; then
    kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout=240s >/dev/null 2>&1 || restore_ok=0
    wait_until 90 "HCC /ready after restore" hcc_ready_now >/dev/null 2>&1 || restore_ok=0
  fi
  [ "$FLEET_CREATED" = 1 ] && { delete_synthetic_fleet || cleanup_failed=1; }
  [ "$restore_ok" = 1 ] || {
    print_repair_instructions
    cleanup_failed=1
  }
  finalize_hcc_watch_gate_lock "$cleanup_failed" "$restore_ok" || cleanup_failed=1
  print_results || status=1
  [ "$cleanup_failed" = 0 ] || status=1
  rm -f "$READY_SERIES"
  exit "$status"
}
trap cleanup EXIT

# ── FASE A: branch-owned proof + single-writer lock + premise snapshot ──
require_branch_owned_hcc_gate "$HCC_NS"
acquire_hcc_watch_gate_lock
ORIGINAL_REPLICAS="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.replicas}')"
[ "$ORIGINAL_REPLICAS" = 1 ] || die "expected exactly one HCC replica, found ${ORIGINAL_REPLICAS:-unknown}"
strategy_type="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.strategy.type}')"
[ "$strategy_type" = Recreate ] ||
  die "expected strategy Recreate — the D1/c4 scenario under measurement — but found '${strategy_type:-unset}'; every number this gate reports would be meaningless"
[ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.strategy.rollingUpdate}')" ] ||
  die "deployment carries rollingUpdate parameters alongside Recreate; refusing to measure a mixed-strategy object"
HCC_IMAGE="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" \
  -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
[ -n "$HCC_IMAGE" ] || die "could not resolve the running HCC image"

# ── FASE B: synthetic clerum-dev-scale fleet (HCC stopped while CRs land) ──
if [ "$WITH_SYNTHETIC_FLEET" = 1 ]; then
  log "Creating synthetic fleet: ${FLEET_CONTEXTS} Contexts / ${FLEET_MCPSERVERS} McpServers / ${FLEET_HOSTS} Hosts"
  kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=0 >/dev/null
  HCC_SCALED_DOWN=1
  wait_until 120 "HCC pods to stop" hcc_pods_absent || die "HCC did not stop before fleet creation"
  create_synthetic_fleet "$FLEET_CONTEXTS" "$FLEET_MCPSERVERS" "$FLEET_HOSTS"
  ok "synthetic fleet created (${FLEET_MCPSERVERS} McpServers)"
  kctl scale deployment "$HCC_DEPLOY" -n "$HCC_NS" --replicas=1 >/dev/null
  HCC_SCALED_DOWN=0
  kctl rollout status deployment "$HCC_DEPLOY" -n "$HCC_NS" --timeout="${BOOTSTRAP_READY_BUDGET_SEC}s" >/dev/null ||
    die "HCC did not bootstrap over the synthetic fleet within ${BOOTSTRAP_READY_BUDGET_SEC}s"
fi
wait_until 60 "HCC /ready before the rollout" hcc_ready_now ||
  die "HCC is not serving /ready 200; refusing to measure a rollout from an unhealthy baseline"

# ── FASE C: baseline — the window is only meaningful from a clean 200 floor ──
log "Baseline: sampling /ready at 1Hz for ${BASELINE_SAMPLE_SEC}s"
sample_ready_series "$BASELINE_SAMPLE_SEC" baseline 0
read -r baseline_total baseline_200 baseline_503 _ _ <<<"$(series_metrics baseline)"
if [ "$baseline_total" -gt 0 ] && [ "$baseline_503" -eq 0 ]; then
  ok "baseline stable: ${baseline_200}/${baseline_total} samples at 200 before the rollout"
else
  die "baseline unstable (${baseline_503}/${baseline_total} samples at 503) — refusing to mutate an unhealthy HCC"
fi
OLD_POD_NAME="$(running_hcc_pod)"
[ -n "$OLD_POD_NAME" ] || die "no running HCC pod after a clean baseline"
OLD_POD_UID="$(kctl get pod "$OLD_POD_NAME" -n "$HCC_NS" -o jsonpath='{.metadata.uid}')"
[ -n "$OLD_POD_UID" ] || die "could not pin the pre-rollout pod uid"
if [ "$EXPECT_STUCK" = 0 ] && [ "$EXPECT_RECOVERY" = 0 ]; then
  # ── FASE D (healthy): Recreate rollout, sampled at 1Hz end to end ──
  log "Triggering a Recreate rollout (kubectl rollout restart) and sampling /ready at 1Hz"
  kctl rollout restart deployment/"$HCC_DEPLOY" -n "$HCC_NS" >/dev/null
  ROLLOUT_TRIGGERED=1
  rollout_t0="$(date +%s)"
  if sample_until_new_pod_ready "$ROLLOUT_OBSERVE_BUDGET_SEC" rollout "$OLD_POD_UID"; then
    ROLLOUT_CONVERGED=1
    rollout_recovered_at="$(date +%s)"
    NEW_POD_RESTARTS="$(hcc_restart_count "$NEW_POD_NAME" || true)"
    log "Post-rollout hold: sampling /ready at 1Hz for ${STABILITY_WINDOW_SEC}s"
    sample_ready_series "$STABILITY_WINDOW_SEC" post-hold 0
  fi
  read -r roll_total roll_200 roll_503 roll_maxstreak roll_transitions <<<"$(series_metrics rollout)"
  read -r hold_total _ hold_503 _ _ <<<"$(series_metrics post-hold)"
  write_evidence_artifact

  # ── Verdict (healthy rollout): measure the window, prove it was a rollout ──
  [ "$roll_total" -gt 0 ] && ok "sampled /ready ${roll_total}x across the rollout" ||
    fail "ZERO /ready samples across the rollout — the sampler never ran, and a zero-sample run must never pass"
  if [ "$ROLLOUT_CONVERGED" = 1 ]; then
    ok "replacement pod certified /ready ${rollout_recovered_at:+$((rollout_recovered_at - rollout_t0))}s after the rollout was triggered"
  else
    fail "no replacement pod certified /ready within ${ROLLOUT_OBSERVE_BUDGET_SEC}s — the rollout never recovered (see ${LOG_ARTIFACT})"
  fi
  if [ -n "$NEW_POD_UID" ] && [ "$NEW_POD_UID" != "$OLD_POD_UID" ] && [ "$NEW_POD_NAME" != "$OLD_POD_NAME" ]; then
    ok "pod was REPLACED: ${OLD_POD_NAME} -> ${NEW_POD_NAME} (uid changed) — the Recreate rollout was real"
  else
    fail "pod did not change across the rollout (${OLD_POD_NAME} -> ${NEW_POD_NAME:-none}) — nothing was measured"
  fi
  [ "${NEW_POD_RESTARTS:-}" = 0 ] &&
    ok "replacement pod is a fresh boot: restartCount 0" ||
    fail "replacement pod restartCount is '${NEW_POD_RESTARTS:-unknown}' — expected a fresh pod, not a restarted container"
  [ "$roll_503" -ge 1 ] &&
    ok "the rollout actually cut availability: ${roll_503} sample(s) at 503 (Recreate closes the old pod before the new one certifies)" ||
    fail "zero 503 samples across a Recreate rollout — either RollingUpdate crept in or the measurement is broken; the window claim is VACUOUS"
  [ "$roll_maxstreak" -le "$ROLLOUT_DOWNTIME_BUDGET_SEC" ] &&
    ok "MEASURED rollout downtime window: ${roll_maxstreak}s wall-clock 503 streak <= ${ROLLOUT_DOWNTIME_BUDGET_SEC}s budget" ||
    fail "MEASURED rollout downtime window ${roll_maxstreak}s EXCEEDS the ${ROLLOUT_DOWNTIME_BUDGET_SEC}s budget (see ${LOG_ARTIFACT})"
  [ "$hold_total" -gt 0 ] && [ "$hold_503" -eq 0 ] &&
    ok "post-rollout hold: ${hold_total} samples over ${STABILITY_WINDOW_SEC}s, zero 503 — the replacement is stable" ||
    fail "post-rollout hold saw ${hold_503} 503(s) across ${hold_total} sample(s) — the replacement did not stabilize"
elif [ "$EXPECT_STUCK" = 1 ]; then
  # ── FASE D (botched, D1b): roll to an unpullable image and watch the outage ──
  BOTCHED_IMAGE="$(botched_image_ref)"
  log "Botched rollout (D1b): rolling to unpullable image ${BOTCHED_IMAGE}; observing /ready for ${STUCK_OBSERVE_SEC}s"
  kctl set image deployment/"$HCC_DEPLOY" -n "$HCC_NS" \
    host-context-controller="$BOTCHED_IMAGE" >/dev/null
  ROLLOUT_TRIGGERED=1
  IMAGE_BROKEN=1
  sample_ready_series "$STUCK_OBSERVE_SEC" stuck 0
  read -r stuck_total stuck_200 stuck_503 stuck_maxstreak stuck_transitions <<<"$(series_metrics stuck)"
  assert_botched_outage stuck
  write_evidence_artifact
else
  # ── FASE D (recovery, evenfire#391): botch, prove the outage, then undo ──
  LAST_GOOD_REVISION="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
  [ -n "$LAST_GOOD_REVISION" ] || die "could not capture the last-known-good HCC revision"
  BOTCHED_IMAGE="$(botched_image_ref)"
  PRE_BOTCH_UID="$OLD_POD_UID"
  log "Botched rollout (EXPECT_RECOVERY): observing /ready for ${STUCK_OBSERVE_SEC}s"
  kctl set image deployment/"$HCC_DEPLOY" -n "$HCC_NS" host-context-controller="$BOTCHED_IMAGE" >/dev/null
  ROLLOUT_TRIGGERED=1
  IMAGE_BROKEN=1
  sample_ready_series "$STUCK_OBSERVE_SEC" stuck 0
  read -r stuck_total stuck_200 stuck_503 stuck_maxstreak stuck_transitions <<<"$(series_metrics stuck)"
  # D1b outage pins, including stuck_maxstreak -gt ROLLOUT_DOWNTIME_BUDGET_SEC.
  assert_botched_outage recovery
  BOTCHED_POD_UID="$(awk -F '\t' '{ print $2; exit }' <<<"$replacement_row")"

  live_revision_before_undo="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
  [ "$live_revision_before_undo" != "$LAST_GOOD_REVISION" ] ||
    fail "live revision ${live_revision_before_undo} still equals last-good ${LAST_GOOD_REVISION} — undo would be a no-op"

  log "TEST ACTION (evenfire#391): rollout undo --to-revision=${LAST_GOOD_REVISION}"
  kctl rollout undo deployment/"$HCC_DEPLOY" -n "$HCC_NS" --to-revision="$LAST_GOOD_REVISION" >/dev/null ||
    fail "rollout undo --to-revision=${LAST_GOOD_REVISION} failed"
  kctl rollout status deployment/"$HCC_DEPLOY" -n "$HCC_NS" --timeout=240s >/dev/null ||
    fail "last-known-good revision ${LAST_GOOD_REVISION} did not become Ready"
  wait_until 90 "HCC /ready after undo" hcc_ready_now ||
    fail "HCC /ready did not return 200 after last-good restore"

  recovered_image="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}')"
  recovered_revision="$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" -o jsonpath='{.metadata.annotations.deployment\.kubernetes\.io/revision}')"
  NEW_POD_NAME="$(running_hcc_pod)"
  NEW_POD_UID="$(kctl get pod "$NEW_POD_NAME" -n "$HCC_NS" -o jsonpath='{.metadata.uid}')"
  NEW_POD_RESTARTS="$(hcc_restart_count "$NEW_POD_NAME" || true)"
  sample_ready_series "$STABILITY_WINDOW_SEC" post-hold 0
  read -r hold_total _ hold_503 _ _ <<<"$(series_metrics post-hold)"
  write_evidence_artifact

  [ "$recovered_image" = "$HCC_IMAGE" ] &&
    ok "undo restored the original image ${HCC_IMAGE}" ||
    fail "image after undo is '${recovered_image:-unset}', expected ${HCC_IMAGE}"
  [ "$recovered_revision" != "$live_revision_before_undo" ] &&
    ok "undo created a new revision ${recovered_revision} from last-good ${LAST_GOOD_REVISION} (was ${live_revision_before_undo})" ||
    fail "revision after undo is still ${recovered_revision:-unset} — undo was a no-op"
  if [ -n "$NEW_POD_UID" ] && [ "$NEW_POD_UID" != "$PRE_BOTCH_UID" ] && [ "$NEW_POD_UID" != "$BOTCHED_POD_UID" ]; then
    ok "recovered pod uid ${NEW_POD_UID} is neither the pre-botch nor the botched pod"
  else
    fail "recovered pod uid '${NEW_POD_UID:-none}' matches pre-botch ${PRE_BOTCH_UID} or botched ${BOTCHED_POD_UID:-none} — undo was a no-op"
  fi
  [ "$hold_total" -gt 0 ] && [ "$hold_503" -eq 0 ] &&
    ok "post-undo hold: ${hold_total} samples over ${STABILITY_WINDOW_SEC}s, zero 503" ||
    fail "post-undo hold saw ${hold_503} 503(s) across ${hold_total} sample(s)"
  # fail() records and continues, so only clear the broken-image flag when
  # every post-undo proof above actually passed. Otherwise cleanup must still
  # restore the original image.
  [ "$e2e_fail" -eq 0 ] && IMAGE_BROKEN=0
fi


log "Evidence artifact: ${LOG_ARTIFACT}"
# cleanup() (EXIT trap) restores the image if broken, waits for re-certification,
# deletes the fleet, finalizes the single-writer lock, and runs print_results.