Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
28.7 KB
Raw Blame
#!/usr/bin/env bash
# ======================================================================
# E2E — Stateless T_idle calibration sweep (compressed-local)
#
# PURPOSE: empirically ground the Time-to-Idle (T_idle) choice. The
# product hypothesis is T_idle = 30 min in production
# (CONTEXT_MAPPER_STATELESS_IDLE_MINUTES default, host-context-controller/
# src/config.ts:524). A local minikube cannot replay 30-minute human
# gaps, so this harness runs a TIME-COMPRESSED sweep over small T
# candidates and session gaps expressed as MULTIPLES of each candidate.
# That validates the tradeoff-curve MECHANICS (threshold correctness,
# resume-outcome classification, recovery latency per class, pod-uptime
# cost integration) — NOT the production constant. The transfer function
# to production data lives in docs/testing/stateless-idle-calibration.md.
#
# THE KNOB (verified — do not "fix" this to a Host CRD patch): T_idle is
# NOT a per-Host CRD field. Host CRD spec.lifecycle carries only
# `stateless: boolean` (charts/clerum-crds/crds/host.yaml). The idle
# threshold is HCC-global env:
#   CONTEXT_MAPPER_STATELESS_IDLE_MINUTES        (default 30)
#   CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES  (default 15)
# and the tracker applies T_idle = max(idleMinutes, idleFloorMinutes)
# (statelessLifecycleTracker.ts:220; drain rule in its doc comment:
# drain = statelessEnabled && !activeWork && now-lastActivityTs >= T_idle).
# The sweep therefore sets the candidate via `kubectl set env` on the HCC
# deployment (floor pinned to 1, the legal minimum — tracker rejects <=0),
# exactly like e2e-stateless-suspend-wake.sh does, and RESTORES the
# original env in the cleanup trap.
#
# CADENCE MATH (mirrors the sibling suite header — read before touching
# timeouts): the mcp-host heartbeat emitter runs at a FIXED 30s cadence
# (CLERUM_STATELESS_HEARTBEAT_INTERVAL_MS default 30000, mcp-host/src/
# config.ts; HCC does not inject it). Suspend detection after idle expiry
# therefore lags by up to emitter(30s) + drainGrace(20s, set below) +
# HCC poll(5s, set below) + reconcile slack. On top of that, a gap that
# follows a COLD wake sits in the wakePending settling window: the tracker
# answers drain:false and cancels in-flight drains while
# wakeRequestedGeneration > wakeHandledGeneration (statelessLifecycleTracker
# "Cancel-drain wins: never answer drain:true over a pending wake"), and the
# handled marker only catches up on a reconciler pass (observed cadence
# ~3min). Measured 2026-07-05 (T=1, event after a cold_wake): suspend was
# still blocked at +150s, while steady-state suspend completes in ~15s once
# eligible. Hence SUSPEND_DETECT_SLACK=300s: one reconcile interval (~200s)
# + clean drain tail (~35s) + margin. Supra-threshold gaps are judged
# against T*60 + SUSPEND_DETECT_SLACK, never against the raw gap alone.
#
# GUARDIAN CONTRACT (all HARD FAILURES, no silent skips):
#   (a) a sub-threshold gap (g < 1x) that produces state=suspended FAILS
#       — T_idle not honored is a product bug detector, not tuning noise;
#   (b) a supra-threshold gap (g > 1x) that never suspends within
#       T*60 + SUSPEND_DETECT_SLACK FAILS with pod/HCC diagnostics;
#   (c) a resume whose turn times out, hits transport failure, or returns
#       a non-retryable status FAILS (only 503 {code:"host_waking"} is
#       retried, under the same bounded WAKE_HOLD_DEADLINE the sibling
#       suite uses).
# All user turns go through rpc-proxy (POST /rpc/hosts/:hostRef/messages);
# kubectl is used only for observation and labeled preconditions.
#
# DEFAULT MATRIX WALL TIME: candidates "1 2" x 4 events x multiplier
# cycle (0.5 1.5 3 0.5) => gap sleep (1+2)*5.5 = 16.5 min + per-candidate
# overhead (HCC rollout + resumes) ~= 26 min total. EVENTS_PER_CANDIDATE
# defaults to 4 (NOT 6): with 6 the estimate is ~40 min, over the ~35 min
# budget. Set EVENTS_PER_CANDIDATE=6 explicitly for the denser sweep.
#
# Usage:
#   KUBECONTEXT=clerum-test bash scripts/e2e/e2e-stateless-idle-calibration.sh
#   IDLE_CANDIDATES_MIN="1 2" GAP_MULTIPLIERS="0.5 1.5 3" \
#   EVENTS_PER_CANDIDATE=4 DISRUPTION_THRESHOLD=0.34 ...
# ======================================================================
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=scripts/e2e/e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"
require_safe_kube_context

HOST_REF="${E2E_STATELESS_HOST_REF:-chatllm-stateless}"
EXT_BASE="${EXTERNAL_REST_API_BASE_URL:-http://127.0.0.1:8091}"
RPC_BASE="${RPC_PROXY_BASE_URL:-http://127.0.0.1:8094}"
CONTROL_BASE="${CONTROL_API_BASE_URL:-http://127.0.0.1:8090}"
WAKE_BASE="${RPC_GATEWAY_BASE_URL:-${CONTROL_BASE}}"
DEV_EMAIL="${E2E_DEV_LOGIN_EMAIL:-test@clerum.io}"
DEV_PASSWORD="${E2E_USER_PASSWORD:-${ADMIN_PASSWORD:-changeme123!}}"

# --- Sweep parameters -------------------------------------------------
IDLE_CANDIDATES_MIN="${IDLE_CANDIDATES_MIN:-1 2}"
GAP_MULTIPLIERS="${GAP_MULTIPLIERS:-0.5 1.5 3}"
EVENTS_PER_CANDIDATE="${EVENTS_PER_CANDIDATE:-4}"
DISRUPTION_THRESHOLD="${DISRUPTION_THRESHOLD:-0.34}"

# --- Budgets (inherited from the sibling suite) -----------------------
E2E_TURN_TIMEOUT="${E2E_TURN_TIMEOUT:-120}"
POD_READY_TIMEOUT="${POD_READY_TIMEOUT:-180}"
WAKE_HOLD_DEADLINE="${WAKE_HOLD_DEADLINE:-270}"
SUSPEND_DETECT_SLACK="${SUSPEND_DETECT_SLACK:-300}"  # emitter 30 + grace 20 + poll 5 + post-cold-wake wakePending settling (~1 reconcile interval; see header)
UPTIME_SAMPLE_INTERVAL="${UPTIME_SAMPLE_INTERVAL:-5}"

# --- HCC cadence overlay (candidate T is set per-loop; rest fixed) ----
HCC_DEPLOY="host-context-controller"
HCC_NS="${CONTROL_NS:-control-plane}"
TEST_IDLE_FLOOR_MINUTES="1"     # legal minimum; tracker rejects 0 -> candidate T governs
TEST_DRAIN_GRACE_MS="20000"
TEST_POLL_MS="5000"

RUN_ID="$(date +%s)-$$-${RANDOM}"
THREAD_ID="e2e-stateless-idle-calibration-${RUN_ID}"
OUT_JSON="${IDLE_CALIBRATION_OUT:-/tmp/stateless-idle-calibration-${RUN_ID}.json}"
RECORDS_FILE="$(mktemp)"
UPTIME_SUMMARY_FILE="$(mktemp)"
UPTIME_SAMPLES_FILE=""
UPTIME_PID=""

# --- Parameter validation (fail loud on nonsense) ---------------------
for T in $IDLE_CANDIDATES_MIN; do
  case "$T" in ''|*[!0-9]*) echo "IDLE_CANDIDATES_MIN entry '${T}' is not a positive integer (minutes)" >&2; exit 1 ;; esac
  [ "$T" -ge 1 ] || { echo "IDLE_CANDIDATES_MIN entry '${T}' must be >= 1 (tracker rejects idleMinutes <= 0)" >&2; exit 1; }
done
for g in $GAP_MULTIPLIERS; do
  awk -v g="$g" 'BEGIN{ if (g+0 != g || g <= 0) exit 1 }' || { echo "GAP_MULTIPLIERS entry '${g}' is not a positive number" >&2; exit 1; }
  # Multipliers in (0.9, 1.1) race the threshold itself (drain fires at
  # >= T while the emitter samples every 30s) -> unclassifiable, reject.
  awk -v g="$g" 'BEGIN{ if (g > 0.9 && g < 1.1) exit 1 }' || { echo "GAP_MULTIPLIERS entry '${g}' is ambiguous (within 10% of the threshold); pick a clearly sub- or supra-threshold multiple" >&2; exit 1; }
done
case "$EVENTS_PER_CANDIDATE" in ''|*[!0-9]*) echo "EVENTS_PER_CANDIDATE '${EVENTS_PER_CANDIDATE}' is not a positive integer" >&2; exit 1 ;; esac
[ "$EVENTS_PER_CANDIDATE" -ge 1 ] || { echo "EVENTS_PER_CANDIDATE must be >= 1" >&2; exit 1; }
awk -v t="$DISRUPTION_THRESHOLD" 'BEGIN{ if (t+0 != t || t < 0 || t > 1) exit 1 }' || { echo "DISRUPTION_THRESHOLD '${DISRUPTION_THRESHOLD}' must be a number in [0,1]" >&2; exit 1; }

# --- Cleanup trap ------------------------------------------------------
HCC_ENV_SAVED=""

restore_hcc_env() {
  [ -n "$HCC_ENV_SAVED" ] || return 0
  local args=() key val
  while IFS='=' read -r key val; do
    [ -n "$key" ] || continue
    if [ -n "$val" ]; then args+=("${key}=${val}"); else args+=("${key}-"); fi
  done <<< "$HCC_ENV_SAVED"
  [ ${#args[@]} -gt 0 ] || return 0
  log "Restoring HCC cadence env on deployment/${HCC_DEPLOY}"
  kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null 2>&1 || \
    warn "failed to restore HCC env (manual check advised)"
  kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1 || \
    warn "HCC rollout did not settle after env restore"
}

stop_uptime_sampler() {
  if [ -n "$UPTIME_PID" ]; then
    kill "$UPTIME_PID" 2>/dev/null || true
    wait "$UPTIME_PID" 2>/dev/null || true
    UPTIME_PID=""
  fi
}

cleanup_on_exit() {
  local status=$?
  set +e
  stop_uptime_sampler
  # Leave the host Ready for the next suite (best-effort; a suspended
  # host is idempotent-rerunnable state, so only warn on failure).
  if [ "$(lifecycle_state)" = "suspended" ]; then
    post_wake >/dev/null 2>&1 || warn "cleanup wake POST failed; host left suspended (re-runnable)"
  fi
  restore_hcc_env
  rm -f "$RECORDS_FILE" "$UPTIME_SUMMARY_FILE" "$UPTIME_SAMPLES_FILE" 2>/dev/null
  exit "$status"
}
trap cleanup_on_exit EXIT

# --- Observability helpers (mirrors e2e-stateless-suspend-wake.sh) ----
now_ms() { python3 -c 'import time; print(int(time.time()*1000))'; }

pod_diagnostics() {
  echo "--- diagnostics for app=${HOST_REF} in ${MCP_HOST_NS} ---" >&2
  kctl get pods -n "$MCP_HOST_NS" -l "app=${HOST_REF}" -o wide 2>/dev/null >&2 || true
  kctl get host "$HOST_REF" -n "$MCP_HOST_NS" \
    -o jsonpath='{"lifecycle="}{.status.lifecycle}{"\n"}' 2>/dev/null >&2 || true
  kctl logs "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --tail=60 2>/dev/null >&2 || true
}

lifecycle_state()     { kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true; }
deployment_replicas() { kctl get deployment "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo ""; }
current_ready_pod()   { ready_pod_name "$MCP_HOST_NS" "app=${HOST_REF}"; }

pod_uid() {
  local p
  p=$(current_ready_pod) || return 1
  kctl get pod "$p" -n "$MCP_HOST_NS" -o jsonpath='{.metadata.uid}' 2>/dev/null
}

wait_for_ready_pod() {
  local timeout=$1 elapsed=0 name
  while [ "$elapsed" -lt "$timeout" ]; do
    if name=$(current_ready_pod); then printf "%s\n" "$name"; return 0; fi
    sleep "$POLL_INTERVAL"; elapsed=$((elapsed + POLL_INTERVAL))
  done
  return 1
}

# --- RPC helpers (Desktop App auth path, mirrors the sibling suite) ---
SESSION_TOKEN=""; RPC_TOKEN=""
RPC_SCOPES_JSON='["host:message:invoke","host:session:read","host:status:read","host:health:read","host:wake:write"]'

mint_session_token() {
  local resp code body
  resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST "${EXT_BASE}/api/v1/auth/password-login" \
    -H 'Content-Type: application/json' \
    -d "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')") || {
    echo "password-login request failed at ${EXT_BASE}" >&2; return 1; }
  code="$(echo "$resp" | tail -n1)"; body="$(echo "$resp" | sed '$d')"
  [ "$code" = "200" ] || { echo "password-login -> HTTP ${code}: ${body}" >&2; return 1; }
  SESSION_TOKEN="$(echo "$body" | jq -r '.token // empty')"
  [ -n "$SESSION_TOKEN" ] || { echo "password-login returned no .token" >&2; return 1; }
}

mint_rpc_token() {
  local resp code body
  resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST "${EXT_BASE}/api/v1/rpc/token" \
    -H "Authorization: Bearer ${SESSION_TOKEN}" -H 'Content-Type: application/json' \
    -d "$(jq -cn --arg h "$HOST_REF" --argjson s "$RPC_SCOPES_JSON" '{hostRefs:[$h],scopes:$s}')") || {
    echo "rpc/token request failed at ${EXT_BASE}" >&2; return 1; }
  code="$(echo "$resp" | tail -n1)"; body="$(echo "$resp" | sed '$d')"
  [ "$code" = "200" ] || { echo "rpc/token -> HTTP ${code}: ${body}" >&2; return 1; }
  RPC_TOKEN="$(echo "$body" | jq -r '.token // empty')"
  [ -n "$RPC_TOKEN" ] || { echo "rpc/token returned no .token" >&2; return 1; }
}

TURN_STATUS=""; TURN_BODY=""
send_turn() {
  local content=$1 payload resp
  mint_rpc_token || return 1
  payload="$(jq -cn --arg c "$content" --arg t "$THREAD_ID" '{content:$c,threadId:$t}')"
  resp=$(curl -sS -m "$E2E_TURN_TIMEOUT" -w '\n%{http_code}' -X POST \
    "${RPC_BASE}/api/v1/rpc/hosts/${HOST_REF}/messages" \
    -H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json' \
    -d "$payload") || { echo "message POST failed at ${RPC_BASE}" >&2; return 1; }
  TURN_STATUS="$(echo "$resp" | tail -n1)"; TURN_BODY="$(echo "$resp" | sed '$d')"
}

# send_turn_expect_ok: single bounded deadline; the ONLY retryable is the
# documented 503 {code:"host_waking"} while rpc-proxy's wake-and-hold
# settles. Timeout / transport / non-retryable status => caller HARD FAILS
# (guardian (c)). Also records TURN_RECOVERY_MS (send entry -> definitive
# 200), the resume-latency observable.
TURN_RECOVERY_MS=""
send_turn_expect_ok() {
  local content=$1 deadline=$((SECONDS + WAKE_HOLD_DEADLINE)) attempt=0 t0 t1
  t0="$(now_ms)"
  while [ "$SECONDS" -lt "$deadline" ]; do
    attempt=$((attempt + 1))
    send_turn "$content" || return 1
    if [ "$TURN_STATUS" = "200" ]; then
      t1="$(now_ms)"; TURN_RECOVERY_MS=$((t1 - t0)); return 0
    fi
    if [ "$TURN_STATUS" = "503" ] && echo "$TURN_BODY" | grep -q 'host_waking'; then
      log "attempt ${attempt}: host_waking (retryable) -- re-issuing within wake-hold deadline"
      sleep 3; continue
    fi
    echo "turn returned non-retryable HTTP ${TURN_STATUS}: ${TURN_BODY}" >&2; return 1
  done
  echo "wake-and-hold deadline (${WAKE_HOLD_DEADLINE}s) exceeded; last status=${TURN_STATUS} body=${TURN_BODY}" >&2
  return 1
}

WAKE_STATUS=""; WAKE_BODY=""

# The control-api /rpc facade requires the rpc-proxy PLANE inter-service token
# (requireInternalService) plus the user RPC JWT in x-rpc-access-token — the
# exact contract rpc-proxy's controlApiHeaders uses. Read the plane token from
# the Secret; missing key/entry fails loud.
RPC_PLANE_SERVICE_TOKEN=""
resolve_rpc_plane_service_token() {
  [ -n "$RPC_PLANE_SERVICE_TOKEN" ] && return 0
  local pairs
  pairs=$(kctl -n control-plane get secret control-api-internal-tokens \
    -o jsonpath='{.data.CONTROL_API_INTERNAL_SERVICE_TOKENS}' | base64 -d) || {
    echo "cannot read Secret control-plane/control-api-internal-tokens" >&2; return 1; }
  RPC_PLANE_SERVICE_TOKEN=$(printf '%s' "$pairs" | tr ',' '\n' | sed -n 's/^rpc-proxy=//p' | head -1)
  [ -n "$RPC_PLANE_SERVICE_TOKEN" ] || {
    echo "no rpc-proxy entry in CONTROL_API_INTERNAL_SERVICE_TOKENS" >&2; return 1; }
}

post_wake() {
  local resp
  mint_rpc_token || return 1
  resolve_rpc_plane_service_token || return 1
  resp=$(curl -sS -m 30 -w '\n%{http_code}' -X POST \
    "${WAKE_BASE}/api/v1/rpc/hosts/${HOST_REF}/wake" \
    -H "Authorization: Bearer ${RPC_PLANE_SERVICE_TOKEN}" \
    -H "x-service-token: rpc-proxy" \
    -H "x-rpc-access-token: ${RPC_TOKEN}" -H 'Content-Type: application/json') || {
    echo "wake POST failed at ${WAKE_BASE}" >&2; return 1; }
  WAKE_STATUS="$(echo "$resp" | tail -n1)"; WAKE_BODY="$(echo "$resp" | sed '$d')"
}

# --- Candidate cadence (labeled precondition; restored in trap) --------
save_hcc_env_once() {
  [ -z "$HCC_ENV_SAVED" ] || return 0
  local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES \
    CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS)
  local k cur saved=""
  for k in "${keys[@]}"; do
    cur="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
      -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}" 2>/dev/null || true)"
    saved+="${k}=${cur}"$'\n'
  done
  HCC_ENV_SAVED="$saved"
}

apply_candidate_cadence() {
  local t_min=$1
  header "Precondition: T_idle candidate = ${t_min} min (HCC env overlay; floor=${TEST_IDLE_FLOOR_MINUTES}, drain=${TEST_DRAIN_GRACE_MS}ms, poll=${TEST_POLL_MS}ms)"
  save_hcc_env_once
  kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
    "CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=${t_min}" \
    "CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=${TEST_IDLE_FLOOR_MINUTES}" \
    "CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=${TEST_DRAIN_GRACE_MS}" \
    "CONTEXT_MAPPER_HEARTBEAT_POLL_MS=${TEST_POLL_MS}" >/dev/null 2>&1 || {
    fail "failed to set candidate T_idle=${t_min}min via kubectl set env on ${HCC_DEPLOY}"; return 1; }
  if kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null 2>&1; then
    ok "HCC running with T_idle=${t_min}min (T_idle = max(idleMinutes=${t_min}, floor=${TEST_IDLE_FLOOR_MINUTES}) = ${t_min}min)"
  else
    fail "HCC rollout did not settle after cadence set for T=${t_min}min"; pod_diagnostics; return 1
  fi
}

# --- Pod-uptime sampler (cost proxy) -----------------------------------
start_uptime_sampler() {
  UPTIME_SAMPLES_FILE="$(mktemp)"
  (
    while :; do
      n="$(kctl get pods -n "$MCP_HOST_NS" -l "app=${HOST_REF}" \
        --field-selector=status.phase=Running -o name 2>/dev/null | wc -l | tr -d ' ')"
      if [ "${n:-0}" -gt 0 ]; then echo 1; else echo 0; fi >> "$UPTIME_SAMPLES_FILE"
      sleep "$UPTIME_SAMPLE_INTERVAL"
    done
  ) &
  UPTIME_PID=$!
}

finish_uptime_sample() {
  local t_min=$1 wall_s=$2 up_samples uptime_s
  stop_uptime_sampler
  up_samples="$(awk '$1==1{c++} END{print c+0}' "$UPTIME_SAMPLES_FILE")"
  uptime_s=$((up_samples * UPTIME_SAMPLE_INTERVAL))
  jq -cn --argjson t "$t_min" --argjson u "$uptime_s" --argjson w "$wall_s" \
    '{candidate_min:$t, pod_uptime_seconds:$u, wall_seconds:$w}' >> "$UPTIME_SUMMARY_FILE"
  rm -f "$UPTIME_SAMPLES_FILE"; UPTIME_SAMPLES_FILE=""
}

record_event() {
  # args: candidate_min event multiplier gap_s outcome recovery_ms suspended_during_gap
  jq -cn --argjson t "$1" --argjson i "$2" --arg g "$3" --argjson gs "$4" \
    --arg o "$5" --argjson r "$6" --argjson s "$7" \
    '{candidate_min:$t, event:$i, multiplier:($g|tonumber), gap_seconds:$gs,
      outcome:$o, recovery_ms:$r, suspended_during_gap:($s==1)}' >> "$RECORDS_FILE"
}

# --- Per-candidate synthetic session trace ------------------------------
run_candidate() {
  local t_min=$1
  local -a mults
  read -r -a mults <<< "$GAP_MULTIPLIERS"
  local nm=${#mults[@]}

  apply_candidate_cadence "$t_min" || { print_results; exit 1; }

  # Make sure the trace starts from a served, active host.
  if [ "$(lifecycle_state)" = "suspended" ]; then
    post_wake || { fail "candidate T=${t_min}: wake POST failed reviving suspended host"; print_results; exit 1; }
    case "$WAKE_STATUS" in 200|202) : ;; *) fail "candidate T=${t_min}: wake returned HTTP ${WAKE_STATUS}: ${WAKE_BODY}"; print_results; exit 1 ;; esac
  fi
  wait_for_ready_pod "$POD_READY_TIMEOUT" >/dev/null || {
    fail "candidate T=${t_min}: no Ready pod for app=${HOST_REF} before the trace"; pod_diagnostics; print_results; exit 1; }

  start_uptime_sampler
  local cand_start; cand_start=$(date +%s)

  header "Candidate T=${t_min}min -- baseline turn"
  if send_turn_expect_ok "Reply with exactly: IDLECAL_BASE_${t_min}_${RUN_ID}"; then
    ok "baseline turn served in ${TURN_RECOVERY_MS}ms"
  else
    fail "candidate T=${t_min}: baseline turn failed (guardian (c): timeout/transport/non-retryable are hard failures)"
    pod_diagnostics; print_results; exit 1
  fi
  local uid_last; uid_last="$(pod_uid || true)"
  [ -n "$uid_last" ] || { fail "candidate T=${t_min}: cannot resolve pod UID after baseline turn"; pod_diagnostics; print_results; exit 1; }

  local i g gap_s is_supra suspend_deadline gap_elapsed st suspended_during_gap
  local state_at_send outcome uid_now remaining gap_kind
  for (( i=1; i<=EVENTS_PER_CANDIDATE; i++ )); do
    g="${mults[$(( (i-1) % nm ))]}"
    gap_s="$(awk -v g="$g" -v t="$t_min" 'BEGIN{ printf "%d", (g*t*60)+0.5 }')"
    is_supra="$(awk -v g="$g" 'BEGIN{ print (g>1) ? 1 : 0 }')"
    suspended_during_gap=0
    if [ "$is_supra" = "1" ]; then gap_kind="supra"; else gap_kind="sub"; fi
    header "Candidate T=${t_min}min -- event ${i}/${EVENTS_PER_CANDIDATE}: gap ${g}xT = ${gap_s}s (${gap_kind}-threshold)"

    gap_elapsed=0
    if [ "$is_supra" = "0" ]; then
      # (a) Sub-threshold gap: the host must NEVER suspend during it.
      st=""
      while [ "$gap_elapsed" -lt "$gap_s" ]; do
        st="$(lifecycle_state)"
        if [ "$st" = "suspended" ]; then
          fail "GUARDIAN (a): sub-threshold gap (${g}xT=${gap_s}s < T=${t_min}min) produced state=suspended at +${gap_elapsed}s -- T_idle NOT honored (product bug, not tuning noise)"
          pod_diagnostics; print_results; exit 1
        fi
        sleep "$POLL_INTERVAL"; gap_elapsed=$((gap_elapsed + POLL_INTERVAL))
      done
      ok "sub-threshold gap held without suspend (last state='${st:-active}')"
    else
      # (b) Supra-threshold gap: the host MUST suspend within
      # T*60 + SUSPEND_DETECT_SLACK of the last activity.
      suspend_deadline=$(( t_min * 60 + SUSPEND_DETECT_SLACK ))
      while :; do
        st="$(lifecycle_state)"
        if [ "$st" = "suspended" ] && [ "$(deployment_replicas)" = "0" ]; then
          suspended_during_gap=1
          ok "suspended at +${gap_elapsed}s (bound: T+slack=${suspend_deadline}s)"
          break
        fi
        if [ "$gap_elapsed" -ge "$suspend_deadline" ]; then
          fail "GUARDIAN (b): supra-threshold gap (${g}xT) never suspended within T+slack=${suspend_deadline}s (state='${st:-<empty>}' replicas=$(deployment_replicas))"
          pod_diagnostics; print_results; exit 1
        fi
        sleep "$POLL_INTERVAL"; gap_elapsed=$((gap_elapsed + POLL_INTERVAL))
      done
      remaining=$(( gap_s - gap_elapsed ))
      if [ "$remaining" -gt 0 ]; then
        log "holding remaining ${remaining}s to complete the ${g}xT gap"
        sleep "$remaining"
      fi
    fi

    # Resume with the next turn; classify by lifecycle state at send.
    state_at_send="$(lifecycle_state)"
    if send_turn_expect_ok "Reply with exactly: IDLECAL_${t_min}_${i}_${RUN_ID}"; then
      :
    else
      fail "GUARDIAN (c): resume turn for event ${i} (state_at_send='${state_at_send:-<empty>}') did not reach a definitive 200 (timeout / transport / non-retryable)"
      pod_diagnostics; print_results; exit 1
    fi

    wait_for_ready_pod "$POD_READY_TIMEOUT" >/dev/null || {
      fail "event ${i}: no Ready pod after a served resume turn"; pod_diagnostics; print_results; exit 1; }
    uid_now="$(pod_uid || true)"
    [ -n "$uid_now" ] || { fail "event ${i}: cannot resolve pod UID after resume"; pod_diagnostics; print_results; exit 1; }

    case "$state_at_send" in
      suspended)
        outcome="cold_wake"
        if [ "$uid_now" = "$uid_last" ]; then
          fail "event ${i}: state was suspended at send but pod UID did not change (${uid_now}) -- wake did not replace the pod"
          pod_diagnostics; print_results; exit 1
        fi ;;
      draining)
        outcome="warm_cancel"
        if [ "$uid_now" != "$uid_last" ]; then
          fail "event ${i}: cancel-drain resume changed the pod UID (${uid_last} -> ${uid_now}) -- expected the same pod"
          pod_diagnostics; print_results; exit 1
        fi ;;
      *)
        outcome="warm_active"
        if [ "$uid_now" != "$uid_last" ]; then
          fail "event ${i}: active-state resume changed the pod UID (${uid_last} -> ${uid_now}) -- unexplained pod restart"
          pod_diagnostics; print_results; exit 1
        fi ;;
    esac
    ok "event ${i}: outcome=${outcome} recovery=${TURN_RECOVERY_MS}ms"
    record_event "$t_min" "$i" "$g" "$gap_s" "$outcome" "$TURN_RECOVERY_MS" "$suspended_during_gap"
    uid_last="$uid_now"
  done

  finish_uptime_sample "$t_min" "$(( $(date +%s) - cand_start ))"
}

# ====================================================================== #
#  Prerequisites
# ====================================================================== #
header "Prerequisites"
for bin in jq curl python3; do
  command -v "$bin" >/dev/null 2>&1 || { fail "required tool '${bin}' not installed"; print_results; exit 1; }
done

stateless_flag=$(kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.spec.lifecycle.stateless}' 2>/dev/null || echo "")
if [ "$stateless_flag" = "true" ]; then
  ok "Host '${HOST_REF}' exists with spec.lifecycle.stateless=true"
else
  fail "Host '${HOST_REF}' missing or spec.lifecycle.stateless != true (got '${stateless_flag:-<absent>}'). Seed: bash scripts/e2e/seed-stateless-host.sh"
  print_results; exit 1
fi

for svc in "external-rest-api ${EXT_BASE}" "rpc-proxy ${RPC_BASE}"; do
  name="${svc%% *}"; base="${svc##* }"
  if curl -fsS -m 10 "${base}/health" >/dev/null 2>&1; then
    ok "${name} reachable at ${base}"
  else
    fail "${name} NOT reachable at ${base} -- start port-forwards (make minikube-pf-all)"; print_results; exit 1
  fi
done

if mint_session_token; then ok "session token minted for ${DEV_EMAIL}"; else
  fail "cannot login as ${DEV_EMAIL} on ${EXT_BASE}"; print_results; exit 1; fi
if mint_rpc_token; then ok "user RPC token minted (host:wake:write among scopes)"; else
  fail "cannot mint RPC token for hostRef=${HOST_REF} -- is ${DEV_EMAIL} associated to the stateless host?"; print_results; exit 1; fi

EST_MIN="$(awk -v cands="$IDLE_CANDIDATES_MIN" -v mults="$GAP_MULTIPLIERS" -v ev="$EVENTS_PER_CANDIDATE" 'BEGIN{
  nc=split(cands, C, " "); nm=split(mults, M, " "); total=0
  for (i=1; i<=nc; i++) { s=0; for (e=0; e<ev; e++) s+=M[(e%nm)+1]; total+=s*C[i] }
  printf "%.1f", total + nc*2.5 + 2   # +2.5 min/candidate (HCC rollout + resumes) + 2 min prereq
}')"
log "Estimated wall time for this matrix: ~${EST_MIN} min (gap sleep + per-candidate overhead)"

# ====================================================================== #
#  Sweep
# ====================================================================== #
for T in $IDLE_CANDIDATES_MIN; do
  run_candidate "$T"
done

# ====================================================================== #
#  Aggregation + recommendation
# ====================================================================== #
header "Aggregation"
event_count="$(wc -l < "$RECORDS_FILE" | tr -d ' ')"
expected_count=0
for T in $IDLE_CANDIDATES_MIN; do expected_count=$((expected_count + EVENTS_PER_CANDIDATE)); done
if [ "$event_count" != "$expected_count" ]; then
  fail "aggregation refused: recorded ${event_count} events, expected ${expected_count} -- a missing record means an unmeasured resume"
  print_results; exit 1
fi

jq -s \
  --slurpfile uptime "$UPTIME_SUMMARY_FILE" \
  --argjson threshold "$DISRUPTION_THRESHOLD" \
  --arg gap_multipliers "$GAP_MULTIPLIERS" \
  --arg run_id "$RUN_ID" '
  def p95: sort | if length == 0 then null else .[(((length * 0.95) | ceil) - 1)] end;
  (group_by(.candidate_min) | map(
    . as $ev |
    ($uptime | map(select(.candidate_min == $ev[0].candidate_min)) | first) as $up |
    {
      candidate_min: $ev[0].candidate_min,
      resumes: ($ev | length),
      suspend_count: ($ev | map(select(.suspended_during_gap)) | length),
      cold_wake_count: ($ev | map(select(.outcome == "cold_wake")) | length),
      warm_cancel_count: ($ev | map(select(.outcome == "warm_cancel")) | length),
      warm_active_count: ($ev | map(select(.outcome == "warm_active")) | length),
      disruption_rate: (($ev | map(select(.outcome == "cold_wake")) | length) / ($ev | length)),
      p95_recovery_ms: {
        cold_wake:   ($ev | map(select(.outcome == "cold_wake")   | .recovery_ms) | p95),
        warm_cancel: ($ev | map(select(.outcome == "warm_cancel") | .recovery_ms) | p95),
        warm_active: ($ev | map(select(.outcome == "warm_active") | .recovery_ms) | p95)
      },
      pod_uptime_seconds: ($up | if . == null then null else .pod_uptime_seconds end),
      wall_seconds:       ($up | if . == null then null else .wall_seconds end),
      uptime_ratio:       ($up | if . == null or .wall_seconds == 0 then null
                                 else (.pod_uptime_seconds / .wall_seconds) end),
      events: $ev
    })) as $cands |
  ($cands | map(select(.disruption_rate <= $threshold)) | sort_by(.candidate_min) | first) as $pick |
  {
    run_id: $run_id,
    scope: "compressed-local",
    gap_multipliers: $gap_multipliers,
    candidates: $cands,
    recommendation: {
      scope: "compressed-local",
      decision_rule: "smallest T whose disruption_rate <= DISRUPTION_THRESHOLD, on the measured data",
      disruption_threshold: $threshold,
      chosen_candidate_min: ($pick | if . == null then null else .candidate_min end),
      note: (if $pick == null then
        "No candidate met the threshold at compressed scale. EXPECTED with the default multiplier mix: gaps are defined relative to each T, so every candidate sees the same supra/sub ratio and disruption_rate is fixed by GAP_MULTIPLIERS, by construction. The sweep validates threshold mechanics and per-class recovery latency; it does not pick the production constant."
      else
        "Chosen on measured compressed-local data only; do NOT promote to production without the transfer step."
      end),
      production_transfer_rule: "T_idle* ~= the p75-p90 quantile of the PRODUCTION inter-message gap distribution per Host class, bounded below by CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES and above by cost tolerance. Measure gaps as deltas of host_heartbeats.last_activity_ts in control-plane Postgres. The 30-min hypothesis is the right starting default iff ~75-90% of session gaps are under 30 min. See docs/testing/stateless-idle-calibration.md."
    }
  }' "$RECORDS_FILE" > "$OUT_JSON" || { fail "jq aggregation failed"; print_results; exit 1; }

ok "calibration JSON written to ${OUT_JSON}"
jq . "$OUT_JSON"

print_results