Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
29.0 KB
Raw Blame
#!/usr/bin/env bash
# Branch-owned live gate for #825: one storage layout across lifecycle switches.
#
# Every Host mounts its workspace PVC root at the workspace path and keeps its
# conversation database there (CLERUM_SESSION_DB_DIR=<workspacePath>), whatever
# its lifecycle. A `workspace-layout-merge` init container moves a PVC split by
# the earlier stateless template back to the root before mcp-host starts.
#
# Each scenario creates its own Hosts through the control-api admin API and
# changes them only through that API. The trap deletes them. Named fixtures
# (logged as FAULT:<name>) are the only other cluster writes:
#   - hcc-cadence: shortens the HCC idle window so stateless Hosts suspend in
#     about a minute (opt-in E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1),
#     restored in the trap;
#   - pvc-fixture: a short-lived Pod (the Host image, uid 1001, no token) that
#     mounts a Host PVC to list it or, on a suspended Host, to seed a tree;
#   - channel: a CommunicationChannel CR that makes HCC run a stateless Host
#     as stateful;
#   - host-layout-merge: S5 and S6 run deploy/scripts/host-layout-merge.sh,
#     whose check Pods mount Host PVCs read-only and are deleted afterwards.
# No kubectl exec into Host Pods, no SQLite writes, no apply/patch of Hosts or
# their Deployments.
#
# Scenarios: S1-S7 (plan A7.10), RACE (A7.13), NEG_RV (stale resourceVersion),
# NEG_CTRL (the history and chatId-set oracles reject an empty catalog). After
# every transition the Host must list the same non-empty chatId set as before
# it and open the same database (its "[ConversationStore] opened" log line);
# each check prints E2E_825_CONTINUITY. The final line is
# E2E_825_RESULT=PASS only when every defined scenario ran and passed.
set -euo pipefail
umask 077

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
# shellcheck source=scripts/e2e/load-dotenv.sh
source "${REPO_ROOT}/scripts/e2e/load-dotenv.sh"
dotenv_load_canonical_root "${REPO_ROOT}"
# shellcheck source=scripts/e2e/admin-credentials.sh
source "${REPO_ROOT}/scripts/e2e/admin-credentials.sh"
# shellcheck source=scripts/e2e/e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"

die() { fail "$*"; exit 1; }

# ─── Preconditions ────────────────────────────────────────────────────
if [ -z "$E2E_KUBECONTEXT" ] || ! is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"; then
  die 'select an explicit branch-scoped Minikube context (KUBECONTEXT=<profile>)'
fi
require_safe_kube_context
[ "${E2E_ALLOW_STATELESS_CADENCE_ACCELERATION:-0}" = 1 ] ||
  die 'set E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 (S4-S7 shorten the HCC idle window on this owned profile)'
[ -n "${E2E_PROFILE_PORTS_ENV:-}" ] && [ -r "$E2E_PROFILE_PORTS_ENV" ] ||
  die 'provide E2E_PROFILE_PORTS_ENV from the owned branch profile'
[ -n "${E2E_DEV_LOGIN_EMAIL:-}" ] || die 'set E2E_DEV_LOGIN_EMAIL to the seeded login ([email protected] on a T2 profile)'
profile_port_value() {
  awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "$E2E_PROFILE_PORTS_ENV"
}
CONTROL_API_BASE_URL="$(profile_port_value CONTROL_API_URL)"
CONTROL_UI_BASE_URL="$(profile_port_value CONTROL_UI_URL)"
EXTERNAL_REST_API_BASE_URL="$(profile_port_value EXTERNAL_REST_API_URL)"
RPC_PROXY_BASE_URL="$(profile_port_value RPC_PROXY_URL)"
export CONTROL_API_BASE_URL CONTROL_UI_BASE_URL EXTERNAL_REST_API_BASE_URL RPC_PROXY_BASE_URL
require_branch_profile_urls "$E2E_KUBECONTEXT" "$E2E_PROFILE_PORTS_ENV" ||
  die 'profile URLs do not match the owned ports'
kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \
  'any(.items[]; .metadata.labels["minikube.k8s.io/name"]==$c)' >/dev/null ||
  die 'target is not the selected Minikube profile'

HOST_NS=mcp-host CHANNEL_NS=channels
HCC_NS=control-plane HCC_DEPLOY=host-context-controller
SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm}"
CAPI="${CONTROL_API_BASE_URL%/}/api/v1"
EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"
RPC_BASE="${RPC_PROXY_BASE_URL%/}"
ADMIN_USERNAME="${ADMIN_USERNAME:-admin}"
ADMIN_PASSWORD="${ADMIN_PASSWORD:-}"
[ -n "$ADMIN_PASSWORD" ] || ADMIN_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"
[ -n "$ADMIN_PASSWORD" ] || die 'admin password could not be resolved (ADMIN_PASSWORD or the canonical .env)'
DEV_EMAIL="$E2E_DEV_LOGIN_EMAIL"
DEV_PASSWORD="${E2E_USER_PASSWORD:-$ADMIN_PASSWORD}"
# od reads a fixed byte count; `tr </dev/urandom | head` dies of SIGPIPE under pipefail.
RUN="$(od -An -N3 -tx1 /dev/urandom | tr -d ' \n')"
POD_READY_TIMEOUT="${E2E_825_POD_READY_TIMEOUT:-300}"
SUSPEND_TIMEOUT="${E2E_825_SUSPEND_TIMEOUT:-300}"
TURN_TIMEOUT="${E2E_825_TURN_TIMEOUT:-240}"
STORE_LOG_TIMEOUT="${E2E_825_STORE_LOG_TIMEOUT:-30}"
BASE_INIT_FIXTURE="${REPO_ROOT}/host-context-controller/test/__fixtures__/base-workspace-layout-init.sh"
[ -r "$BASE_INIT_FIXTURE" ] || die "missing base init fixture ${BASE_INIT_FIXTURE}"
LAYOUT_SCRIPT="${REPO_ROOT}/deploy/scripts/host-layout-merge.sh"
[ -r "$LAYOUT_SCRIPT" ] || die "missing operator script ${LAYOUT_SCRIPT}"

DEFINED_SCENARIOS=(NEG_CTRL NEG_RV S1 S2 S3 RACE S4 S5 S6 S7)
EXECUTED=0 SCENARIO_FAILURES=0 SCENARIO_LOG=''
CREATED_HOSTS=() CREATED_CHANNELS=()
HCC_ENV_SAVED='' HCC_ENV_CHANGED=0
SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/e2e-825.XXXXXX")"
FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"
: >"$FIXTURE_POD_REGISTRY"
ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID=''
HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()

# shellcheck source=scripts/e2e/e2e-825-lib.sh
source "${SCRIPT_DIR}/e2e-825-lib.sh"

# switch_and_witness NAME STATELESS: request the switch, then require a new
# Pod with the requested lifecycle and the same storage as before.
switch_and_witness() {
  local name=$1 stateless=$2 old=$POD_UID
  set_lifecycle "$name" "$stateless" || return 1
  wait_new_ready_pod "$name" "$old" "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" "$stateless" /workspace
}

# The inventoried dev split shape: DB family and a backup under state/, every
# other root entry under workspace/, plus an agent-made symlink (A7.16).
# shellcheck disable=SC2016
SEED_SPLIT_SCRIPT='set -eu; cd "$1"; mkdir state workspace
for e in * .[!.]*; do
  [ -e "$e" ] || [ -L "$e" ] || continue
  case "$e" in lost+found|state|workspace) continue ;; state.db*) mv "$e" state/ ;; *) mv "$e" workspace/ ;; esac
done
cp state/state.db state/state.db.pre-20261014.bak
ln -s /tmp/e2e-825-link-target workspace/e2e-825-link
ls -la state workspace'
# Two live database families: the root one and a copy under state/ (F2),
# with workspace/ present so the tree carries the split-layout signature and
# a file under each directory the merge would move.
# shellcheck disable=SC2016
SEED_TWO_DB_SCRIPT='set -eu; cd "$1"; mkdir state workspace; [ -f state.db ]
for s in "" -wal -shm; do [ -e "state.db$s" ] || continue; cp "state.db$s" "workspace/state.db$s"; mv "state.db$s" "state/state.db$s"; done
printf "notes\n" > state/notes.txt; printf "x\n" > workspace/x; ls -la . state workspace'

# A new stateless Host with history, a seeded file, then suspended.
suspended_host_with_history() { # NAME
  create_host "$1" true || return 1
  wait_new_ready_pod "$1" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$1" true /workspace || return 1
  seed_history "$1" 2 1 || return 1
  pvc_fixture "$1" rw seed-file "$WRITE_FILE_SCRIPT" "seed ${RUN}" seed.txt >/dev/null || return 1
  wait_suspended "$1"
}

# ─── Scenarios ────────────────────────────────────────────────────────
scenario_NEG_CTRL() {
  local digest set_a empty='{"agent":"x","chatId":"y","totalTurns":0,"turns":[]}'
  local real='{"turns":[{"number":1,"user_input":"a","response":"b"}]}'
  digest="$(history_digest "$real")"
  history_matches "$digest" "$real" || { fail 'NEG_CTRL witness: the oracle rejects a matching history'; return 1; }
  if history_matches "$digest" "$empty"; then fail 'NEG_CTRL: the history oracle accepts an empty catalog'; return 1; fi
  if history_matches "0 $(jq -cS '[]' <<<'null' | shasum -a 256 | cut -d' ' -f1)" "$empty"; then
    fail 'NEG_CTRL: the history oracle accepts zero turns as a match'
    return 1
  fi
  session_listed '{"items":[{"agent":"h","chatId":"t"}]}' h t || { fail 'NEG_CTRL witness: list oracle rejects a listed session'; return 1; }
  if session_listed '{"items":[]}' h t; then fail 'NEG_CTRL: the list oracle accepts an empty catalog'; return 1; fi
  set_a="$(chat_set '{"items":[{"agent":"h","chatId":"b"},{"agent":"h","chatId":"a"},{"agent":"x","chatId":"c"}]}' h)"
  [ "$set_a" = '2:a,b' ] || { fail "NEG_CTRL witness: chat_set returned '${set_a}', expected '2:a,b'"; return 1; }
  chats_match "$set_a" "$set_a" || { fail 'NEG_CTRL witness: the set oracle rejects an equal set'; return 1; }
  if chats_match "$set_a" "$(chat_set '{"items":[{"agent":"h","chatId":"a"}]}' h)"; then
    fail 'NEG_CTRL: the set oracle accepts a lost chat'
    return 1
  fi
  if chats_match "$(chat_set '{"items":[]}' h)" "$(chat_set '{"items":[]}' h)"; then
    fail 'NEG_CTRL: the set oracle accepts an empty set on both sides'
    return 1
  fi
  ok 'NEG_CTRL: oracles reject an empty catalog, a lost chat and empty-on-both-sides, and accept a real one'
}

scenario_NEG_RV() {
  local name rv1 spec
  name="$(truncate_rfc1123 "e2e-825-rv-${RUN}")"
  create_host "$name" false || return 1
  admin GET "${CAPI}/admin/hosts/${name}" || true
  [ "$HTTP_CODE" = 200 ] || { fail "NEG_RV: GET returned ${HTTP_CODE}"; return 1; }
  rv1="$(jq -r '.metadata.resourceVersion' <<<"$HTTP_BODY")"
  spec="$(jq -c '.spec | .lifecycle = {stateless:true}' <<<"$HTTP_BODY")"
  admin PUT "${CAPI}/admin/hosts/${name}" "$(jq -cn --arg rv "$rv1" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
  [ "$HTTP_CODE" = 200 ] || { fail "NEG_RV witness: fresh PUT returned ${HTTP_CODE}"; return 1; }
  spec="$(jq -c '.lifecycle = {stateless:false}' <<<"$spec")"
  admin PUT "${CAPI}/admin/hosts/${name}" "$(jq -cn --arg rv "$rv1" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
  if [ "$HTTP_CODE" != 409 ] || ! jq -e '.reason == "resource_changed"' <<<"$HTTP_BODY" >/dev/null; then
    fail "NEG_RV: stale PUT returned ${HTTP_CODE}: ${HTTP_BODY}"
    return 1
  fi
  [ "$(kctl get host "$name" -n "$HOST_NS" -o jsonpath='{.spec.lifecycle.stateless}')" = true ] ||
    { fail 'NEG_RV: the stale PUT changed the lifecycle'; return 1; }
  ok 'NEG_RV: stale resourceVersion → 409 resource_changed, lifecycle unchanged; fresh PUT applied'
}

# S1: stateful → stateless → stateful, 3 conversations x 2 turns, a file.
scenario_S1() {
  local name
  name="$(truncate_rfc1123 "e2e-825-s1-${RUN}")"
  create_host "$name" false || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" false /workspace || return 1
  seed_history "$name" 3 2 || return 1
  pvc_fixture "$name" rw seed-file "$WRITE_FILE_SCRIPT" "s1 ${RUN}" seed.txt >/dev/null || return 1
  switch_and_witness "$name" true || return 1
  assert_history "$name" 'stateful->stateless' || return 1
  read_seed_file "$name" seed.txt "s1 ${RUN}" || return 1
  switch_and_witness "$name" false || return 1
  assert_history "$name" 'stateless->stateful' || return 1
  read_seed_file "$name" seed.txt "s1 ${RUN}"
}

# S2: created stateless → stateful → stateless.
scenario_S2() {
  local name
  name="$(truncate_rfc1123 "e2e-825-s2-${RUN}")"
  create_host "$name" true || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" true /workspace || return 1
  seed_history "$name" 2 2 || return 1
  switch_and_witness "$name" false || return 1
  assert_history "$name" 'stateless->stateful' || return 1
  switch_and_witness "$name" true || return 1
  assert_history "$name" 'stateful->stateless'
}

# S3: a CommunicationChannel makes HCC run the stateless Host as stateful;
# deleting it returns the Host to stateless.
scenario_S3() {
  local name channel old reason elapsed=0
  name="$(truncate_rfc1123 "e2e-825-s3-${RUN}")"
  channel="$(truncate_rfc1123 "e2e-825-s3-${RUN}")"
  create_host "$name" true || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" true /workspace || return 1
  seed_history "$name" 2 1 || return 1
  log "FAULT:channel ${CHANNEL_NS}/${channel} → ${name}"
  printf '%s\n' 'apiVersion: clerum.io/v1alpha1' 'kind: CommunicationChannel' 'metadata:' \
    "  name: ${channel}" "  namespace: ${CHANNEL_NS}" 'spec:' "  hostRef: ${name}" '  telegram:' \
    "    - channelId: \"e2e-825-${RUN}\"" '      chatType: "private"' '      userIds: ["000000000"]' |
    kctl create -f - >/dev/null || { fail 'S3: CommunicationChannel create failed'; return 1; }
  CREATED_CHANNELS+=("$channel")
  old=$POD_UID
  wait_new_ready_pod "$name" "$old" "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" false /workspace || return 1
  reason="$(kctl get host "$name" -n "$HOST_NS" -o jsonpath='{.status.conditions[?(@.type=="StatelessEnableRejected")].status}')"
  [ "$reason" = True ] || { fail "S3: StatelessEnableRejected is '${reason}', expected True"; return 1; }
  assert_history "$name" 'stateless->stateful(channel)' || return 1
  kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --wait=true --timeout=60s >/dev/null ||
    { fail 'S3: CommunicationChannel delete failed'; return 1; }
  old=$POD_UID
  wait_new_ready_pod "$name" "$old" "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" true /workspace || return 1
  assert_history "$name" 'stateful->stateless(channel-deleted)'
}

# S4: switch to stateful while suspended, then back to stateless (history
# checked on the active Pod), suspend and wake.
scenario_S4() {
  local name
  name="$(truncate_rfc1123 "e2e-825-s4-${RUN}")"
  suspended_host_with_history "$name" || return 1
  switch_and_witness "$name" false || return 1
  assert_history "$name" 'suspended->stateful' || return 1
  read_seed_file "$name" seed.txt "seed ${RUN}" || return 1
  switch_and_witness "$name" true || return 1
  # The active stateless Pod between the switch and the suspend serves the
  # same history too, not only the Pod that wakes afterwards.
  assert_history "$name" 'stateful->stateless(active)' || return 1
  wait_suspended "$name" || return 1
  post_wake "$name" || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" true /workspace || return 1
  assert_history "$name" 'stateless-suspend->wake'
}

# S5: the inventoried split tree (state/ with a backup, workspace/ with a
# symlink) merges on wake; history, file, backup and link at the root.
scenario_S5() {
  local name listing row
  name="$(truncate_rfc1123 "e2e-825-s5-${RUN}")"
  suspended_host_with_history "$name" || return 1
  pvc_fixture "$name" rw seed-split "$SEED_SPLIT_SCRIPT" >/dev/null || return 1
  listing="$(pvc_fixture "$name" ro list-split "$LIST_SCRIPT")" || return 1
  if ! grep -qx 'd ./state' <<<"$listing" || ! grep -qx 'f ./state/state.db'<<<"$listing"; then
    fail "S5: seeded tree is not split: ${listing}"
    return 1
  fi
  row="$(layout_row inventory "$name" '0 1')" || return 1
  [ "$row" = WILL_MERGE ] || { fail "S5: inventory reads ${name}-workspace as '${row}', expected WILL_MERGE"; return 1; }
  # A split tree is a blocker for turning the flag off, whatever the rest of the fleet.
  row="$(layout_row verify "$name" 1)" || return 1
  [ "$row" = SPLIT ] || { fail "S5: verify reads ${name}-workspace as '${row}', expected SPLIT"; return 1; }
  ok "S5: host-layout-merge.sh reads the seeded tree as WILL_MERGE (inventory) and SPLIT (verify)"
  post_wake "$name" || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" true /workspace || return 1
  listing="$(pvc_fixture "$name" ro list-merged "$LIST_SCRIPT")" || return 1
  if grep -Eq '^d \./(state|workspace)$' <<<"$listing"; then fail "S5: split directories remain: ${listing}"; return 1; fi
  if ! grep -qx 'f ./state.db'<<<"$listing" ||
    ! grep -qx 'f ./state.db.pre-20261014.bak'<<<"$listing" ||
    ! grep -qx 'l ./e2e-825-link /tmp/e2e-825-link-target' <<<"$listing"; then
    fail "S5: merged tree lacks the database, backup or link at the root: ${listing}"
    return 1
  fi
  ok "S5: split tree merged to the root"
  row="$(layout_row verify "$name" '0 1')" || return 1
  [ "$row" = POD_OK ] || { fail "S5: verify reads ${name}-workspace as '${row}' after the merge, expected POD_OK"; return 1; }
  ok "S5: host-layout-merge.sh verify reads the merged Host as POD_OK"
  assert_history "$name" 'suspend->wake(split-tree-merged)' || return 1
  read_seed_file "$name" seed.txt "seed ${RUN}"
}

# S6: two live database families, under state/ and workspace/ with none at
# the root (a T5 tree with a root database is superseded, not refused): the
# merge refuses, the Pod stays in Init:Error and the tree is unchanged.
scenario_S6() {
  local name before after list_before list_after elapsed=0 pod logs row
  local f2='workspace layout merge refused: more than one live SQLite database across'
  name="$(truncate_rfc1123 "e2e-825-s6-${RUN}")"
  suspended_host_with_history "$name" || return 1
  pvc_fixture "$name" rw seed-two-db "$SEED_TWO_DB_SCRIPT" >/dev/null || return 1
  before="$(pvc_fixture "$name" ro hash-before "$HASH_SCRIPT")" || return 1
  list_before="$(pvc_fixture "$name" ro list-before "$LIST_SCRIPT")" || return 1
  grep -q ' ./state/state.db$' <<<"$before" || { fail "S6 witness: state/state.db not seeded: ${before}"; return 1; }
  grep -q ' ./workspace/state.db$' <<<"$before" || { fail "S6 witness: workspace/state.db not seeded: ${before}"; return 1; }
  ! grep -qx 'f ./state.db' <<<"$list_before" || { fail "S6 witness: a root state.db remains: ${list_before}"; return 1; }
  if ! grep -qx 'f ./state/notes.txt' <<<"$list_before" || ! grep -qx 'f ./workspace/x' <<<"$list_before"; then
    fail "S6 witness: state/notes.txt or workspace/x not seeded: ${list_before}"
    return 1
  fi
  # inventory must name the seeded tree as an upgrade blocker before the merge
  # runs. Its
  # check Pod runs between the two hashes, so the comparison below also
  # proves it changed nothing.
  row="$(layout_row inventory "$name" 1)" || return 1
  case "$row" in "WILL_REFUSE_F2 ${f2}"*) ;; *) fail "S6: inventory reads ${name}-workspace as '${row}', expected WILL_REFUSE_F2"; return 1 ;; esac
  ok "S6: host-layout-merge.sh inventory predicts the F2 refusal"
  post_wake "$name" || return 1
  while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do
    pod="$(host_pods "$name" | jq -r '[.items[] | select(any(.status.initContainerStatuses[]?;
      .name == "workspace-layout-merge" and ((.state.terminated.exitCode // 0) == 1 or (.lastState.terminated.exitCode // 0) == 1)))][0].metadata.name // ""')"
    [ -n "$pod" ] && break
    sleep 3
    elapsed=$((elapsed + 3))
  done
  [ -n "$pod" ] || { fail "S6: no Pod with workspace-layout-merge exit 1 within ${POD_READY_TIMEOUT}s"; return 1; }
  logs="$(kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge --previous 2>/dev/null ||
    kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge)"
  grep -q "$f2" <<<"$logs" || { fail "S6: merge refused without the F2 line: ${logs}"; return 1; }
  row="$(layout_row verify "$name" 1)" || return 1
  case "$row" in "MERGE_REFUSED ${f2}"*) ;; *) fail "S6: verify reads ${name}-workspace as '${row}', expected MERGE_REFUSED with the F2 line"; return 1 ;; esac
  ok "S6: host-layout-merge.sh verify reports MERGE_REFUSED with the F2 line"
  if host_pods "$name" | jq -e 'any(.items[]; any(.status.conditions[]?; .type == "Ready" and .status == "True"))' >/dev/null; then
    fail 'S6: a Pod became Ready on a refused tree'
    return 1
  fi
  after="$(pvc_fixture "$name" ro hash-after "$HASH_SCRIPT")" || return 1
  [ "$before" = "$after" ] || { fail "S6: tree changed after the refusal"; diff <(echo "$before") <(echo "$after") >&2 || true; return 1; }
  list_after="$(pvc_fixture "$name" ro list-after "$LIST_SCRIPT")" || return 1
  [ "$list_before" = "$list_after" ] ||
    { fail "S6: tree layout changed after the refusal"; diff <(echo "$list_before") <(echo "$list_after") >&2 || true; return 1; }
  ok 'S6: two live families → Init:Error with the F2 line, tree unchanged'
}

# S7: a merged tree re-split by the previous init (an HCC rollback), then
# woken: the merge restores the root layout and the history.
scenario_S7() {
  local name before listing script
  name="$(truncate_rfc1123 "e2e-825-s7-${RUN}")"
  suspended_host_with_history "$name" || return 1
  before="$(pvc_fixture "$name" ro list-before "$LIST_SCRIPT")" || return 1
  script="$(cat "$BASE_INIT_FIXTURE")"
  pvc_fixture "$name" rw base-init "$script" >/dev/null || return 1
  listing="$(pvc_fixture "$name" ro list-split "$LIST_SCRIPT")" || return 1
  if ! grep -qx 'd ./state' <<<"$listing" || ! grep -qx 'd ./workspace' <<<"$listing"; then
    fail "S7: the previous init did not split the tree: ${listing}"
    return 1
  fi
  post_wake "$name" || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" true /workspace || return 1
  listing="$(pvc_fixture "$name" ro list-after "$LIST_SCRIPT")" || return 1
  # Only the live writer's journals may differ: the suspended Host closed its
  # database (no -wal/-shm), the woken one holds it open. state.db itself and
  # any state.db*.bak must match, so a lost or extra backup fails here.
  grep -qx 'f ./state.db' <<<"$listing" || { fail "S7: no state.db at the PVC root after the merge: ${listing}"; return 1; }
  [ "$(grep -vxE 'f \./state\.db-(wal|shm)' <<<"$before")" = "$(grep -vxE 'f \./state\.db-(wal|shm)' <<<"$listing")" ] ||
    { fail 'S7: root entries differ from the tree before the re-split'; diff <(echo "$before") <(echo "$listing") >&2 || true; return 1; }
  ok 'S7: re-split tree merged back to the same root entries'
  assert_history "$name" 'suspend->wake(re-split-tree-merged)' || return 1
  read_seed_file "$name" seed.txt "seed ${RUN}"
}

# RACE: concurrent model selections on new chats create one session each and
# never restart the Host (A7.13).
race_host() { # NAME STATELESS
  local name=$1 stateless=$2 model uid restarts round i chat codes pids pods rpc_header
  create_host "$name" "$stateless" || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  assert_pod "$name" "$stateless" /workspace || return 1
  uid=$POD_UID
  restarts="$(jq -r '[.status.containerStatuses[] | select(.name == "mcp-host") | .restartCount] | first' <<<"$POD_JSON")"
  rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${name}/models" || true
  model="$(jq -er '.hostDefault' <<<"$HTTP_BODY")" || { fail "${name}: models returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
  rpc_header="$(auth_header rpc "Authorization: Bearer ${RPC_TOKEN}")"
  for round in 1 2 3 4 5; do
    chat="e2e-825-race-${RUN}-${round}"
    pids=()
    for i in 1 2 3 4; do
      curl -sS -m 60 -o /dev/null -w '%{http_code}\n' -X POST "${RPC_BASE}/api/v1/rpc/hosts/${name}/model" \
        -H "@${rpc_header}" -H 'Content-Type: application/json' \
        -d "$(jq -cn --arg c "$chat" --arg m "$model" '{chatId:$c,model:$m}')" >"${SCRATCH}/race-${round}-${i}" &
      pids+=("$!")
    done
    for i in "${pids[@]}"; do wait "$i" || true; done
    codes="$(cat "${SCRATCH}"/race-"${round}"-* | sort | uniq -c | tr -s ' ' | tr '\n' ';')"
    [ "$(cat "${SCRATCH}"/race-"${round}"-* | grep -cx 200)" = 4 ] ||
      { fail "${name}: round ${round} model selections returned ${codes}"; return 1; }
    send_turn "$name" "$chat" "e2e-825 ${RUN} race ${round}: reply with one short sentence." || return 1
    rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${name}/sessions?agent=${name}&limit=100" || true
    session_listed "$HTTP_BODY" "$name" "$chat" || { fail "${name}: round ${round} chat not listed exactly once"; return 1; }
  done
  # The original Pod must be the only Pod, not terminating: a replacement
  # leaves the old Pod Terminating next to a new one, and items[0] could be
  # either of them.
  pods="$(host_pods "$name")" || { fail "${name}: could not list Pods after the race"; return 1; }
  POD_JSON="$(jq -c --arg uid "$uid" '[.items[] | select(.metadata.uid == $uid and .metadata.deletionTimestamp == null)] | first // empty' <<<"$pods")"
  [ "$(jq '.items | length' <<<"$pods")" = 1 ] && [ -n "$POD_JSON" ] &&
    [ "$(jq -r '[.status.containerStatuses[] | select(.name == "mcp-host") | .restartCount] | first' <<<"$POD_JSON")" = "$restarts" ] ||
    { fail "${name}: the Pod restarted or was replaced during the race: $(jq -c '[.items[] | {uid: .metadata.uid, deleting: (.metadata.deletionTimestamp != null)}]' <<<"$pods")"; return 1; }
  ok "${name}: 5 rounds x 4 concurrent selections, one session per chat, Pod ${uid:0:8} restartCount ${restarts}"
}
scenario_RACE() {
  race_host "$(truncate_rfc1123 "e2e-825-race-sf-${RUN}")" false || return 1
  race_host "$(truncate_rfc1123 "e2e-825-race-sl-${RUN}")" true
}

run_scenario() {
  local id=$1
  header "Scenario ${id}"
  EXECUTED=$((EXECUTED + 1))
  if "scenario_${id}"; then
    ok "SCENARIO ${id} PASS"
    SCENARIO_LOG+="${id}=PASS "
  else
    fail "SCENARIO ${id} FAIL"
    SCENARIO_LOG+="${id}=FAIL "
    SCENARIO_FAILURES=$((SCENARIO_FAILURES + 1))
  fi
}

# ─── Cleanup ──────────────────────────────────────────────────────────
cleanup() {
  local status=$? failed=0 name pod channel
  # bash 3.2 treats "${empty[@]}" as unbound under set -u.
  set +eu
  while IFS= read -r pod; do
    kctl delete pod "$pod" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
  done <"$FIXTURE_POD_REGISTRY"
  for channel in "${CREATED_CHANNELS[@]}"; do
    kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
  done
  restore_hcc_cadence || { failed=1; echo "HCC cadence restore failed; check deployment/${HCC_DEPLOY}" >&2; }
  for name in "${CREATED_HOSTS[@]}"; do
    admin DELETE "${CAPI}/admin/hosts/${name}" >/dev/null 2>&1
    case "$HTTP_CODE" in 200 | 404) ;; *) failed=1; echo "delete Host ${name} returned ${HTTP_CODE}" >&2 ;; esac
    [ -z "$USER_ID" ] || set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
    [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
  done
  for name in "${CREATED_HOSTS[@]}"; do
    kctl wait --for=delete "deployment/${name}" -n "$HOST_NS" --timeout=120s >/dev/null 2>&1
    kctl delete pvc "${name}-workspace" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
  done
  rm -rf "$SCRATCH"
  [ "$failed" = 0 ] || { status=1; fail 'cleanup left resources behind (see stderr)'; }
  echo "E2E_825_SCENARIOS ${SCENARIO_LOG}"
  echo "E2E_825_EXECUTED=${EXECUTED} E2E_825_DEFINED=${#DEFINED_SCENARIOS[@]}"
  print_results || status=1
  if [ "$status" = 0 ] && [ "$SCENARIO_FAILURES" = 0 ] && [ "$EXECUTED" = "${#DEFINED_SCENARIOS[@]}" ]; then
    echo 'E2E_825_RESULT=PASS'
  else
    echo 'E2E_825_RESULT=FAIL'
    status=1
  fi
  exit "$status"
}
trap cleanup EXIT

# ─── Run ──────────────────────────────────────────────────────────────
header "#825 storage layout across lifecycle switches (run ${RUN}, ${E2E_KUBECONTEXT})"
# S5-S7 need HCC to render workspace-layout-merge; its flag defaults to true
# when unset or empty (A7.24). A literal value is the only one checkable here.
MERGE_FLAG="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" -o json | jq -r '
  [.spec.template.spec.containers[].env[]? | select(.name == "CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE")]
  | if length == 0 then "<unset>" elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end')" ||
  die "could not read deployment/${HCC_DEPLOY}"
case "$MERGE_FLAG" in
  '<unset>' | '' | true) ok "HCC CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=${MERGE_FLAG:-<empty>} (merge init rendered)" ;;
  *) die "HCC CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE is '${MERGE_FLAG}'; this E2E needs it unset, empty or 'true'" ;;
esac
SOURCE_HOST_JSON="$(kctl get host "$SOURCE_HOST" -n "$HOST_NS" -o json)" || die "source Host ${SOURCE_HOST} not found"
FIXTURE_IMAGE="$(kctl get deployment "$SOURCE_HOST" -n "$HOST_NS" \
  -o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].image}')"
[ -n "$FIXTURE_IMAGE" ] || die "could not read the ${SOURCE_HOST} image for the PVC fixture"
admin_login || die 'admin login failed'
admin GET "${CAPI}/admin/users?q=$(jq -rn --arg q "$DEV_EMAIL" '$q|@uri')" || true
USER_ID="$(jq -r --arg e "$DEV_EMAIL" '.items[]? | select((.email // "") == $e) | .id' <<<"$HTTP_BODY" | head -n1)"
[ -n "$USER_ID" ] || die "user ${DEV_EMAIL} not found (${HTTP_CODE})"
admin GET "${CAPI}/admin/users/${USER_ID}/teams" || true
TEAM_ID="$(jq -r '.items[0].id // empty' <<<"$HTTP_BODY")"
user_login || die 'user login failed'
ok "logged in as ${DEV_EMAIL}; fixture image ${FIXTURE_IMAGE}"

for id in NEG_CTRL NEG_RV S1 S2 S3 RACE; do run_scenario "$id"; done
header 'Suspended-Host scenarios'
if set_hcc_cadence; then
  for id in S4 S5 S6 S7; do run_scenario "$id"; done
else
  fail 'hcc-cadence fixture failed; S4-S7 not executed'
fi