Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
29.9 KB
Raw Blame
#!/usr/bin/env bash
# Branch-owned live lane for #825: upgrade an installation from v0.11.0 to this
# head and prove the Host PVC layout and history survive it (plan A7.40).
#
# Usage: e2e-hcc-layout-upgrade.sh main|control, each on a FRESH branch
# profile (no host-context-controller yet). The launcher deletes and recreates
# the profile between the two modes.
#
#   main     installs v0.11.0 (published images), seeds one Host per row, runs
#            deploy/scripts/host-layout-merge.sh inventory, upgrades to this
#            head with no manual step (the merge init supersedes the T5 split
#            database, plan A7.46; images
#            built from this tree, database kept), then checks verify and the
#            history of every row. Rows:
#              T1 stateful;
#              T2 stateless, active at the upgrade;
#              T3 stateless, suspended at the upgrade (woken afterwards);
#              T5 created stateless, used, switched to stateful by a
#                 CommunicationChannel and used again (root database plus the
#                 older state/ one, renamed by the merge init);
#              T6 stateless -> stateful -> stateless.
#            Prints E2E_UPGRADE_MAIN=PASS|FAIL and E2E_UPGRADE_HEAD=<sha>.
#   control  the negative control: CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false
#            in the upgrade deploy. inventory must report
#            FLAG_FALSE_WITH_SPLIT_TREES and the history check must fail on a
#            T2-shaped Host. Needs E2E_UPGRADE_MAIN_EVIDENCE (the main log of
#            the same HEAD). Prints E2E_UPGRADE_RESULT=PASS only when main
#            passed and the control failed as expected.
#
# Install path (plan A7.43): `git archive v0.11.0 deploy` into a scratch
# directory, then this head's `make minikube-setup IMAGE_SOURCE=ghcr
# MINIKUBE_IMAGE_TAG=v0.11.0` with BRANCH_PROFILE_DEPLOY_DIR pointing at it, so
# the manifests and images are the release's. v0.11.0's own full-setup.sh cannot
# run here: it requires a branch checkout with origin/dev as an ancestor. The
# setup applies this tree's CRDs, so the lane refuses unless they equal
# v0.11.0's apart from descriptions.
#
# Cluster writes besides the setups and the admin API: the named fixtures of
# e2e-hcc-storage-layout-switch.sh (hcc-cadence, pvc-fixture, channel,
# host-layout-merge) and, in control mode,
# FAULT:flag-false (kubectl set env on the HCC Deployment, undone in the
# trap). No kubectl exec into Host Pods, no SQLite reads or writes.
set -euo pipefail
umask 077

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
# shellcheck source=scripts/e2e/load-dotenv.sh
source "${REPO_ROOT}/scripts/e2e/load-dotenv.sh"
dotenv_load_canonical_root "${REPO_ROOT}"
# shellcheck source=scripts/e2e/admin-credentials.sh
source "${REPO_ROOT}/scripts/e2e/admin-credentials.sh"
# shellcheck source=scripts/e2e/e2e-lib.sh
source "${SCRIPT_DIR}/e2e-lib.sh"

die() { fail "$*"; exit 1; }

MODE="${1:-}"
case "$MODE" in main | control) ;; *) die 'usage: e2e-hcc-layout-upgrade.sh main|control' ;; esac

RELEASE=v0.11.0
# main_evidence_ok FILE HEAD: the main log passed on this same HEAD.
main_evidence_ok() {
  [ -r "$1" ] && grep -qx 'E2E_UPGRADE_MAIN=PASS' "$1" && grep -qx "E2E_UPGRADE_HEAD=$2" "$1"
}

# crd_schema REPO REF FILE: the CRD without its string-valued descriptions (a
# property named description is a map and stays).
crd_schema() {
  git -C "$1" show "$2:$3" |
    yq -o=json -I=0 'del(.. | select(tag == "!!map") | select(.description | tag == "!!str") | .description)'
}
# release_crds_match REPO RELEASE: HEAD has the release's CRD files, each equal
# to the release's apart from descriptions.
release_crds_match() {
  local repo=$1 release=$2 dir=charts/clerum-crds/crds file a b mismatched=''
  a="$(git -C "$repo" ls-tree --name-only "$release" "$dir/")" || { fail "cannot list ${dir} at ${release}"; return 1; }
  b="$(git -C "$repo" ls-tree --name-only HEAD "$dir/")" || { fail "cannot list ${dir} at HEAD"; return 1; }
  [ -n "$a" ] || { fail "no CRDs under ${dir} at ${release}"; return 1; }
  if [ "$a" != "$b" ]; then
    fail "CRD files differ from ${release}: $(diff <(echo "$a") <(echo "$b") | grep '^[<>]' | tr '\n' ' ')"
    return 1
  fi
  while IFS= read -r file; do
    [ "$(crd_schema "$repo" "$release" "$file")" = "$(crd_schema "$repo" HEAD "$file")" ] || mismatched="${mismatched} ${file}"
  done <<<"$a"
  [ -z "$mismatched" ] || { fail "CRD schema differs from ${release}:${mismatched}"; return 1; }
}
# extract_release_deploy REPO RELEASE DIR: the release's deploy/ tree in DIR.
extract_release_deploy() {
  # Through a file, so a failed archive is not hidden behind tar's exit code.
  mkdir -p "$3" || { fail "cannot create $3"; return 1; }
  git -C "$1" archive -o "$3/deploy.tar" "$2" deploy || { fail "git archive of deploy/ at $2 failed"; return 1; }
  tar -x -f "$3/deploy.tar" -C "$3" || { fail "could not extract deploy/ at $2"; return 1; }
  rm -f "$3/deploy.tar"
  [ -d "$3/deploy/overlays/minikube-ghcr" ] || { fail "$2 deploy tree has no minikube-ghcr overlay"; return 1; }
}

# ─── Preconditions ────────────────────────────────────────────────────
if [ -z "$E2E_KUBECONTEXT" ] || ! is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"; then
  die 'select an explicit branch-scoped Minikube context (KUBECONTEXT=<profile>)'
fi
require_safe_kube_context
[ "${E2E_ALLOW_STATELESS_CADENCE_ACCELERATION:-0}" = 1 ] ||
  die 'set E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 (T3 shortens the HCC idle window on this owned profile)'
[ -n "${E2E_PROFILE_PORTS_ENV:-}" ] && [ -r "$E2E_PROFILE_PORTS_ENV" ] ||
  die 'provide E2E_PROFILE_PORTS_ENV from the owned branch profile'
[ -n "${E2E_DEV_LOGIN_EMAIL:-}" ] || die 'set E2E_DEV_LOGIN_EMAIL to the seeded login ([email protected])'
[ -n "${E2E_UPGRADE_EVIDENCE_DIR:-}" ] && [ -d "$E2E_UPGRADE_EVIDENCE_DIR" ] ||
  die 'set E2E_UPGRADE_EVIDENCE_DIR to an existing directory for the setup logs'
HEAD_SHA="$(git -C "$REPO_ROOT" rev-parse HEAD)"
[ -z "$(git -C "$REPO_ROOT" status --porcelain)" ] ||
  die 'the worktree is dirty; the head images would not be this HEAD'
if [ "$MODE" = control ]; then
  main_evidence_ok "${E2E_UPGRADE_MAIN_EVIDENCE:-}" "$HEAD_SHA" ||
    die "E2E_UPGRADE_MAIN_EVIDENCE must be a main log with E2E_UPGRADE_MAIN=PASS and E2E_UPGRADE_HEAD=${HEAD_SHA}"
fi
git -C "$REPO_ROOT" rev-parse -q --verify "refs/tags/${RELEASE}" >/dev/null || die "tag ${RELEASE} is not in this clone"
command -v yq >/dev/null || die 'yq is required to compare the CRDs with the release'
release_crds_match "$REPO_ROOT" "$RELEASE" ||
  die "this tree's CRDs differ from ${RELEASE} beyond descriptions; the setup would not install the release's CRDs"
profile_port_value() {
  awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "$E2E_PROFILE_PORTS_ENV"
}
CONTROL_API_BASE_URL="$(profile_port_value CONTROL_API_URL)"
CONTROL_UI_BASE_URL="$(profile_port_value CONTROL_UI_URL)"
EXTERNAL_REST_API_BASE_URL="$(profile_port_value EXTERNAL_REST_API_URL)"
RPC_PROXY_BASE_URL="$(profile_port_value RPC_PROXY_URL)"
export CONTROL_API_BASE_URL CONTROL_UI_BASE_URL EXTERNAL_REST_API_BASE_URL RPC_PROXY_BASE_URL
require_branch_profile_urls "$E2E_KUBECONTEXT" "$E2E_PROFILE_PORTS_ENV" ||
  die 'profile URLs do not match the owned ports'
kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \
  'any(.items[]; .metadata.labels["minikube.k8s.io/name"]==$c)' >/dev/null ||
  die 'target is not the selected Minikube profile'

HOST_NS=mcp-host CHANNEL_NS=channels
HCC_NS=control-plane HCC_DEPLOY=host-context-controller
SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm}"
CAPI="${CONTROL_API_BASE_URL%/}/api/v1"
EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"
RPC_BASE="${RPC_PROXY_BASE_URL%/}"
ADMIN_USERNAME="${ADMIN_USERNAME:-admin}"
ADMIN_PASSWORD="${ADMIN_PASSWORD:-}"
[ -n "$ADMIN_PASSWORD" ] || ADMIN_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"
[ -n "$ADMIN_PASSWORD" ] || die 'admin password could not be resolved (ADMIN_PASSWORD or the canonical .env)'
DEV_EMAIL="$E2E_DEV_LOGIN_EMAIL"
DEV_PASSWORD="${E2E_USER_PASSWORD:-$ADMIN_PASSWORD}"
RUN="$(od -An -N3 -tx1 /dev/urandom | tr -d ' \n')"
POD_READY_TIMEOUT="${E2E_825_POD_READY_TIMEOUT:-300}"
SUSPEND_TIMEOUT="${E2E_825_SUSPEND_TIMEOUT:-300}"
TURN_TIMEOUT="${E2E_825_TURN_TIMEOUT:-240}"
STORE_LOG_TIMEOUT="${E2E_825_STORE_LOG_TIMEOUT:-30}"
LAYOUT_SCRIPT="${REPO_ROOT}/deploy/scripts/host-layout-merge.sh"
[ -r "$LAYOUT_SCRIPT" ] || die "missing operator script ${LAYOUT_SCRIPT}"

ROWS=(T1 T2 T3 T5 T6)
ROWS_DONE=0 LANE_OK=0
CREATED_HOSTS=() CREATED_CHANNELS=()
HCC_ENV_SAVED='' HCC_ENV_CHANGED=0 FLAG_FALSE_SET=0
SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/e2e-825-upgrade.XXXXXX")"
RELEASE_TREE="${SCRATCH}/release-${RELEASE}"
FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"
: >"$FIXTURE_POD_REGISTRY"
ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID='' FIXTURE_IMAGE='' POD_UID=''
HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()

# shellcheck source=scripts/e2e/e2e-825-lib.sh
source "${SCRIPT_DIR}/e2e-825-lib.sh"

# Seed files sit where the agent sees /workspace: the PVC root for a stateful
# Host, PVC/workspace for a stateless one on the old release.
# shellcheck disable=SC2016
WRITE_AT_SCRIPT='mkdir -p "$1/$2/e2e-825" && printf "%s\n" "$3" > "$1/$2/e2e-825/$4"'
# shellcheck disable=SC2016
READ_AT_SCRIPT='cat "$1/$2/e2e-825/$3"'

# run_logged LABEL CMD...: runs CMD from the repository root, its output in the
# evidence directory, and reports the command's own exit code.
run_logged() {
  local label=$1 rc=0 logf
  shift
  logf="${E2E_UPGRADE_EVIDENCE_DIR}/${MODE}-${RUN}-${label}.log"
  log "${label}: $* (log ${logf})"
  (cd "$REPO_ROOT" && "$@") >"$logf" 2>&1 || rc=$?
  echo "${label}_EXIT=${rc}" >>"$logf"
  [ "$rc" = 0 ] || { fail "${label} exited ${rc}: $(tail -n 5 "$logf" | tr '\n' ' ')"; return 1; }
  ok "${label} exit 0"
}

# Every setup restarts the services, so the forwards are re-held and probed.
hold_forwards() {
  local label=$1 key url code failures=0
  run_logged "${label}-stop-pf" make -f scripts/minikube-profiles/branch.mk branch-profile-stop-pf || return 1
  run_logged "${label}-pf" make -f scripts/minikube-profiles/branch.mk branch-profile-pf || return 1
  for key in CONTROL_API_URL CONTROL_UI_URL EXTERNAL_REST_API_URL RPC_PROXY_URL; do
    url="$(profile_port_value "$key")"
    code="$(curl -sS -o /dev/null -m 6 -w '%{http_code}' "$url" 2>/dev/null)" || code=000
    if [ "$code" = 000 ]; then
      fail "forward ${key} (${url}) did not answer"
      failures=$((failures + 1))
    else
      ok "forward ${key} answers HTTP ${code}"
    fi
  done
  [ "$failures" = 0 ]
}

login_all() {
  admin_login || return 1
  admin GET "${CAPI}/admin/users?q=$(jq -rn --arg q "$DEV_EMAIL" '$q|@uri')" || true
  USER_ID="$(jq -r --arg e "$DEV_EMAIL" '.items[]? | select((.email // "") == $e) | .id' <<<"$HTTP_BODY" | head -n1)"
  [ -n "$USER_ID" ] || { fail "user ${DEV_EMAIL} not found (${HTTP_CODE})"; return 1; }
  admin GET "${CAPI}/admin/users/${USER_ID}/teams" || true
  TEAM_ID="$(jq -r '.items[0].id // empty' <<<"$HTTP_BODY")"
  user_login || return 1
  # bash 3.2 treats "${empty[@]}" as unbound under set -u.
  [ "${#CREATED_HOSTS[@]}" -eq 0 ] && return 0
  if [ "${1:-}" = regrant ]; then
    local host
    for host in "${CREATED_HOSTS[@]}"; do grant_host "$host" || return 1; done
  fi
  mint_rpc_token
}

hcc_image() {
  kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
    -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}'
}
read_fixture_image() {
  FIXTURE_IMAGE="$(kctl get deployment "$SOURCE_HOST" -n "$HOST_NS" \
    -o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].image}')"
  [ -n "$FIXTURE_IMAGE" ] || { fail "could not read the ${SOURCE_HOST} image for the PVC fixture"; return 1; }
}

install_release() {
  [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" --ignore-not-found -o name)" ] ||
    { fail "deployment/${HCC_DEPLOY} already exists; the lane needs a fresh profile"; return 1; }
  extract_release_deploy "$REPO_ROOT" "$RELEASE" "$RELEASE_TREE" || return 1
  run_logged "install-${RELEASE}" env BRANCH_PROFILE_DEPLOY_DIR="$RELEASE_TREE/deploy" \
    make minikube-setup MINIKUBE_PROFILE="$E2E_KUBECONTEXT" \
    IMAGE_SOURCE=ghcr MINIKUBE_IMAGE_TAG="$RELEASE" || return 1
  case "$(hcc_image)" in
    *":${RELEASE}") ok "HCC runs $(hcc_image)" ;;
    *) fail "HCC runs '$(hcc_image)', expected the ${RELEASE} image"; return 1 ;;
  esac
  # Witness that the release's manifests were applied: v0.11.0's HCC Deployment
  # sets STDIO_BRIDGE_IMAGE, this head's does not.
  kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" -o json | jq -e \
    '[.spec.template.spec.containers[] | select(.name=="host-context-controller") | .env[]? | select(.name=="STDIO_BRIDGE_IMAGE")] | length == 1' >/dev/null ||
    { fail "HCC Deployment lacks STDIO_BRIDGE_IMAGE; the ${RELEASE} manifests were not applied"; return 1; }
  ok "HCC Deployment carries the ${RELEASE} manifest (STDIO_BRIDGE_IMAGE set)"
  # Built now and loaded next to the release images, so the upgrade below is a
  # deploy of minutes and the active Hosts stay active through it.
  run_logged build-head-images make minikube-build-images MINIKUBE_PROFILE="$E2E_KUBECONTEXT" || return 1
  hold_forwards release || return 1
  login_all || return 1
  SOURCE_HOST_JSON="$(kctl get host "$SOURCE_HOST" -n "$HOST_NS" -o json)" ||
    { fail "source Host ${SOURCE_HOST} not found"; return 1; }
  # Personalization writes IDENTITY.md and friends into the workspace; the
  # clones would then collide on F3 instead of exercising the rows.
  jq -e '.spec.personalization.enabled != true' <<<"$SOURCE_HOST_JSON" >/dev/null ||
    { fail "source Host ${SOURCE_HOST} has personalization enabled"; return 1; }
  # verify's fleet verdict below must be decided by the rows' Hosts alone.
  kctl get hosts -n "$HOST_NS" -o json | jq -e '[.items[] | select(.spec.lifecycle.stateless == true)] | length == 0' >/dev/null ||
    { fail 'the fresh install already has stateless Hosts'; return 1; }
  read_fixture_image
}

upgrade_to_head() {
  run_logged upgrade-to-head make minikube-setup MINIKUBE_PROFILE="$E2E_KUBECONTEXT" \
    IMAGE_SOURCE=local REUSE_DB=true ARGS=--skip-build || return 1
  case "$(hcc_image)" in
    *":${RELEASE}") fail "HCC still runs $(hcc_image) after the upgrade"; return 1 ;;
    *) ok "HCC runs $(hcc_image)" ;;
  esac
  hold_forwards head || return 1
  # The setup's seed (seed-e2e-data.sh put_agents) sets the user's and team's
  # agents to the source Host only, revoking the rows' grants.
  login_all regrant || return 1
  read_fixture_image
}

HOST_T1='' HOST_T2='' HOST_T3='' HOST_T5='' HOST_T6=''
new_host() { # ROW STATELESS: creates the row's Host, sets HOST_<row>
  local name
  name="$(truncate_rfc1123 "e2e-up-$(tr '[:upper:]' '[:lower:]' <<<"$1")-${RUN}")"
  printf -v "HOST_$1" '%s' "$name"
  create_host "$name" "$2" || return 1
  wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
  printf '%s' "$POD_UID" >"${SCRATCH}/${name}.uid"
}

seed_file() { # HOST DIR(.|workspace)
  pvc_fixture "$1" rw write-file "$WRITE_AT_SCRIPT" "$2" "upgrade ${RUN} $1" seed.txt >/dev/null
}

# snapshot_served HOST: what the Host serves now, the history oracle's
# baseline. Every listed chat gets a digest; an empty catalog fails.
snapshot_served() {
  local host=$1 sessions chats chat digest
  rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
  [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
  sessions="$HTTP_BODY"
  chat_set "$sessions" "$host" >"${SCRATCH}/${host}.chats"
  chats="$(jq -r --arg a "$host" '[.items[]? | select(.agent == $a) | .chatId] | unique | .[]' <<<"$sessions")"
  [ -n "$chats" ] || { fail "${host}: serves no conversation before the upgrade"; return 1; }
  : >"${SCRATCH}/${host}.threads"
  for chat in $chats; do
    read_messages "$host" "$chat" || return 1
    digest="$(history_digest "$HTTP_BODY")"
    [ "${digest%% *}" -gt 0 ] || { fail "${host}/${chat}: empty history"; return 1; }
    printf '%s %s\n' "$chat" "$digest" >>"${SCRATCH}/${host}.threads"
  done
  ok "${host}: serves $(cat "${SCRATCH}/${host}.chats" | cut -d: -f1) conversation(s) before the upgrade"
}

# wait_upgraded_pod HOST: the Host's Pod after the upgrade (a new UID), waking
# a stateless Host that suspended while the deploy ran.
wait_upgraded_pod() {
  local host=$1 state
  state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"
  if [ "$state" = suspended ]; then
    log "${host} suspended during the upgrade; waking it"
    post_wake "$host" || return 1
  fi
  wait_new_ready_pod "$host" "$(cat "${SCRATCH}/${host}.uid")" "$POD_READY_TIMEOUT"
}

# The head HCC renders the merge init on every Host Deployment it converged.
wait_merge_templates() {
  local host elapsed=0 pending
  while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do
    pending=''
    for host in "$@"; do
      kctl get deployment "$host" -n "$HOST_NS" -o json | jq -e '
        [.spec.template.spec.initContainers[]?.name] == ["workspace-layout-merge"]' >/dev/null || pending+="${host} "
    done
    [ -n "$pending" ] || { ok "every row's Deployment renders workspace-layout-merge"; return 0; }
    sleep 5
    elapsed=$((elapsed + 5))
  done
  fail "Deployments not re-rendered by the head HCC within ${POD_READY_TIMEOUT}s: ${pending}"
  return 1
}

# run_layout CMD: one run of the operator script; output in LAYOUT_OUT, exit in LAYOUT_RC.
run_layout() {
  LAYOUT_OUT="${SCRATCH}/layout-$1-${SECONDS}.out"
  LAYOUT_RC=0
  bash "$LAYOUT_SCRIPT" "$1" --context "$E2E_KUBECONTEXT" >"$LAYOUT_OUT" 2>&1 || LAYOUT_RC=$?
  cp "$LAYOUT_OUT" "${E2E_UPGRADE_EVIDENCE_DIR}/${MODE}-${RUN}-layout-$1-${SECONDS}.out"
}
row_of() { # HOST → the result of the Host's PVC row in LAYOUT_OUT
  local row
  row="$(grep -F "PVC ${HOST_NS}/${1}-workspace " "$LAYOUT_OUT")" || return 1
  printf '%s' "${row#*result=}"
}
expect_row() { # HOST PATTERN LABEL
  local row
  row="$(row_of "$1")" || { fail "$3: no row for ${1}-workspace"; return 1; }
  # shellcheck disable=SC2254 # PATTERN is a case pattern on purpose
  case "$row" in $2) ok "$3: ${1}-workspace is ${row%% workspace layout*}" ;; *) fail "$3: ${1}-workspace is '${row}', expected $2"; return 1 ;; esac
}

# ─── main ─────────────────────────────────────────────────────────────
seed_rows() {
  # T3 first: the shortened idle window suspends every idle stateless Host,
  # so the others are created after it is restored.
  new_host T3 true && seed_history "$HOST_T3" 3 1 && seed_file "$HOST_T3" workspace && snapshot_served "$HOST_T3" || return 1
  set_hcc_cadence && wait_suspended "$HOST_T3" && restore_hcc_cadence || return 1
  new_host T1 false && seed_history "$HOST_T1" 3 1 && seed_file "$HOST_T1" . && snapshot_served "$HOST_T1" || return 1
  # T5: the old history under state/, then a channel makes HCC run it
  # stateful on the PVC root, where mcp-host opens a second database.
  new_host T5 true && seed_history "$HOST_T5" 3 1 && seed_file "$HOST_T5" workspace || return 1
  local channel
  channel="$(truncate_rfc1123 "e2e-up-t5-${RUN}")"
  log "FAULT:channel ${CHANNEL_NS}/${channel} → ${HOST_T5}"
  printf '%s\n' 'apiVersion: clerum.io/v1alpha1' 'kind: CommunicationChannel' 'metadata:' \
    "  name: ${channel}" "  namespace: ${CHANNEL_NS}" 'spec:' "  hostRef: ${HOST_T5}" '  telegram:' \
    "    - channelId: \"e2e-up-${RUN}\"" '      chatType: "private"' '      userIds: ["000000000"]' |
    kctl create -f - >/dev/null || { fail 'T5: CommunicationChannel create failed'; return 1; }
  CREATED_CHANNELS+=("$channel")
  wait_new_ready_pod "$HOST_T5" "$(cat "${SCRATCH}/${HOST_T5}.uid")" "$POD_READY_TIMEOUT" || return 1
  printf '%s' "$POD_UID" >"${SCRATCH}/${HOST_T5}.uid"
  seed_history "$HOST_T5" 1 1 && snapshot_served "$HOST_T5" || return 1
  # T6: the release's stateless entry keeps the newer database, so what T6
  # serves afterwards is what the snapshot records.
  new_host T6 true && seed_history "$HOST_T6" 2 1 && seed_file "$HOST_T6" workspace || return 1
  local stateless
  for stateless in false true; do
    set_lifecycle "$HOST_T6" "$stateless" || return 1
    wait_new_ready_pod "$HOST_T6" "$(cat "${SCRATCH}/${HOST_T6}.uid")" "$POD_READY_TIMEOUT" || return 1
    printf '%s' "$POD_UID" >"${SCRATCH}/${HOST_T6}.uid"
  done
  seed_history "$HOST_T6" 1 1 && snapshot_served "$HOST_T6" || return 1
  # T2 last, so it is still active when the upgrade replaces its Pod.
  new_host T2 true && seed_history "$HOST_T2" 3 1 && seed_file "$HOST_T2" workspace && snapshot_served "$HOST_T2"
}

predict_rows() {
  run_layout inventory
  [ "$LAYOUT_RC" = 0 ] || { fail "inventory exited ${LAYOUT_RC}, expected 0: $(tail -n 3 "$LAYOUT_OUT" | tr '\n' ' ')"; return 1; }
  grep -qx 'UPGRADE_BLOCKERS=0' "$LAYOUT_OUT" || { fail "inventory: $(grep UPGRADE_BLOCKERS "$LAYOUT_OUT"), expected UPGRADE_BLOCKERS=0"; return 1; }
  expect_row "$HOST_T5" WILL_SUPERSEDE 'inventory T5' &&
    expect_row "$HOST_T2" WILL_MERGE 'inventory T2' &&
    expect_row "$HOST_T3" WILL_MERGE 'inventory T3' &&
    expect_row "$HOST_T6" WILL_MERGE 'inventory T6' &&
    expect_row "$HOST_T1" NO_SPLIT_SIGNATURE 'inventory T1'
}

check_upgraded() {
  local host
  wait_merge_templates "$HOST_T1" "$HOST_T2" "$HOST_T3" "$HOST_T5" "$HOST_T6" || return 1
  wait_upgraded_pod "$HOST_T1" && assert_pod "$HOST_T1" false /workspace || return 1
  wait_upgraded_pod "$HOST_T2" && assert_pod "$HOST_T2" true /workspace || return 1
  wait_upgraded_pod "$HOST_T5" && assert_pod "$HOST_T5" false /workspace || return 1
  wait_upgraded_pod "$HOST_T6" && assert_pod "$HOST_T6" true /workspace || return 1
  [ "$(kctl get host "$HOST_T3" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}')" = suspended ] ||
    { fail 'T3: no longer suspended before verify'; return 1; }
  run_layout verify
  [ "$LAYOUT_RC" = 1 ] || { fail "verify exited ${LAYOUT_RC}, expected 1 (T3 split)"; return 1; }
  for host in "$HOST_T1" "$HOST_T2" "$HOST_T5" "$HOST_T6"; do expect_row "$host" POD_OK 'verify' || return 1; done
  expect_row "$HOST_T3" SPLIT 'verify T3' || return 1

  # T3 merges on its natural wake.
  post_wake "$HOST_T3" && wait_new_ready_pod "$HOST_T3" '' "$POD_READY_TIMEOUT" && assert_pod "$HOST_T3" true /workspace || return 1

  run_layout verify
  if [ "$LAYOUT_RC" != 0 ] || ! grep -qx 'HOST_LAYOUT_VERIFY=PASS' "$LAYOUT_OUT"; then
    fail "verify after the T3 wake exited ${LAYOUT_RC}: $(tail -n 3 "$LAYOUT_OUT" | tr '\n' ' ')"
    return 1
  fi
  for host in "$HOST_T1" "$HOST_T2" "$HOST_T3" "$HOST_T5" "$HOST_T6"; do expect_row "$host" POD_OK 'verify after wake' || return 1; done
}

check_history() {
  local listing
  assert_history "$HOST_T1" "upgrade(stateful)" && read_seed_file "$HOST_T1" seed.txt "upgrade ${RUN} ${HOST_T1}" || return 1
  ROWS_DONE=$((ROWS_DONE + 1))
  assert_history "$HOST_T2" "upgrade(stateless-active)" && read_seed_file "$HOST_T2" seed.txt "upgrade ${RUN} ${HOST_T2}" || return 1
  ROWS_DONE=$((ROWS_DONE + 1))
  assert_history "$HOST_T3" "upgrade(stateless-suspended)->wake" && read_seed_file "$HOST_T3" seed.txt "upgrade ${RUN} ${HOST_T3}" || return 1
  ROWS_DONE=$((ROWS_DONE + 1))
  # T5 serves the root family it served before the upgrade; the merge init
  # renamed the older state/ to state.db.superseded-<ts>, and the old files
  # stay under workspace/.
  assert_history "$HOST_T5" "two-databases->upgrade(superseded)" || return 1
  listing="$(pvc_fixture "$HOST_T5" ro list-superseded "$LIST_SCRIPT")" || return 1
  grep -Eq '^f \./state\.db\.superseded-[0-9]{8}T[0-9]{6}Z/state\.db$' <<<"$listing" ||
    { fail "T5: no state.db.superseded-*/state.db: ${listing}"; return 1; }
  ! grep -Eq '^d \./state$' <<<"$listing" || { fail "T5: state/ still present: ${listing}"; return 1; }
  [ "$(pvc_fixture "$HOST_T5" ro read-file "$READ_AT_SCRIPT" workspace seed.txt)" = "upgrade ${RUN} ${HOST_T5}" ] ||
    { fail 'T5: the old seed file is not under workspace/'; return 1; }
  ok 'T5: old database kept as state.db.superseded-*/state.db, old files under workspace/'
  ROWS_DONE=$((ROWS_DONE + 1))
  assert_history "$HOST_T6" "upgrade(stateless-stateful-stateless)" && read_seed_file "$HOST_T6" seed.txt "upgrade ${RUN} ${HOST_T6}" || return 1
  ROWS_DONE=$((ROWS_DONE + 1))
}

run_main() {
  header "Install ${RELEASE}" && install_release &&
    header 'Seed rows on the release' && seed_rows &&
    header 'Predict with inventory' && predict_rows &&
    header 'Upgrade to this head' && upgrade_to_head &&
    header 'Verify and wake' && check_upgraded &&
    header 'History of every row' && check_history
}

# ─── control ──────────────────────────────────────────────────────────
run_control() {
  local flag sessions before after
  header "Install ${RELEASE}" && install_release || return 1
  header 'Seed a T2-shaped Host'
  new_host T2 true && seed_history "$HOST_T2" 2 1 && snapshot_served "$HOST_T2" || return 1
  header 'Upgrade with the merge flag off'
  # Set on the release HCC, which ignores it; the setup's client-side apply
  # keeps an env entry it never owned, so the head HCC starts with "false".
  log "FAULT:flag-false CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false on deployment/${HCC_DEPLOY}"
  FLAG_FALSE_SET=1
  kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false >/dev/null ||
    { fail 'flag-false: kubectl set env failed'; return 1; }
  kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||
    { fail 'flag-false: HCC rollout did not complete'; return 1; }
  upgrade_to_head || return 1
  flag="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" -o json | jq -r '
    [.spec.template.spec.containers[].env[]? | select(.name == "CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE") | .value] | first // "<unset>"')"
  [ "$flag" = false ] || { fail "control: the head HCC runs with the flag '${flag}', the control did not happen"; return 1; }
  ok 'control: the head HCC runs with CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false'
  wait_upgraded_pod "$HOST_T2" || return 1
  jq -e '.inits == [] and .mounts == [{mountPath:"/workspace", name:"workspace"}]' <<<"$(pod_storage)" >/dev/null ||
    { fail "control: T2 Pod storage is $(pod_storage), expected the root mount without the merge init"; return 1; }
  [ "$(store_path "$HOST_T2")" = /workspace/state.db ] || { fail 'control: T2 did not open /workspace/state.db'; return 1; }
  ok 'control: T2 starts on the root mount with no merge init'
  run_layout inventory
  if [ "$LAYOUT_RC" != 1 ] || ! grep -q '^FLAG_FALSE_WITH_SPLIT_TREES=' "$LAYOUT_OUT"; then
    fail "control: inventory exited ${LAYOUT_RC} without FLAG_FALSE_WITH_SPLIT_TREES"
    return 1
  fi
  ok "control: inventory reports $(grep '^FLAG_FALSE_WITH_SPLIT_TREES=' "$LAYOUT_OUT")"
  # The oracle has to run (sessions 200) and then disagree with the baseline.
  rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${HOST_T2}/sessions?agent=${HOST_T2}&limit=100" || true
  [ "$HTTP_CODE" = 200 ] || { fail "control: sessions returned ${HTTP_CODE}"; return 1; }
  sessions="$HTTP_BODY"
  before="$(cat "${SCRATCH}/${HOST_T2}.chats")"
  after="$(chat_set "$sessions" "$HOST_T2")"
  if chats_match "$before" "$after"; then
    fail "control: T2 still serves its history (${after}); the control did not fail"
    return 1
  fi
  echo "E2E_UPGRADE_CONTROL_DIVERGED ${HOST_T2} before=${before} after=${after}"
  ok 'control: history check fails with the flag off, as expected'
}

# ─── Cleanup ──────────────────────────────────────────────────────────
cleanup() {
  local status=$? failed=0 name pod channel
  set +eu
  while IFS= read -r pod; do
    kctl delete pod "$pod" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
  done <"$FIXTURE_POD_REGISTRY"
  for channel in "${CREATED_CHANNELS[@]}"; do
    kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
  done
  restore_hcc_cadence || { failed=1; echo "HCC cadence restore failed; check deployment/${HCC_DEPLOY}" >&2; }
  if [ "$FLAG_FALSE_SET" = 1 ]; then
    kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE- >/dev/null 2>&1 || failed=1
  fi
  for name in "${CREATED_HOSTS[@]}"; do
    admin DELETE "${CAPI}/admin/hosts/${name}" >/dev/null 2>&1
    case "$HTTP_CODE" in 200 | 404) ;; *) failed=1; echo "delete Host ${name} returned ${HTTP_CODE}" >&2 ;; esac
    [ -z "$USER_ID" ] || set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
    [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
  done
  for name in "${CREATED_HOSTS[@]}"; do
    kctl wait --for=delete "deployment/${name}" -n "$HOST_NS" --timeout=120s >/dev/null 2>&1
    kctl delete pvc "${name}-workspace" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
  done
  rm -rf "$SCRATCH"
  [ "$failed" = 0 ] || { status=1; fail 'cleanup left resources behind (see stderr)'; }
  print_results || status=1
  echo "E2E_UPGRADE_HEAD=${HEAD_SHA}"
  if [ "$MODE" = main ]; then
    echo "E2E_UPGRADE_ROWS_EXECUTED=${ROWS_DONE} E2E_UPGRADE_ROWS_DEFINED=${#ROWS[@]}"
    if [ "$status" = 0 ] && [ "$LANE_OK" = 1 ] && [ "$ROWS_DONE" = "${#ROWS[@]}" ]; then
      echo 'E2E_UPGRADE_MAIN=PASS'
    else
      echo 'E2E_UPGRADE_MAIN=FAIL'
      status=1
    fi
  else
    if [ "$status" = 0 ] && [ "$LANE_OK" = 1 ]; then
      echo 'E2E_UPGRADE_CONTROL=FAILED_AS_EXPECTED'
      echo 'E2E_UPGRADE_RESULT=PASS'
    else
      echo 'E2E_UPGRADE_CONTROL=NOT_PROVEN'
      echo 'E2E_UPGRADE_RESULT=FAIL'
      status=1
    fi
  fi
  echo "E2E_UPGRADE_EXIT=${status}"
  exit "$status"
}
trap cleanup EXIT

header "#825 upgrade lane ${RELEASE} → ${HEAD_SHA:0:9} (${MODE}, run ${RUN}, ${E2E_KUBECONTEXT})"
if [ "$MODE" = main ]; then run_main && LANE_OK=1; else run_control && LANE_OK=1; fi