Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): upgrade from v0.11.0 to this head

Alfredo Lopez228ab974fc55
diff --git a/Makefile b/Makefile
index 850fcc2e6..456eddc65 100644
--- a/Makefile
+++ b/Makefile
@@ -1406,6 +1406,11 @@ test-e2e-hcc-storage-layout-switch: ## Run the #825 storage-layout gate across l
@echo "Running HCC storage-layout switch E2E gate..."
E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" bash scripts/e2e/e2e-hcc-storage-layout-switch.sh
+.PHONY: test-e2e-hcc-layout-upgrade
+test-e2e-hcc-layout-upgrade: ## Run the #825 upgrade lane from v0.11.0 to this head (UPGRADE_MODE=main|control; fresh owned branch profile, E2E_PROFILE_PORTS_ENV, E2E_DEV_LOGIN_EMAIL, E2E_UPGRADE_EVIDENCE_DIR)
+ @echo "Running HCC layout upgrade E2E lane ($(UPGRADE_MODE))..."
+ E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" bash scripts/e2e/e2e-hcc-layout-upgrade.sh "$(UPGRADE_MODE)"
+
.PHONY: test-e2e-825-desktop-continuity
test-e2e-825-desktop-continuity: ## Run the #825 Desktop journey across stateful/stateless switches (owned branch profile, forwards held, real model turns)
@set -eu; \
diff --git a/scripts/e2e/e2e-825-lib.sh b/scripts/e2e/e2e-825-lib.sh
new file mode 100644
index 000000000..0fe55ab2f
--- /dev/null
+++ b/scripts/e2e/e2e-825-lib.sh
@@ -0,0 +1,512 @@
+# shellcheck shell=bash
+# Helpers shared by the #825 live gates (e2e-hcc-storage-layout-switch.sh and
+# e2e-hcc-layout-upgrade.sh). Sourced after e2e-lib.sh; defines functions and
+# fixture scripts only. The caller sets HOST_NS, HCC_NS, HCC_DEPLOY, CAPI,
+# EXT_BASE, RPC_BASE, ADMIN_USERNAME, ADMIN_PASSWORD, DEV_EMAIL, DEV_PASSWORD,
+# SOURCE_HOST_JSON, SCRATCH, FIXTURE_POD_REGISTRY, FIXTURE_IMAGE, RUN,
+# LAYOUT_SCRIPT, the timeouts, and the state below.
+
+# ─── HTTP ─────────────────────────────────────────────────────────────
+# http METHOD URL [BODY] [MAX_SECONDS] → HTTP_CODE, HTTP_BODY. A transport
+# failure is HTTP_CODE=000 and a non-zero return. The body goes to curl on
+# stdin, never in argv, because login bodies carry passwords.
+http() {
+ local method=$1 url=$2 body=${3:-} max=${4:-60} resp
+ local args=(-sS -m "$max" -w '\n%{http_code}' -X "$method" "$url")
+ [ ${#HTTP_HEADERS[@]} -eq 0 ] || args+=("${HTTP_HEADERS[@]}")
+ [ -z "$body" ] || args+=(-H 'Content-Type: application/json' --data-binary @-)
+ if ! resp="$(printf '%s' "$body" | curl "${args[@]}")"; then
+ HTTP_CODE=000 HTTP_BODY=''
+ return 1
+ fi
+ HTTP_CODE="$(tail -n1 <<<"$resp")"
+ HTTP_BODY="$(sed '$d' <<<"$resp")"
+}
+# auth_header KIND LINE: write one header line to a 0600 file under $SCRATCH
+# and print its path. Credentials reach curl through `-H @file`, never argv,
+# where any local user can read them with ps.
+auth_header() {
+ local file="${SCRATCH}/${1}.header"
+ (umask 077 && printf '%s\n' "$2" >"$file")
+ printf '%s' "$file"
+}
+admin() { HTTP_HEADERS=(-H "@$(auth_header admin "Cookie: control_ui_admin_session=${ADMIN_COOKIE}")"); http "$@"; }
+rpc() { HTTP_HEADERS=(-H "@$(auth_header rpc "Authorization: Bearer ${RPC_TOKEN}")"); http "$@"; }
+
+admin_login() {
+ local headers="${SCRATCH}/admin-login.headers" line cookie
+ HTTP_HEADERS=(-D "$headers")
+ http POST "${CAPI}/admin/auth/login" \
+ "$(jq -cn --arg u "$ADMIN_USERNAME" --arg p "$ADMIN_PASSWORD" '{username:$u,password:$p}')" || true
+ [[ "$HTTP_CODE" =~ ^2 ]] || { fail "admin login returned ${HTTP_CODE}"; return 1; }
+ while IFS= read -r line; do
+ line="${line%$'\r'}"
+ case "$(tr '[:upper:]' '[:lower:]' <<<"$line")" in
+ set-cookie:*control_ui_admin_session=*)
+ cookie="${line#*control_ui_admin_session=}"
+ ADMIN_COOKIE="${cookie%%;*}"
+ ;;
+ esac
+ done <"$headers"
+ rm -f "$headers"
+ [ -n "$ADMIN_COOKIE" ] || { fail 'admin login set no control_ui_admin_session cookie'; return 1; }
+}
+
+user_login() {
+ HTTP_HEADERS=()
+ http POST "${EXT_BASE}/api/v1/auth/password-login" \
+ "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "password login for ${DEV_EMAIL} returned ${HTTP_CODE} (a 429 means the lockout window; wait, do not retry blindly)"; return 1; }
+ SESSION_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'password login returned no token'; return 1; }
+}
+
+# The rpc token names every Host this run created so far.
+mint_rpc_token() {
+ local refs
+ refs="$(printf '%s\n' "${CREATED_HOSTS[@]}" | jq -R . | jq -cs .)"
+ HTTP_HEADERS=(-H "@$(auth_header session "Authorization: Bearer ${SESSION_TOKEN}")")
+ http POST "${EXT_BASE}/api/v1/rpc/token" "$(jq -cn --argjson refs "$refs" '{hostRefs:$refs,scopes:[
+ "host:message:invoke","host:session:read","host:status:read","host:health:read",
+ "host:wake:write","host:model:write"]}')" || true
+ [[ "$HTTP_CODE" =~ ^2 ]] || { fail "rpc token mint returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ RPC_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'rpc token response has no token'; return 1; }
+}
+
+# ─── Host lifecycle through the admin API ─────────────────────────────
+# PUT /admin/users/:id/agents filters names against the active agent set, so a
+# just-created Host can lag; retry until the response echoes it back.
+set_agent_grant() {
+ local url=$1 name=$2 op=$3 attempt=1 names expected
+ while :; do
+ admin GET "$url" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "GET ${url} returned ${HTTP_CODE}"; return 1; }
+ expected="$(jq -c '((.agentNames // []) + (.deletedAgentNames // []))' <<<"$HTTP_BODY")"
+ if [ "$op" = add ]; then
+ names="$(jq -c --arg a "$name" '((.agentNames // []) + [$a]) | unique' <<<"$HTTP_BODY")"
+ else
+ names="$(jq -c --arg a "$name" '(.agentNames // []) - [$a]' <<<"$HTTP_BODY")"
+ fi
+ admin PUT "$url" "$(jq -cn --argjson n "$names" --argjson e "$expected" \
+ '{agentNames:$n,expectedCurrentAgentNames:$e}')" || true
+ if [[ "$HTTP_CODE" =~ ^2 ]]; then
+ if [ "$op" = remove ] ||
+ jq -e --arg a "$name" '(.agentNames // []) | index($a) != null' <<<"$HTTP_BODY" >/dev/null; then
+ return 0
+ fi
+ fi
+ [ "$attempt" -lt 20 ] || { fail "${op} grant '${name}' on ${url} did not converge (${HTTP_CODE})"; return 1; }
+ attempt=$((attempt + 1))
+ sleep 3
+ done
+}
+
+grant_host() {
+ set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$1" add || return 1
+ [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$1" add
+}
+
+# create_host NAME STATELESS(true|false): clones the seeded source Host's spec
+# (context, secret, model) without desktop or channels and creates it via the API.
+create_host() {
+ local name=$1 stateless=$2 spec
+ spec="$(jq -c --arg n "$name" --argjson s "$stateless" '
+ .spec + {host:$n} | del(.channels) | del(.desktop)
+ | .lifecycle = {stateless:$s}' <<<"$SOURCE_HOST_JSON")"
+ admin POST "${CAPI}/admin/hosts" "$(jq -cn --arg n "$name" --argjson s "$spec" '{metadata:{name:$n},spec:$s}')" || true
+ [ "$HTTP_CODE" = 201 ] || { fail "create Host ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ CREATED_HOSTS+=("$name")
+ grant_host "$name" || return 1
+ mint_rpc_token
+}
+
+# set_lifecycle NAME STATELESS(true|false): GET then PUT with the read's
+# resourceVersion, echoing the full spec as Control UI does.
+set_lifecycle() {
+ local name=$1 stateless=$2 rv spec
+ admin GET "${CAPI}/admin/hosts/${name}" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "GET Host ${name} returned ${HTTP_CODE}"; return 1; }
+ rv="$(jq -er '.metadata.resourceVersion' <<<"$HTTP_BODY")" || { fail "Host ${name} read has no resourceVersion"; return 1; }
+ spec="$(jq -c --argjson s "$stateless" '.spec | .lifecycle = {stateless:$s}' <<<"$HTTP_BODY")"
+ admin PUT "${CAPI}/admin/hosts/${name}" \
+ "$(jq -cn --arg rv "$rv" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "PUT lifecycle stateless=${stateless} on ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ log "lifecycle request on ${name}: stateless=${stateless}"
+}
+
+# ─── Pods and the lifecycle witness ───────────────────────────────────
+host_pods() { kctl get pods -n "$HOST_NS" -l "app=$1" -o json; }
+
+# wait_new_ready_pod NAME OLD_UID TIMEOUT: exactly one Pod exists, it is Ready,
+# and its UID differs from OLD_UID. Sets POD_JSON, POD_UID.
+wait_new_ready_pod() {
+ local name=$1 old=$2 timeout=$3 elapsed=0 pods
+ while [ "$elapsed" -lt "$timeout" ]; do
+ if pods="$(host_pods "$name")" && POD_JSON="$(jq -ce --arg old "$old" '
+ .items | select(length == 1) | .[0]
+ | select(.metadata.deletionTimestamp == null and .metadata.uid != $old)
+ | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))' <<<"$pods")"; then
+ POD_UID="$(jq -r '.metadata.uid' <<<"$POD_JSON")"
+ jq -r '.spec.nodeName' <<<"$POD_JSON" >"${SCRATCH}/${name}.node"
+ return 0
+ fi
+ sleep 3
+ elapsed=$((elapsed + 3))
+ done
+ fail "${name}: no new Ready Pod within ${timeout}s"
+ kctl get pods -n "$HOST_NS" -l "app=${name}" -o wide >&2 || true
+ kctl describe pods -n "$HOST_NS" -l "app=${name}" 2>&1 | tail -n 40 >&2 || true
+ return 1
+}
+
+# Storage seen by the running Pod: workspace mounts, storage env, init names.
+pod_storage() {
+ jq -cS '{
+ mounts: [.spec.containers[] | select(.name == "mcp-host") | .volumeMounts[] | select(.name == "workspace")],
+ store: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_STORE") | .value] | first),
+ dbDir: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_DB_DIR") | .value] | first),
+ inits: [.spec.initContainers[]? | .name]
+ }' <<<"$POD_JSON"
+}
+
+# assert_pod NAME STATELESS WORKSPACE_PATH: lifecycle markers of the new Pod
+# match the requested lifecycle and its storage is the one root layout.
+assert_pod() {
+ local name=$1 stateless=$2 path=$3 env_flag priority merge_exit storage db
+ env_flag="$(jq -r '[.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") | .value] | first // ""' <<<"$POD_JSON")"
+ priority="$(jq -r '.spec.priorityClassName // ""' <<<"$POD_JSON")"
+ if [ "$stateless" = true ]; then
+ [ "$env_flag" = true ] && [ "$priority" = clerum-interactive-host ] ||
+ { fail "${name}: Pod is not stateless (env=${env_flag:-<none>} priority=${priority:-<none>})"; return 1; }
+ else
+ [ -z "$env_flag" ] && [ "$priority" != clerum-interactive-host ] ||
+ { fail "${name}: Pod is not stateful (env=${env_flag} priority=${priority})"; return 1; }
+ fi
+ storage="$(pod_storage)"
+ jq -e --arg p "$path" '
+ .mounts == [{mountPath:$p, name:"workspace"}] and .store == "sqlite"
+ and .dbDir == $p and .inits == ["workspace-layout-merge"]' <<<"$storage" >/dev/null ||
+ { fail "${name}: Pod storage is not the root layout: ${storage}"; return 1; }
+ merge_exit="$(jq -r '[.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge") | .state.terminated.exitCode] | first // "none"' <<<"$POD_JSON")"
+ [ "$merge_exit" = 0 ] || { fail "${name}: workspace-layout-merge exit ${merge_exit}"; return 1; }
+ if [ -s "${SCRATCH}/${name}.storage" ] && [ "$(cat "${SCRATCH}/${name}.storage")" != "$storage" ]; then
+ fail "${name}: storage changed across the switch: $(cat "${SCRATCH}/${name}.storage") -> ${storage}"
+ return 1
+ fi
+ printf '%s' "$storage" >"${SCRATCH}/${name}.storage"
+ db="$(store_path "$name")" || { fail "${name}: no store-opened log line (reason above)"; return 1; }
+ [ "$db" = "${path}/state.db" ] || { fail "${name}: mcp-host opened ${db}, expected ${path}/state.db"; return 1; }
+ if [ -s "${SCRATCH}/${name}.dbpath" ] && [ "$(cat "${SCRATCH}/${name}.dbpath")" != "$db" ]; then
+ fail "${name}: store changed across the switch: $(cat "${SCRATCH}/${name}.dbpath") -> ${db}"
+ return 1
+ fi
+ printf '%s' "$db" >"${SCRATCH}/${name}.dbpath"
+ ok "${name}: Pod ${POD_UID:0:8} stateless=${stateless}, root layout at ${path}, merge exit 0, store ${db}"
+}
+
+# store_path NAME: the database the running mcp-host opened, read from its one
+# "[ConversationStore] opened" log line (kubectl logs, no exec). mcp-host
+# writes the line after the worker's first answer, which can come after Pod
+# Ready, so the log is re-read for up to STORE_LOG_TIMEOUT seconds. Callers
+# run it inside $(...), so the reason goes to stderr and the caller records
+# the FAIL.
+store_path() {
+ local pod lines deadline=$((SECONDS + STORE_LOG_TIMEOUT))
+ pod="$(jq -r '.metadata.name' <<<"$POD_JSON")"
+ while :; do
+ lines="$(kctl logs "$pod" -n "$HOST_NS" -c mcp-host |
+ jq -Rr 'fromjson? | select(.msg == "[ConversationStore] opened") | .dbPath')" ||
+ { echo "${1}: could not read the mcp-host log of ${pod}" >&2; return 1; }
+ if [ -n "$lines" ]; then
+ [ "$(wc -l <<<"$lines" | tr -d ' ')" = 1 ] ||
+ { echo "${1}: expected one store-opened line in ${pod}, got: ${lines}" >&2; return 1; }
+ printf '%s' "$lines"
+ return 0
+ fi
+ [ "$SECONDS" -lt "$deadline" ] ||
+ { echo "${1}: expected one store-opened line in ${pod}, got: <none> after ${STORE_LOG_TIMEOUT}s" >&2; return 1; }
+ sleep 1
+ done
+}
+
+# ─── Conversations and the history oracle ─────────────────────────────
+# history_digest MESSAGES_JSON → "<turns> <sha256>". The oracle refuses an
+# empty catalog: zero turns is never a match.
+history_digest() {
+ local turns sum
+ turns="$(jq -r '.turns | length' <<<"$1")"
+ sum="$(jq -cS '[.turns[] | {number, user_input, response}]' <<<"$1" | shasum -a 256 | cut -d' ' -f1)"
+ printf '%s %s' "$turns" "$sum"
+}
+history_matches() { # EXPECTED_DIGEST MESSAGES_JSON
+ local actual
+ actual="$(history_digest "$2")"
+ [ "${actual%% *}" -gt 0 ] && [ "$actual" = "$1" ]
+}
+session_listed() { # SESSIONS_JSON HOST THREAD
+ jq -e --arg t "$3" --arg a "$2" '[.items[]? | select(.chatId == $t and .agent == $a)] | length == 1' <<<"$1" >/dev/null
+}
+# chat_set SESSIONS_JSON HOST → "<n>:<sorted chatIds>" of the Host's sessions.
+chat_set() {
+ jq -r --arg a "$2" '[.items[]? | select(.agent == $a) | .chatId] | unique | "\(length):\(join(","))"' <<<"$1"
+}
+# chats_match BEFORE AFTER: the same set on both sides, and not an empty one.
+chats_match() { [ "${1%%:*}" -gt 0 ] && [ "$1" = "$2" ]; }
+
+# fetch_messages records nothing; read_messages counts a non-200 as a failure.
+fetch_messages() { # HOST THREAD → HTTP_CODE, HTTP_BODY
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/$1/sessions/$1/$2/messages?limit=100" || true
+}
+read_messages() { # HOST THREAD → HTTP_BODY
+ fetch_messages "$1" "$2"
+ [ "$HTTP_CODE" = 200 ] || { fail "$1/$2: messages returned ${HTTP_CODE}"; return 1; }
+}
+
+# send_turn HOST THREAD CONTENT: a suspended Host answers 503 host_waking
+# while it wakes; retry only that, bounded by TURN_TIMEOUT. The turn counts
+# only once it is readable with its marker.
+send_turn() {
+ local host=$1 thread=$2 content=$3 elapsed=0 before
+ fetch_messages "$host" "$thread"
+ case "$HTTP_CODE" in
+ 200) before="$(jq -r '.turns | length' <<<"$HTTP_BODY")" ;;
+ 404) before=0 ;;
+ *) fail "${host}/${thread}: messages returned ${HTTP_CODE} before the turn"; return 1 ;;
+ esac
+ while :; do
+ rpc POST "${RPC_BASE}/api/v1/rpc/hosts/${host}/messages" \
+ "$(jq -cn --arg c "$content" --arg t "$thread" '{content:$c,threadId:$t}')" "$TURN_TIMEOUT" || true
+ [[ "$HTTP_CODE" =~ ^2 ]] && break
+ if [ "$HTTP_CODE" = 503 ] && jq -e '.code == "host_waking"' <<<"$HTTP_BODY" >/dev/null 2>&1 &&
+ [ "$elapsed" -lt "$TURN_TIMEOUT" ]; then
+ sleep 5
+ elapsed=$((elapsed + 5))
+ continue
+ fi
+ fail "${host}/${thread}: turn returned ${HTTP_CODE}: ${HTTP_BODY:0:300}"
+ return 1
+ done
+ elapsed=0
+ while [ "$elapsed" -lt "$TURN_TIMEOUT" ]; do
+ fetch_messages "$host" "$thread"
+ if [ "$HTTP_CODE" = 200 ] && jq -e --argjson n "$((before + 1))" --arg c "$content" \
+ '(.turns | length) == $n and (.turns[-1].user_input | contains($c))' <<<"$HTTP_BODY" >/dev/null; then
+ return 0
+ fi
+ sleep 3
+ elapsed=$((elapsed + 3))
+ done
+ fail "${host}/${thread}: turn ${before}+1 not persisted within ${TURN_TIMEOUT}s"
+ return 1
+}
+
+# seed_history HOST THREADS TURNS: writes conversations and records digests.
+seed_history() {
+ local host=$1 threads=$2 turns=$3 i j thread
+ : >"${SCRATCH}/${host}.threads"
+ for i in $(seq 1 "$threads"); do
+ thread="e2e-825-${host}-t${i}"
+ for j in $(seq 1 "$turns"); do
+ send_turn "$host" "$thread" "e2e-825 ${RUN} ${host} thread ${i} turn ${j}: reply with one short sentence." || return 1
+ done
+ read_messages "$host" "$thread" || return 1
+ printf '%s %s\n' "$thread" "$(history_digest "$HTTP_BODY")" >>"${SCRATCH}/${host}.threads"
+ done
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
+ chat_set "$HTTP_BODY" "$host" >"${SCRATCH}/${host}.chats"
+ ok "${host}: ${threads} conversation(s) x ${turns} turn(s) written"
+}
+
+# assert_history HOST TRANSITION: the Host lists the same chatId set as before
+# the transition; every recorded conversation is unchanged; then a resume turn
+# is persisted with the next turn number.
+assert_history() {
+ local host=$1 transition=$2 thread digest sessions before after next=''
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
+ sessions="$HTTP_BODY"
+ before="$(cat "${SCRATCH}/${host}.chats")"
+ after="$(chat_set "$sessions" "$host")"
+ chats_match "$before" "$after" ||
+ { fail "${host} ${transition}: chatId set before=${before} after=${after}"; return 1; }
+ echo "E2E_825_CONTINUITY ${host} ${transition} before=${before} after=${after} dbPath=$(cat "${SCRATCH}/${host}.dbpath")"
+ while read -r thread digest; do
+ session_listed "$sessions" "$host" "$thread" || { fail "${host}/${thread}: not in the session list"; return 1; }
+ read_messages "$host" "$thread" || return 1
+ history_matches "$digest" "$HTTP_BODY" ||
+ { fail "${host}/${thread}: history changed (expected ${digest}, got $(history_digest "$HTTP_BODY"))"; return 1; }
+ send_turn "$host" "$thread" "e2e-825 ${RUN} resume ${thread}: reply with one short sentence." || return 1
+ read_messages "$host" "$thread" || return 1
+ next+="${thread} $(history_digest "$HTTP_BODY")"$'\n'
+ done <"${SCRATCH}/${host}.threads"
+ printf '%s' "$next" >"${SCRATCH}/${host}.threads"
+ ok "${host}: history listed, unchanged and resumed ($(wc -l <"${SCRATCH}/${host}.threads" | tr -d ' ') conversation(s))"
+}
+
+# ─── FAULT:pvc-fixture ────────────────────────────────────────────────
+# pvc_fixture HOST ro|rw LABEL SCRIPT [ARGS...]: runs SCRIPT with the PVC root
+# as $1 in a Pod on the Host's last node. The Pod output goes to stdout; the
+# log and failure lines go to stderr so a caller capturing stdout reads only it.
+pvc_fixture() {
+ local host=$1 mode=$2 label=$3 script=$4 pod node ro body elapsed=0 phase
+ shift 4
+ pod="$(truncate_rfc1123 "e2e825-${label}-${host}")"
+ node="$(cat "${SCRATCH}/${host}.node")"
+ ro=false
+ [ "$mode" = ro ] && ro=true
+ log "FAULT:pvc-fixture ${label} on ${host}-workspace (${mode})" >&2
+ body="$(jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$FIXTURE_IMAGE" \
+ --arg claim "${host}-workspace" --argjson ro "$ro" --arg script "$script" \
+ --args '{
+ apiVersion:"v1", kind:"Pod",
+ metadata:{name:$pod, namespace:$ns, labels:{purpose:"e2e-825-pvc-fixture"}},
+ spec:{
+ restartPolicy:"Never", automountServiceAccountToken:false, enableServiceLinks:false,
+ terminationGracePeriodSeconds:0, nodeName:$node,
+ securityContext:{runAsUser:1001, runAsGroup:1001, runAsNonRoot:true, seccompProfile:{type:"RuntimeDefault"}},
+ containers:[{
+ name:"fixture", image:$image, imagePullPolicy:"IfNotPresent",
+ command:(["/bin/sh","-c",$script,"fixture","/mnt/pvc"] + $ARGS.positional),
+ resources:{requests:{cpu:"10m",memory:"32Mi"}, limits:{cpu:"200m",memory:"128Mi"}},
+ securityContext:{allowPrivilegeEscalation:false, readOnlyRootFilesystem:true, capabilities:{drop:["ALL"]}},
+ volumeMounts:[{name:"pvc", mountPath:"/mnt/pvc", readOnly:$ro}]
+ }],
+ volumes:[{name:"pvc", persistentVolumeClaim:{claimName:$claim, readOnly:$ro}}]
+ }}' "$@")"
+ # Callers usually run this function inside $(...), so an array append would
+ # be lost with the subshell; the registry file is what cleanup() reads.
+ # Registered before the create so a create that times out after the API
+ # accepted it is still cleaned up.
+ printf '%s\n' "$pod" >>"$FIXTURE_POD_REGISTRY"
+ kctl create -f - <<<"$body" >/dev/null || { fail "${label}: fixture Pod create failed" >&2; return 1; }
+ while [ "$elapsed" -lt 180 ]; do
+ phase="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+ case "$phase" in Succeeded | Failed) break ;; esac
+ sleep 2
+ elapsed=$((elapsed + 2))
+ done
+ kctl logs "$pod" -n "$HOST_NS" >"${SCRATCH}/${pod}.log" 2>&1 || true
+ # A Pod left behind keeps the RWO claim attached to its node and blocks the
+ # next fixture or Host Pod on another node, so a failed delete fails the step.
+ if ! kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then
+ fail "${label}: fixture Pod ${pod} delete failed" >&2
+ return 1
+ fi
+ if [ "$phase" != Succeeded ]; then
+ fail "${label}: fixture Pod ended ${phase:-<timeout>}: $(tail -n 5 "${SCRATCH}/${pod}.log")" >&2
+ return 1
+ fi
+ cat "${SCRATCH}/${pod}.log"
+}
+
+# Listing of the PVC root: `<type> <path>`, plus ` <target>` for a symlink,
+# lost+found pruned. The fixture image has busybox find, which lacks -printf.
+# shellcheck disable=SC2016 # expanded inside the fixture Pod
+LIST_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -print | while IFS= read -r p; do
+ if [ -L "$p" ]; then printf "l %s %s\n" "$p" "$(readlink "$p")"
+ elif [ -d "$p" ]; then printf "d %s\n" "$p"
+ else printf "f %s\n" "$p"; fi
+done | LC_ALL=C sort'
+# shellcheck disable=SC2016
+HASH_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum'
+# shellcheck disable=SC2016
+WRITE_FILE_SCRIPT='mkdir -p "$1/e2e-825" && printf "%s\n" "$2" > "$1/e2e-825/$3"'
+# shellcheck disable=SC2016
+READ_FILE_SCRIPT='cat "$1/e2e-825/$2"'
+
+read_seed_file() { # HOST NAME EXPECTED
+ local got
+ got="$(pvc_fixture "$1" ro read-file "$READ_FILE_SCRIPT" "$2")" || return 1
+ [ "$got" = "$3" ] || { fail "$1: seeded file e2e-825/$2 reads '${got}', expected '$3'"; return 1; }
+ ok "$1: seeded file e2e-825/$2 intact at the PVC root"
+}
+
+# ─── Operator script ──────────────────────────────────────────────────
+# layout_row inventory|verify HOST EXITS: runs deploy/scripts/host-layout-merge.sh
+# on this profile and prints the result of the Host's PVC row. EXITS is a case
+# pattern; the fleet verdict is left alone because other Hosts on the profile
+# decide it, and exit 2 (read failure) never matches.
+layout_row() {
+ local cmd=$1 host=$2 exits=$3 rc=0 out row
+ out="$(mktemp "${SCRATCH}/layout-${cmd}-${host}.XXXXXX")"
+ log "host-layout-merge.sh ${cmd} for ${HOST_NS}/${host}-workspace" >&2
+ bash "$LAYOUT_SCRIPT" "$cmd" --context "$E2E_KUBECONTEXT" >"$out" 2>"${out}.err" || rc=$?
+ # shellcheck disable=SC2254 # EXITS is a case pattern on purpose
+ case "$rc" in
+ $exits) ;;
+ *)
+ fail "${host}: host-layout-merge.sh ${cmd} exited ${rc} (expected ${exits}): $(tail -n 3 "${out}.err" | tr '\n' ' ') $(tail -n 3 "$out" | tr '\n' ' ')" >&2
+ return 1
+ ;;
+ esac
+ [ "$rc" != 2 ] || { fail "${host}: host-layout-merge.sh ${cmd} could not read the cluster" >&2; return 1; }
+ row="$(grep -F "PVC ${HOST_NS}/${host}-workspace " "$out")" ||
+ { fail "${host}: host-layout-merge.sh ${cmd} printed no row for ${host}-workspace" >&2; return 1; }
+ printf '%s\n' "${row#*result=}"
+}
+
+# ─── Suspend and wake ─────────────────────────────────────────────────
+wait_suspended() { # HOST: state suspended, replicas 0, zero Pods (Terminating included)
+ local host=$1 elapsed=0 state reps pods
+ log "FAULT:hcc-cadence waiting for ${host} to suspend"
+ while [ "$elapsed" -lt "$SUSPEND_TIMEOUT" ]; do
+ state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"
+ reps="$(kctl get deployment "$host" -n "$HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
+ pods="$(host_pods "$host" | jq '.items | length')" || pods=unknown
+ if [ "$state" = suspended ] && [ "$reps" = 0 ] && [ "$pods" = 0 ]; then
+ ok "${host}: suspended with zero Pods"
+ POD_UID=''
+ return 0
+ fi
+ sleep 5
+ elapsed=$((elapsed + 5))
+ done
+ fail "${host}: not suspended within ${SUSPEND_TIMEOUT}s (state=${state:-<none>} replicas=${reps:-?} pods=${pods})"
+ return 1
+}
+
+post_wake() { # HOST
+ rpc POST "${RPC_BASE}/api/v1/rpc/hosts/$1/wake" || true
+ case "$HTTP_CODE" in 200 | 202 | 204) log "wake requested for $1" ;; *)
+ fail "$1: wake returned ${HTTP_CODE}: ${HTTP_BODY}"
+ return 1
+ ;;
+ esac
+}
+
+set_hcc_cadence() {
+ local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES
+ CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS) k
+ HCC_ENV_SAVED=''
+ for k in "${keys[@]}"; do
+ HCC_ENV_SAVED+="${k}=$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}")"$'\n'
+ done
+ log 'FAULT:hcc-cadence idle 1 min, drain 20 s, heartbeat poll 5 s'
+ HCC_ENV_CHANGED=1
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=1 CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=1 \
+ CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=20000 CONTEXT_MAPPER_HEARTBEAT_POLL_MS=5000 >/dev/null ||
+ { fail 'hcc-cadence: kubectl set env failed'; return 1; }
+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||
+ { fail 'hcc-cadence: HCC rollout did not settle'; return 1; }
+}
+
+restore_hcc_cadence() {
+ local args=() key val actual
+ [ "$HCC_ENV_CHANGED" = 1 ] || return 0
+ while IFS='=' read -r key val; do
+ [ -n "$key" ] || continue
+ if [ -n "$val" ]; then args+=("${key}=${val}"); else args+=("${key}-"); fi
+ done <<<"$HCC_ENV_SAVED"
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null || return 1
+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null || return 1
+ while IFS='=' read -r key val; do
+ [ -n "$key" ] || continue
+ actual="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${key}')].value}")"
+ [ "$actual" = "$val" ] || { echo "HCC ${key} restored to '${actual}', expected '${val}'" >&2; return 1; }
+ done <<<"$HCC_ENV_SAVED"
+ HCC_ENV_CHANGED=0
+ log 'FAULT:hcc-cadence restored'
+}
diff --git a/scripts/e2e/e2e-hcc-layout-upgrade.sh b/scripts/e2e/e2e-hcc-layout-upgrade.sh
new file mode 100644
index 000000000..b50d7a2d8
--- /dev/null
+++ b/scripts/e2e/e2e-hcc-layout-upgrade.sh
@@ -0,0 +1,542 @@
+#!/usr/bin/env bash
+# Branch-owned live lane for #825: upgrade an installation from v0.11.0 to this
+# head and prove the Host PVC layout and history survive it (plan A7.40).
+#
+# Usage: e2e-hcc-layout-upgrade.sh main|control, each on a FRESH branch
+# profile (no host-context-controller yet). The launcher deletes and recreates
+# the profile between the two modes.
+#
+# main installs v0.11.0 (published images), seeds one Host per row, runs
+# deploy/scripts/host-layout-merge.sh inventory, upgrades to this
+# head (images built from this tree, database kept), then checks
+# verify, repairs T5 with the upgrade-note procedure and checks the
+# history of every row. Rows:
+# T1 stateful;
+# T2 stateless, active at the upgrade;
+# T3 stateless, suspended at the upgrade (woken afterwards);
+# T5 created stateless, used, switched to stateful by a
+# CommunicationChannel and used again (two live databases);
+# T6 stateless -> stateful -> stateless.
+# Prints E2E_UPGRADE_MAIN=PASS|FAIL and E2E_UPGRADE_HEAD=<sha>.
+# control the negative control: CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false
+# in the upgrade deploy. inventory must report
+# FLAG_FALSE_WITH_SPLIT_TREES and the history check must fail on a
+# T2-shaped Host. Needs E2E_UPGRADE_MAIN_EVIDENCE (the main log of
+# the same HEAD). Prints E2E_UPGRADE_RESULT=PASS only when main
+# passed and the control failed as expected.
+#
+# Install path: `make minikube-setup IMAGE_SOURCE=ghcr MINIKUBE_IMAGE_TAG=v0.11.0`
+# from this tree. That is the release's install only while this tree's
+# deploy/base, deploy/components, deploy/overlays and CRDs equal v0.11.0's;
+# the lane checks that and refuses otherwise.
+#
+# Cluster writes besides the setups and the admin API: the named fixtures of
+# e2e-hcc-storage-layout-switch.sh (hcc-cadence, pvc-fixture, channel,
+# host-layout-merge), the T5 repair (a pvc-fixture rename plus the deletion of
+# the refused Host Pod, as the upgrade note says) and, in control mode,
+# FAULT:flag-false (kubectl set env on the HCC Deployment, undone in the
+# trap). No kubectl exec into Host Pods, no SQLite reads or writes.
+set -euo pipefail
+umask 077
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+# shellcheck source=scripts/e2e/load-dotenv.sh
+source "${REPO_ROOT}/scripts/e2e/load-dotenv.sh"
+dotenv_load_canonical_root "${REPO_ROOT}"
+# shellcheck source=scripts/e2e/admin-credentials.sh
+source "${REPO_ROOT}/scripts/e2e/admin-credentials.sh"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+source "${SCRIPT_DIR}/e2e-lib.sh"
+
+die() { fail "$*"; exit 1; }
+
+MODE="${1:-}"
+case "$MODE" in main | control) ;; *) die 'usage: e2e-hcc-layout-upgrade.sh main|control' ;; esac
+
+RELEASE=v0.11.0
+# main_evidence_ok FILE HEAD: the main log passed on this same HEAD.
+main_evidence_ok() {
+ [ -r "$1" ] && grep -qx 'E2E_UPGRADE_MAIN=PASS' "$1" && grep -qx "E2E_UPGRADE_HEAD=$2" "$1"
+}
+
+# ─── Preconditions ────────────────────────────────────────────────────
+if [ -z "$E2E_KUBECONTEXT" ] || ! is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"; then
+ die 'select an explicit branch-scoped Minikube context (KUBECONTEXT=<profile>)'
+fi
+require_safe_kube_context
+[ "${E2E_ALLOW_STATELESS_CADENCE_ACCELERATION:-0}" = 1 ] ||
+ die 'set E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 (T3 shortens the HCC idle window on this owned profile)'
+[ -n "${E2E_PROFILE_PORTS_ENV:-}" ] && [ -r "$E2E_PROFILE_PORTS_ENV" ] ||
+ die 'provide E2E_PROFILE_PORTS_ENV from the owned branch profile'
+[ -n "${E2E_DEV_LOGIN_EMAIL:-}" ] || die 'set E2E_DEV_LOGIN_EMAIL to the seeded login ([email protected])'
+[ -n "${E2E_UPGRADE_EVIDENCE_DIR:-}" ] && [ -d "$E2E_UPGRADE_EVIDENCE_DIR" ] ||
+ die 'set E2E_UPGRADE_EVIDENCE_DIR to an existing directory for the setup logs'
+HEAD_SHA="$(git -C "$REPO_ROOT" rev-parse HEAD)"
+[ -z "$(git -C "$REPO_ROOT" status --porcelain)" ] ||
+ die 'the worktree is dirty; the head images would not be this HEAD'
+if [ "$MODE" = control ]; then
+ main_evidence_ok "${E2E_UPGRADE_MAIN_EVIDENCE:-}" "$HEAD_SHA" ||
+ die "E2E_UPGRADE_MAIN_EVIDENCE must be a main log with E2E_UPGRADE_MAIN=PASS and E2E_UPGRADE_HEAD=${HEAD_SHA}"
+fi
+git -C "$REPO_ROOT" rev-parse -q --verify "refs/tags/${RELEASE}" >/dev/null || die "tag ${RELEASE} is not in this clone"
+if ! git -C "$REPO_ROOT" diff --quiet "$RELEASE" HEAD -- deploy/base deploy/components deploy/overlays charts/clerum-crds; then
+ die "manifests or CRDs differ from ${RELEASE}; this tree no longer installs the release: $(git -C "$REPO_ROOT" diff --name-only "$RELEASE" HEAD -- deploy/base deploy/components deploy/overlays charts/clerum-crds | tr '\n' ' ')"
+fi
+profile_port_value() {
+ awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "$E2E_PROFILE_PORTS_ENV"
+}
+CONTROL_API_BASE_URL="$(profile_port_value CONTROL_API_URL)"
+CONTROL_UI_BASE_URL="$(profile_port_value CONTROL_UI_URL)"
+EXTERNAL_REST_API_BASE_URL="$(profile_port_value EXTERNAL_REST_API_URL)"
+RPC_PROXY_BASE_URL="$(profile_port_value RPC_PROXY_URL)"
+export CONTROL_API_BASE_URL CONTROL_UI_BASE_URL EXTERNAL_REST_API_BASE_URL RPC_PROXY_BASE_URL
+require_branch_profile_urls "$E2E_KUBECONTEXT" "$E2E_PROFILE_PORTS_ENV" ||
+ die 'profile URLs do not match the owned ports'
+kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"]==$c)' >/dev/null ||
+ die 'target is not the selected Minikube profile'
+
+HOST_NS=mcp-host CHANNEL_NS=channels
+HCC_NS=control-plane HCC_DEPLOY=host-context-controller
+SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm}"
+CAPI="${CONTROL_API_BASE_URL%/}/api/v1"
+EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"
+RPC_BASE="${RPC_PROXY_BASE_URL%/}"
+ADMIN_USERNAME="${ADMIN_USERNAME:-admin}"
+ADMIN_PASSWORD="${ADMIN_PASSWORD:-}"
+[ -n "$ADMIN_PASSWORD" ] || ADMIN_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"
+[ -n "$ADMIN_PASSWORD" ] || die 'admin password could not be resolved (ADMIN_PASSWORD or the canonical .env)'
+DEV_EMAIL="$E2E_DEV_LOGIN_EMAIL"
+DEV_PASSWORD="${E2E_USER_PASSWORD:-$ADMIN_PASSWORD}"
+RUN="$(od -An -N3 -tx1 /dev/urandom | tr -d ' \n')"
+POD_READY_TIMEOUT="${E2E_825_POD_READY_TIMEOUT:-300}"
+SUSPEND_TIMEOUT="${E2E_825_SUSPEND_TIMEOUT:-300}"
+TURN_TIMEOUT="${E2E_825_TURN_TIMEOUT:-240}"
+STORE_LOG_TIMEOUT="${E2E_825_STORE_LOG_TIMEOUT:-30}"
+LAYOUT_SCRIPT="${REPO_ROOT}/deploy/scripts/host-layout-merge.sh"
+[ -r "$LAYOUT_SCRIPT" ] || die "missing operator script ${LAYOUT_SCRIPT}"
+
+ROWS=(T1 T2 T3 T5 T6)
+ROWS_DONE=0 LANE_OK=0
+CREATED_HOSTS=() CREATED_CHANNELS=()
+HCC_ENV_SAVED='' HCC_ENV_CHANGED=0 FLAG_FALSE_SET=0
+SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/e2e-825-upgrade.XXXXXX")"
+FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"
+: >"$FIXTURE_POD_REGISTRY"
+ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID='' FIXTURE_IMAGE='' POD_UID=''
+HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()
+
+# shellcheck source=scripts/e2e/e2e-825-lib.sh
+source "${SCRIPT_DIR}/e2e-825-lib.sh"
+
+F2='workspace layout merge refused: more than one live SQLite database across'
+# The upgrade note's repair for a T5 tree, keeping the chats since the switch:
+# the database family under state/ is renamed, never deleted.
+# shellcheck disable=SC2016 # expanded inside the fixture Pod
+REPAIR_STATE_FAMILY_SCRIPT='set -eu; cd "$1/state"; [ -f state.db ]
+for s in "" -wal -shm; do [ -e "state.db$s" ] || continue; mv "state.db$s" "state.db.upgrade-lane.bak$s"; done; ls -la'
+# Seed files sit where the agent sees /workspace: the PVC root for a stateful
+# Host, PVC/workspace for a stateless one on the old release.
+# shellcheck disable=SC2016
+WRITE_AT_SCRIPT='mkdir -p "$1/$2/e2e-825" && printf "%s\n" "$3" > "$1/$2/e2e-825/$4"'
+# shellcheck disable=SC2016
+READ_AT_SCRIPT='cat "$1/$2/e2e-825/$3"'
+
+# run_logged LABEL CMD...: runs CMD from the repository root, its output in the
+# evidence directory, and reports the command's own exit code.
+run_logged() {
+ local label=$1 rc=0 logf
+ shift
+ logf="${E2E_UPGRADE_EVIDENCE_DIR}/${MODE}-${RUN}-${label}.log"
+ log "${label}: $* (log ${logf})"
+ (cd "$REPO_ROOT" && "$@") >"$logf" 2>&1 || rc=$?
+ echo "${label}_EXIT=${rc}" >>"$logf"
+ [ "$rc" = 0 ] || { fail "${label} exited ${rc}: $(tail -n 5 "$logf" | tr '\n' ' ')"; return 1; }
+ ok "${label} exit 0"
+}
+
+# Every setup restarts the services, so the forwards are re-held and probed.
+hold_forwards() {
+ local label=$1 key url code failures=0
+ run_logged "${label}-stop-pf" make -f scripts/minikube-profiles/branch.mk branch-profile-stop-pf || return 1
+ run_logged "${label}-pf" make -f scripts/minikube-profiles/branch.mk branch-profile-pf || return 1
+ for key in CONTROL_API_URL CONTROL_UI_URL EXTERNAL_REST_API_URL RPC_PROXY_URL; do
+ url="$(profile_port_value "$key")"
+ code="$(curl -sS -o /dev/null -m 6 -w '%{http_code}' "$url" 2>/dev/null)" || code=000
+ if [ "$code" = 000 ]; then
+ fail "forward ${key} (${url}) did not answer"
+ failures=$((failures + 1))
+ else
+ ok "forward ${key} answers HTTP ${code}"
+ fi
+ done
+ [ "$failures" = 0 ]
+}
+
+login_all() {
+ admin_login || return 1
+ admin GET "${CAPI}/admin/users?q=$(jq -rn --arg q "$DEV_EMAIL" '$q|@uri')" || true
+ USER_ID="$(jq -r --arg e "$DEV_EMAIL" '.items[]? | select((.email // "") == $e) | .id' <<<"$HTTP_BODY" | head -n1)"
+ [ -n "$USER_ID" ] || { fail "user ${DEV_EMAIL} not found (${HTTP_CODE})"; return 1; }
+ admin GET "${CAPI}/admin/users/${USER_ID}/teams" || true
+ TEAM_ID="$(jq -r '.items[0].id // empty' <<<"$HTTP_BODY")"
+ user_login || return 1
+ # bash 3.2 treats "${empty[@]}" as unbound under set -u.
+ [ "${#CREATED_HOSTS[@]}" -eq 0 ] || mint_rpc_token
+}
+
+hcc_image() {
+ kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}'
+}
+read_fixture_image() {
+ FIXTURE_IMAGE="$(kctl get deployment "$SOURCE_HOST" -n "$HOST_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].image}')"
+ [ -n "$FIXTURE_IMAGE" ] || { fail "could not read the ${SOURCE_HOST} image for the PVC fixture"; return 1; }
+}
+
+install_release() {
+ [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" --ignore-not-found -o name)" ] ||
+ { fail "deployment/${HCC_DEPLOY} already exists; the lane needs a fresh profile"; return 1; }
+ run_logged "install-${RELEASE}" make minikube-setup MINIKUBE_PROFILE="$E2E_KUBECONTEXT" \
+ IMAGE_SOURCE=ghcr MINIKUBE_IMAGE_TAG="$RELEASE" || return 1
+ case "$(hcc_image)" in
+ *":${RELEASE}") ok "HCC runs $(hcc_image)" ;;
+ *) fail "HCC runs '$(hcc_image)', expected the ${RELEASE} image"; return 1 ;;
+ esac
+ # Built now and loaded next to the release images, so the upgrade below is a
+ # deploy of minutes and the active Hosts stay active through it.
+ run_logged build-head-images make minikube-build-images MINIKUBE_PROFILE="$E2E_KUBECONTEXT" || return 1
+ hold_forwards release || return 1
+ login_all || return 1
+ SOURCE_HOST_JSON="$(kctl get host "$SOURCE_HOST" -n "$HOST_NS" -o json)" ||
+ { fail "source Host ${SOURCE_HOST} not found"; return 1; }
+ # Personalization writes IDENTITY.md and friends into the workspace; the
+ # clones would then collide on F3 instead of exercising the rows.
+ jq -e '.spec.personalization.enabled != true' <<<"$SOURCE_HOST_JSON" >/dev/null ||
+ { fail "source Host ${SOURCE_HOST} has personalization enabled"; return 1; }
+ # verify's fleet verdict below must be decided by the rows' Hosts alone.
+ kctl get hosts -n "$HOST_NS" -o json | jq -e '[.items[] | select(.spec.lifecycle.stateless == true)] | length == 0' >/dev/null ||
+ { fail 'the fresh install already has stateless Hosts'; return 1; }
+ read_fixture_image
+}
+
+upgrade_to_head() {
+ run_logged upgrade-to-head make minikube-setup MINIKUBE_PROFILE="$E2E_KUBECONTEXT" \
+ IMAGE_SOURCE=local REUSE_DB=true ARGS=--skip-build || return 1
+ case "$(hcc_image)" in
+ *":${RELEASE}") fail "HCC still runs $(hcc_image) after the upgrade"; return 1 ;;
+ *) ok "HCC runs $(hcc_image)" ;;
+ esac
+ hold_forwards head || return 1
+ login_all || return 1
+ read_fixture_image
+}
+
+HOST_T1='' HOST_T2='' HOST_T3='' HOST_T5='' HOST_T6=''
+new_host() { # ROW STATELESS: creates the row's Host, sets HOST_<row>
+ local name
+ name="$(truncate_rfc1123 "e2e-up-$(tr '[:upper:]' '[:lower:]' <<<"$1")-${RUN}")"
+ printf -v "HOST_$1" '%s' "$name"
+ create_host "$name" "$2" || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ printf '%s' "$POD_UID" >"${SCRATCH}/${name}.uid"
+}
+
+seed_file() { # HOST DIR(.|workspace)
+ pvc_fixture "$1" rw write-file "$WRITE_AT_SCRIPT" "$2" "upgrade ${RUN} $1" seed.txt >/dev/null
+}
+
+# snapshot_served HOST: what the Host serves now, the history oracle's
+# baseline. Every listed chat gets a digest; an empty catalog fails.
+snapshot_served() {
+ local host=$1 sessions chats chat digest
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
+ sessions="$HTTP_BODY"
+ chat_set "$sessions" "$host" >"${SCRATCH}/${host}.chats"
+ chats="$(jq -r --arg a "$host" '[.items[]? | select(.agent == $a) | .chatId] | unique | .[]' <<<"$sessions")"
+ [ -n "$chats" ] || { fail "${host}: serves no conversation before the upgrade"; return 1; }
+ : >"${SCRATCH}/${host}.threads"
+ for chat in $chats; do
+ read_messages "$host" "$chat" || return 1
+ digest="$(history_digest "$HTTP_BODY")"
+ [ "${digest%% *}" -gt 0 ] || { fail "${host}/${chat}: empty history"; return 1; }
+ printf '%s %s\n' "$chat" "$digest" >>"${SCRATCH}/${host}.threads"
+ done
+ ok "${host}: serves $(cat "${SCRATCH}/${host}.chats" | cut -d: -f1) conversation(s) before the upgrade"
+}
+
+# wait_upgraded_pod HOST: the Host's Pod after the upgrade (a new UID), waking
+# a stateless Host that suspended while the deploy ran.
+wait_upgraded_pod() {
+ local host=$1 state
+ state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"
+ if [ "$state" = suspended ]; then
+ log "${host} suspended during the upgrade; waking it"
+ post_wake "$host" || return 1
+ fi
+ wait_new_ready_pod "$host" "$(cat "${SCRATCH}/${host}.uid")" "$POD_READY_TIMEOUT"
+}
+
+# The head HCC renders the merge init on every Host Deployment it converged.
+wait_merge_templates() {
+ local host elapsed=0 pending
+ while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do
+ pending=''
+ for host in "$@"; do
+ kctl get deployment "$host" -n "$HOST_NS" -o json | jq -e '
+ [.spec.template.spec.initContainers[]?.name] == ["workspace-layout-merge"]' >/dev/null || pending+="${host} "
+ done
+ [ -n "$pending" ] || { ok "every row's Deployment renders workspace-layout-merge"; return 0; }
+ sleep 5
+ elapsed=$((elapsed + 5))
+ done
+ fail "Deployments not re-rendered by the head HCC within ${POD_READY_TIMEOUT}s: ${pending}"
+ return 1
+}
+
+# wait_refused_pod HOST → prints the Pod whose merge init exited 1.
+wait_refused_pod() {
+ local host=$1 elapsed=0 pod
+ while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do
+ pod="$(host_pods "$host" | jq -r '[.items[] | select(any(.status.initContainerStatuses[]?;
+ .name == "workspace-layout-merge" and ((.state.terminated.exitCode // 0) == 1 or (.lastState.terminated.exitCode // 0) == 1)))][0].metadata.name // ""')"
+ [ -z "$pod" ] || { printf '%s' "$pod"; return 0; }
+ sleep 3
+ elapsed=$((elapsed + 3))
+ done
+ echo "${host}: no Pod with workspace-layout-merge exit 1 within ${POD_READY_TIMEOUT}s" >&2
+ return 1
+}
+
+# run_layout CMD: one run of the operator script; output in LAYOUT_OUT, exit in LAYOUT_RC.
+run_layout() {
+ LAYOUT_OUT="${SCRATCH}/layout-$1-${SECONDS}.out"
+ LAYOUT_RC=0
+ bash "$LAYOUT_SCRIPT" "$1" --context "$E2E_KUBECONTEXT" >"$LAYOUT_OUT" 2>&1 || LAYOUT_RC=$?
+ cp "$LAYOUT_OUT" "${E2E_UPGRADE_EVIDENCE_DIR}/${MODE}-${RUN}-layout-$1-${SECONDS}.out"
+}
+row_of() { # HOST → the result of the Host's PVC row in LAYOUT_OUT
+ local row
+ row="$(grep -F "PVC ${HOST_NS}/${1}-workspace " "$LAYOUT_OUT")" || return 1
+ printf '%s' "${row#*result=}"
+}
+expect_row() { # HOST PATTERN LABEL
+ local row
+ row="$(row_of "$1")" || { fail "$3: no row for ${1}-workspace"; return 1; }
+ # shellcheck disable=SC2254 # PATTERN is a case pattern on purpose
+ case "$row" in $2) ok "$3: ${1}-workspace is ${row%% workspace layout*}" ;; *) fail "$3: ${1}-workspace is '${row}', expected $2"; return 1 ;; esac
+}
+
+# ─── main ─────────────────────────────────────────────────────────────
+seed_rows() {
+ # T3 first: the shortened idle window suspends every idle stateless Host,
+ # so the others are created after it is restored.
+ new_host T3 true && seed_history "$HOST_T3" 3 1 && seed_file "$HOST_T3" workspace && snapshot_served "$HOST_T3" || return 1
+ set_hcc_cadence && wait_suspended "$HOST_T3" && restore_hcc_cadence || return 1
+ new_host T1 false && seed_history "$HOST_T1" 3 1 && seed_file "$HOST_T1" . && snapshot_served "$HOST_T1" || return 1
+ # T5: the old history under state/, then a channel makes HCC run it
+ # stateful on the PVC root, where mcp-host opens a second database.
+ new_host T5 true && seed_history "$HOST_T5" 3 1 && seed_file "$HOST_T5" workspace || return 1
+ local channel
+ channel="$(truncate_rfc1123 "e2e-up-t5-${RUN}")"
+ log "FAULT:channel ${CHANNEL_NS}/${channel} → ${HOST_T5}"
+ printf '%s\n' 'apiVersion: clerum.io/v1alpha1' 'kind: CommunicationChannel' 'metadata:' \
+ " name: ${channel}" " namespace: ${CHANNEL_NS}" 'spec:' " hostRef: ${HOST_T5}" ' telegram:' \
+ " - channelId: \"e2e-up-${RUN}\"" ' chatType: "private"' ' userIds: ["000000000"]' |
+ kctl create -f - >/dev/null || { fail 'T5: CommunicationChannel create failed'; return 1; }
+ CREATED_CHANNELS+=("$channel")
+ wait_new_ready_pod "$HOST_T5" "$(cat "${SCRATCH}/${HOST_T5}.uid")" "$POD_READY_TIMEOUT" || return 1
+ printf '%s' "$POD_UID" >"${SCRATCH}/${HOST_T5}.uid"
+ seed_history "$HOST_T5" 1 1 && snapshot_served "$HOST_T5" || return 1
+ # T6: the release's stateless entry keeps the newer database, so what T6
+ # serves afterwards is what the snapshot records.
+ new_host T6 true && seed_history "$HOST_T6" 2 1 && seed_file "$HOST_T6" workspace || return 1
+ local stateless
+ for stateless in false true; do
+ set_lifecycle "$HOST_T6" "$stateless" || return 1
+ wait_new_ready_pod "$HOST_T6" "$(cat "${SCRATCH}/${HOST_T6}.uid")" "$POD_READY_TIMEOUT" || return 1
+ printf '%s' "$POD_UID" >"${SCRATCH}/${HOST_T6}.uid"
+ done
+ seed_history "$HOST_T6" 1 1 && snapshot_served "$HOST_T6" || return 1
+ # T2 last, so it is still active when the upgrade replaces its Pod.
+ new_host T2 true && seed_history "$HOST_T2" 3 1 && seed_file "$HOST_T2" workspace && snapshot_served "$HOST_T2"
+}
+
+predict_rows() {
+ run_layout inventory
+ [ "$LAYOUT_RC" = 1 ] || { fail "inventory exited ${LAYOUT_RC}, expected 1 (T5 blocks): $(tail -n 3 "$LAYOUT_OUT" | tr '\n' ' ')"; return 1; }
+ grep -qx 'UPGRADE_BLOCKERS=1' "$LAYOUT_OUT" || { fail "inventory: $(grep UPGRADE_BLOCKERS "$LAYOUT_OUT"), expected UPGRADE_BLOCKERS=1"; return 1; }
+ expect_row "$HOST_T5" "WILL_REFUSE_F2 ${F2}*" 'inventory T5' &&
+ expect_row "$HOST_T2" WILL_MERGE 'inventory T2' &&
+ expect_row "$HOST_T3" WILL_MERGE 'inventory T3' &&
+ expect_row "$HOST_T6" WILL_MERGE 'inventory T6' &&
+ expect_row "$HOST_T1" NO_SPLIT_SIGNATURE 'inventory T1'
+}
+
+check_upgraded() {
+ local pod logs host refused_uid
+ wait_merge_templates "$HOST_T1" "$HOST_T2" "$HOST_T3" "$HOST_T5" "$HOST_T6" || return 1
+ wait_upgraded_pod "$HOST_T1" && assert_pod "$HOST_T1" false /workspace || return 1
+ wait_upgraded_pod "$HOST_T2" && assert_pod "$HOST_T2" true /workspace || return 1
+ wait_upgraded_pod "$HOST_T6" && assert_pod "$HOST_T6" true /workspace || return 1
+ pod="$(wait_refused_pod "$HOST_T5")" || { fail 'T5: the merge did not refuse'; return 1; }
+ logs="$(kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge --previous 2>/dev/null ||
+ kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge)"
+ grep -q "$F2" <<<"$logs" || { fail "T5: merge refused without the F2 line: ${logs}"; return 1; }
+ [ "$(kctl get host "$HOST_T3" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}')" = suspended ] ||
+ { fail 'T3: no longer suspended before verify'; return 1; }
+ run_layout verify
+ [ "$LAYOUT_RC" = 1 ] || { fail "verify exited ${LAYOUT_RC}, expected 1 (T3 split, T5 refused)"; return 1; }
+ for host in "$HOST_T1" "$HOST_T2" "$HOST_T6"; do expect_row "$host" POD_OK 'verify' || return 1; done
+ expect_row "$HOST_T3" SPLIT 'verify T3' && expect_row "$HOST_T5" "MERGE_REFUSED ${F2}*" 'verify T5' || return 1
+
+ # T3 merges on its natural wake.
+ post_wake "$HOST_T3" && wait_new_ready_pod "$HOST_T3" '' "$POD_READY_TIMEOUT" && assert_pod "$HOST_T3" true /workspace || return 1
+ # T5: the upgrade note's repair, keeping the chats since the switch.
+ log "REPAIR:${HOST_T5} rename the state/ database family (upgrade note)"
+ pvc_fixture "$HOST_T5" rw repair "$REPAIR_STATE_FAMILY_SCRIPT" >/dev/null || return 1
+ refused_uid="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.metadata.uid}')" || { fail "T5: could not read refused Pod ${pod}"; return 1; }
+ kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null || { fail "T5: delete of refused Pod ${pod} failed"; return 1; }
+ wait_new_ready_pod "$HOST_T5" "$refused_uid" "$POD_READY_TIMEOUT" &&
+ assert_pod "$HOST_T5" false /workspace || return 1
+
+ run_layout verify
+ if [ "$LAYOUT_RC" != 0 ] || ! grep -qx 'HOST_LAYOUT_VERIFY=PASS' "$LAYOUT_OUT"; then
+ fail "verify after the repairs exited ${LAYOUT_RC}: $(tail -n 3 "$LAYOUT_OUT" | tr '\n' ' ')"
+ return 1
+ fi
+ for host in "$HOST_T1" "$HOST_T2" "$HOST_T3" "$HOST_T5" "$HOST_T6"; do expect_row "$host" POD_OK 'verify after repair' || return 1; done
+}
+
+check_history() {
+ local listing
+ assert_history "$HOST_T1" "upgrade(stateful)" && read_seed_file "$HOST_T1" seed.txt "upgrade ${RUN} ${HOST_T1}" || return 1
+ ROWS_DONE=$((ROWS_DONE + 1))
+ assert_history "$HOST_T2" "upgrade(stateless-active)" && read_seed_file "$HOST_T2" seed.txt "upgrade ${RUN} ${HOST_T2}" || return 1
+ ROWS_DONE=$((ROWS_DONE + 1))
+ assert_history "$HOST_T3" "upgrade(stateless-suspended)->wake" && read_seed_file "$HOST_T3" seed.txt "upgrade ${RUN} ${HOST_T3}" || return 1
+ ROWS_DONE=$((ROWS_DONE + 1))
+ # T5 serves the root family it served before the upgrade; the old family is
+ # kept as a backup, and its old files stay under workspace/ (no signature).
+ assert_history "$HOST_T5" "upgrade(two-databases)->repair" || return 1
+ listing="$(pvc_fixture "$HOST_T5" ro list-repaired "$LIST_SCRIPT")" || return 1
+ grep -qx 'f ./state/state.db.upgrade-lane.bak' <<<"$listing" ||
+ { fail "T5: the renamed family is missing: ${listing}"; return 1; }
+ [ "$(pvc_fixture "$HOST_T5" ro read-file "$READ_AT_SCRIPT" workspace seed.txt)" = "upgrade ${RUN} ${HOST_T5}" ] ||
+ { fail 'T5: the old seed file is not under workspace/'; return 1; }
+ ok 'T5: old family kept as state/state.db.upgrade-lane.bak, old files under workspace/'
+ ROWS_DONE=$((ROWS_DONE + 1))
+ assert_history "$HOST_T6" "upgrade(stateless-stateful-stateless)" && read_seed_file "$HOST_T6" seed.txt "upgrade ${RUN} ${HOST_T6}" || return 1
+ ROWS_DONE=$((ROWS_DONE + 1))
+}
+
+run_main() {
+ header "Install ${RELEASE}" && install_release &&
+ header 'Seed rows on the release' && seed_rows &&
+ header 'Predict with inventory' && predict_rows &&
+ header 'Upgrade to this head' && upgrade_to_head &&
+ header 'Verify and repair' && check_upgraded &&
+ header 'History of every row' && check_history
+}
+
+# ─── control ──────────────────────────────────────────────────────────
+run_control() {
+ local flag sessions before after
+ header "Install ${RELEASE}" && install_release || return 1
+ header 'Seed a T2-shaped Host'
+ new_host T2 true && seed_history "$HOST_T2" 2 1 && snapshot_served "$HOST_T2" || return 1
+ header 'Upgrade with the merge flag off'
+ # Set on the release HCC, which ignores it; the setup's client-side apply
+ # keeps an env entry it never owned, so the head HCC starts with "false".
+ log "FAULT:flag-false CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false on deployment/${HCC_DEPLOY}"
+ FLAG_FALSE_SET=1
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false >/dev/null ||
+ { fail 'flag-false: kubectl set env failed'; return 1; }
+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||
+ { fail 'flag-false: HCC rollout did not complete'; return 1; }
+ upgrade_to_head || return 1
+ flag="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" -o json | jq -r '
+ [.spec.template.spec.containers[].env[]? | select(.name == "CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE") | .value] | first // "<unset>"')"
+ [ "$flag" = false ] || { fail "control: the head HCC runs with the flag '${flag}', the control did not happen"; return 1; }
+ ok 'control: the head HCC runs with CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false'
+ wait_upgraded_pod "$HOST_T2" || return 1
+ jq -e '.inits == [] and .mounts == [{mountPath:"/workspace", name:"workspace"}]' <<<"$(pod_storage)" >/dev/null ||
+ { fail "control: T2 Pod storage is $(pod_storage), expected the root mount without the merge init"; return 1; }
+ [ "$(store_path "$HOST_T2")" = /workspace/state.db ] || { fail 'control: T2 did not open /workspace/state.db'; return 1; }
+ ok 'control: T2 starts on the root mount with no merge init'
+ run_layout inventory
+ if [ "$LAYOUT_RC" != 1 ] || ! grep -q '^FLAG_FALSE_WITH_SPLIT_TREES=' "$LAYOUT_OUT"; then
+ fail "control: inventory exited ${LAYOUT_RC} without FLAG_FALSE_WITH_SPLIT_TREES"
+ return 1
+ fi
+ ok "control: inventory reports $(grep '^FLAG_FALSE_WITH_SPLIT_TREES=' "$LAYOUT_OUT")"
+ # The oracle has to run (sessions 200) and then disagree with the baseline.
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${HOST_T2}/sessions?agent=${HOST_T2}&limit=100" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "control: sessions returned ${HTTP_CODE}"; return 1; }
+ sessions="$HTTP_BODY"
+ before="$(cat "${SCRATCH}/${HOST_T2}.chats")"
+ after="$(chat_set "$sessions" "$HOST_T2")"
+ if chats_match "$before" "$after"; then
+ fail "control: T2 still serves its history (${after}); the control did not fail"
+ return 1
+ fi
+ echo "E2E_UPGRADE_CONTROL_DIVERGED ${HOST_T2} before=${before} after=${after}"
+ ok 'control: history check fails with the flag off, as expected'
+}
+
+# ─── Cleanup ──────────────────────────────────────────────────────────
+cleanup() {
+ local status=$? failed=0 name pod channel
+ set +eu
+ while IFS= read -r pod; do
+ kctl delete pod "$pod" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ done <"$FIXTURE_POD_REGISTRY"
+ for channel in "${CREATED_CHANNELS[@]}"; do
+ kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ done
+ restore_hcc_cadence || { failed=1; echo "HCC cadence restore failed; check deployment/${HCC_DEPLOY}" >&2; }
+ if [ "$FLAG_FALSE_SET" = 1 ]; then
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE- >/dev/null 2>&1 || failed=1
+ fi
+ for name in "${CREATED_HOSTS[@]}"; do
+ admin DELETE "${CAPI}/admin/hosts/${name}" >/dev/null 2>&1
+ case "$HTTP_CODE" in 200 | 404) ;; *) failed=1; echo "delete Host ${name} returned ${HTTP_CODE}" >&2 ;; esac
+ [ -z "$USER_ID" ] || set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
+ [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
+ done
+ for name in "${CREATED_HOSTS[@]}"; do
+ kctl wait --for=delete "deployment/${name}" -n "$HOST_NS" --timeout=120s >/dev/null 2>&1
+ kctl delete pvc "${name}-workspace" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
+ done
+ rm -rf "$SCRATCH"
+ [ "$failed" = 0 ] || { status=1; fail 'cleanup left resources behind (see stderr)'; }
+ print_results || status=1
+ echo "E2E_UPGRADE_HEAD=${HEAD_SHA}"
+ if [ "$MODE" = main ]; then
+ echo "E2E_UPGRADE_ROWS_EXECUTED=${ROWS_DONE} E2E_UPGRADE_ROWS_DEFINED=${#ROWS[@]}"
+ if [ "$status" = 0 ] && [ "$LANE_OK" = 1 ] && [ "$ROWS_DONE" = "${#ROWS[@]}" ]; then
+ echo 'E2E_UPGRADE_MAIN=PASS'
+ else
+ echo 'E2E_UPGRADE_MAIN=FAIL'
+ status=1
+ fi
+ else
+ if [ "$status" = 0 ] && [ "$LANE_OK" = 1 ]; then
+ echo 'E2E_UPGRADE_CONTROL=FAILED_AS_EXPECTED'
+ echo 'E2E_UPGRADE_RESULT=PASS'
+ else
+ echo 'E2E_UPGRADE_CONTROL=NOT_PROVEN'
+ echo 'E2E_UPGRADE_RESULT=FAIL'
+ status=1
+ fi
+ fi
+ echo "E2E_UPGRADE_EXIT=${status}"
+ exit "$status"
+}
+trap cleanup EXIT
+
+header "#825 upgrade lane ${RELEASE} → ${HEAD_SHA:0:9} (${MODE}, run ${RUN}, ${E2E_KUBECONTEXT})"
+if [ "$MODE" = main ]; then run_main && LANE_OK=1; else run_control && LANE_OK=1; fi
diff --git a/scripts/e2e/e2e-hcc-storage-layout-switch.sh b/scripts/e2e/e2e-hcc-storage-layout-switch.sh
index 619e40d09..8a5534f9c 100755
--- a/scripts/e2e/e2e-hcc-storage-layout-switch.sh
+++ b/scripts/e2e/e2e-hcc-storage-layout-switch.sh
@@ -99,227 +99,8 @@ FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"
ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID=''
HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()
-# ─── HTTP ─────────────────────────────────────────────────────────────
-# http METHOD URL [BODY] [MAX_SECONDS] → HTTP_CODE, HTTP_BODY. A transport
-# failure is HTTP_CODE=000 and a non-zero return. The body goes to curl on
-# stdin, never in argv, because login bodies carry passwords.
-http() {
- local method=$1 url=$2 body=${3:-} max=${4:-60} resp
- local args=(-sS -m "$max" -w '\n%{http_code}' -X "$method" "$url")
- [ ${#HTTP_HEADERS[@]} -eq 0 ] || args+=("${HTTP_HEADERS[@]}")
- [ -z "$body" ] || args+=(-H 'Content-Type: application/json' --data-binary @-)
- if ! resp="$(printf '%s' "$body" | curl "${args[@]}")"; then
- HTTP_CODE=000 HTTP_BODY=''
- return 1
- fi
- HTTP_CODE="$(tail -n1 <<<"$resp")"
- HTTP_BODY="$(sed '$d' <<<"$resp")"
-}
-# auth_header KIND LINE: write one header line to a 0600 file under $SCRATCH
-# and print its path. Credentials reach curl through `-H @file`, never argv,
-# where any local user can read them with ps.
-auth_header() {
- local file="${SCRATCH}/${1}.header"
- (umask 077 && printf '%s\n' "$2" >"$file")
- printf '%s' "$file"
-}
-admin() { HTTP_HEADERS=(-H "@$(auth_header admin "Cookie: control_ui_admin_session=${ADMIN_COOKIE}")"); http "$@"; }
-rpc() { HTTP_HEADERS=(-H "@$(auth_header rpc "Authorization: Bearer ${RPC_TOKEN}")"); http "$@"; }
-
-admin_login() {
- local headers="${SCRATCH}/admin-login.headers" line cookie
- HTTP_HEADERS=(-D "$headers")
- http POST "${CAPI}/admin/auth/login" \
- "$(jq -cn --arg u "$ADMIN_USERNAME" --arg p "$ADMIN_PASSWORD" '{username:$u,password:$p}')" || true
- [[ "$HTTP_CODE" =~ ^2 ]] || { fail "admin login returned ${HTTP_CODE}"; return 1; }
- while IFS= read -r line; do
- line="${line%$'\r'}"
- case "$(tr '[:upper:]' '[:lower:]' <<<"$line")" in
- set-cookie:*control_ui_admin_session=*)
- cookie="${line#*control_ui_admin_session=}"
- ADMIN_COOKIE="${cookie%%;*}"
- ;;
- esac
- done <"$headers"
- rm -f "$headers"
- [ -n "$ADMIN_COOKIE" ] || { fail 'admin login set no control_ui_admin_session cookie'; return 1; }
-}
-
-user_login() {
- HTTP_HEADERS=()
- http POST "${EXT_BASE}/api/v1/auth/password-login" \
- "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')" || true
- [ "$HTTP_CODE" = 200 ] || { fail "password login for ${DEV_EMAIL} returned ${HTTP_CODE} (a 429 means the lockout window; wait, do not retry blindly)"; return 1; }
- SESSION_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'password login returned no token'; return 1; }
-}
-
-# The rpc token names every Host this run created so far.
-mint_rpc_token() {
- local refs
- refs="$(printf '%s\n' "${CREATED_HOSTS[@]}" | jq -R . | jq -cs .)"
- HTTP_HEADERS=(-H "@$(auth_header session "Authorization: Bearer ${SESSION_TOKEN}")")
- http POST "${EXT_BASE}/api/v1/rpc/token" "$(jq -cn --argjson refs "$refs" '{hostRefs:$refs,scopes:[
- "host:message:invoke","host:session:read","host:status:read","host:health:read",
- "host:wake:write","host:model:write"]}')" || true
- [[ "$HTTP_CODE" =~ ^2 ]] || { fail "rpc token mint returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
- RPC_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'rpc token response has no token'; return 1; }
-}
-
-# ─── Host lifecycle through the admin API ─────────────────────────────
-# PUT /admin/users/:id/agents filters names against the active agent set, so a
-# just-created Host can lag; retry until the response echoes it back.
-set_agent_grant() {
- local url=$1 name=$2 op=$3 attempt=1 names expected
- while :; do
- admin GET "$url" || true
- [ "$HTTP_CODE" = 200 ] || { fail "GET ${url} returned ${HTTP_CODE}"; return 1; }
- expected="$(jq -c '((.agentNames // []) + (.deletedAgentNames // []))' <<<"$HTTP_BODY")"
- if [ "$op" = add ]; then
- names="$(jq -c --arg a "$name" '((.agentNames // []) + [$a]) | unique' <<<"$HTTP_BODY")"
- else
- names="$(jq -c --arg a "$name" '(.agentNames // []) - [$a]' <<<"$HTTP_BODY")"
- fi
- admin PUT "$url" "$(jq -cn --argjson n "$names" --argjson e "$expected" \
- '{agentNames:$n,expectedCurrentAgentNames:$e}')" || true
- if [[ "$HTTP_CODE" =~ ^2 ]]; then
- if [ "$op" = remove ] ||
- jq -e --arg a "$name" '(.agentNames // []) | index($a) != null' <<<"$HTTP_BODY" >/dev/null; then
- return 0
- fi
- fi
- [ "$attempt" -lt 20 ] || { fail "${op} grant '${name}' on ${url} did not converge (${HTTP_CODE})"; return 1; }
- attempt=$((attempt + 1))
- sleep 3
- done
-}
-
-grant_host() {
- set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$1" add || return 1
- [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$1" add
-}
-
-# create_host NAME STATELESS(true|false): clones the seeded source Host's spec
-# (context, secret, model) without desktop or channels and creates it via the API.
-create_host() {
- local name=$1 stateless=$2 spec
- spec="$(jq -c --arg n "$name" --argjson s "$stateless" '
- .spec + {host:$n} | del(.channels) | del(.desktop)
- | .lifecycle = {stateless:$s}' <<<"$SOURCE_HOST_JSON")"
- admin POST "${CAPI}/admin/hosts" "$(jq -cn --arg n "$name" --argjson s "$spec" '{metadata:{name:$n},spec:$s}')" || true
- [ "$HTTP_CODE" = 201 ] || { fail "create Host ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
- CREATED_HOSTS+=("$name")
- grant_host "$name" || return 1
- mint_rpc_token
-}
-
-# set_lifecycle NAME STATELESS(true|false): GET then PUT with the read's
-# resourceVersion, echoing the full spec as Control UI does.
-set_lifecycle() {
- local name=$1 stateless=$2 rv spec
- admin GET "${CAPI}/admin/hosts/${name}" || true
- [ "$HTTP_CODE" = 200 ] || { fail "GET Host ${name} returned ${HTTP_CODE}"; return 1; }
- rv="$(jq -er '.metadata.resourceVersion' <<<"$HTTP_BODY")" || { fail "Host ${name} read has no resourceVersion"; return 1; }
- spec="$(jq -c --argjson s "$stateless" '.spec | .lifecycle = {stateless:$s}' <<<"$HTTP_BODY")"
- admin PUT "${CAPI}/admin/hosts/${name}" \
- "$(jq -cn --arg rv "$rv" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
- [ "$HTTP_CODE" = 200 ] || { fail "PUT lifecycle stateless=${stateless} on ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
- log "lifecycle request on ${name}: stateless=${stateless}"
-}
-
-# ─── Pods and the lifecycle witness ───────────────────────────────────
-host_pods() { kctl get pods -n "$HOST_NS" -l "app=$1" -o json; }
-
-# wait_new_ready_pod NAME OLD_UID TIMEOUT: exactly one Pod exists, it is Ready,
-# and its UID differs from OLD_UID. Sets POD_JSON, POD_UID.
-wait_new_ready_pod() {
- local name=$1 old=$2 timeout=$3 elapsed=0 pods
- while [ "$elapsed" -lt "$timeout" ]; do
- if pods="$(host_pods "$name")" && POD_JSON="$(jq -ce --arg old "$old" '
- .items | select(length == 1) | .[0]
- | select(.metadata.deletionTimestamp == null and .metadata.uid != $old)
- | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))' <<<"$pods")"; then
- POD_UID="$(jq -r '.metadata.uid' <<<"$POD_JSON")"
- jq -r '.spec.nodeName' <<<"$POD_JSON" >"${SCRATCH}/${name}.node"
- return 0
- fi
- sleep 3
- elapsed=$((elapsed + 3))
- done
- fail "${name}: no new Ready Pod within ${timeout}s"
- kctl get pods -n "$HOST_NS" -l "app=${name}" -o wide >&2 || true
- kctl describe pods -n "$HOST_NS" -l "app=${name}" 2>&1 | tail -n 40 >&2 || true
- return 1
-}
-
-# Storage seen by the running Pod: workspace mounts, storage env, init names.
-pod_storage() {
- jq -cS '{
- mounts: [.spec.containers[] | select(.name == "mcp-host") | .volumeMounts[] | select(.name == "workspace")],
- store: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_STORE") | .value] | first),
- dbDir: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_DB_DIR") | .value] | first),
- inits: [.spec.initContainers[]? | .name]
- }' <<<"$POD_JSON"
-}
-
-# assert_pod NAME STATELESS WORKSPACE_PATH: lifecycle markers of the new Pod
-# match the requested lifecycle and its storage is the one root layout.
-assert_pod() {
- local name=$1 stateless=$2 path=$3 env_flag priority merge_exit storage db
- env_flag="$(jq -r '[.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") | .value] | first // ""' <<<"$POD_JSON")"
- priority="$(jq -r '.spec.priorityClassName // ""' <<<"$POD_JSON")"
- if [ "$stateless" = true ]; then
- [ "$env_flag" = true ] && [ "$priority" = clerum-interactive-host ] ||
- { fail "${name}: Pod is not stateless (env=${env_flag:-<none>} priority=${priority:-<none>})"; return 1; }
- else
- [ -z "$env_flag" ] && [ "$priority" != clerum-interactive-host ] ||
- { fail "${name}: Pod is not stateful (env=${env_flag} priority=${priority})"; return 1; }
- fi
- storage="$(pod_storage)"
- jq -e --arg p "$path" '
- .mounts == [{mountPath:$p, name:"workspace"}] and .store == "sqlite"
- and .dbDir == $p and .inits == ["workspace-layout-merge"]' <<<"$storage" >/dev/null ||
- { fail "${name}: Pod storage is not the root layout: ${storage}"; return 1; }
- merge_exit="$(jq -r '[.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge") | .state.terminated.exitCode] | first // "none"' <<<"$POD_JSON")"
- [ "$merge_exit" = 0 ] || { fail "${name}: workspace-layout-merge exit ${merge_exit}"; return 1; }
- if [ -s "${SCRATCH}/${name}.storage" ] && [ "$(cat "${SCRATCH}/${name}.storage")" != "$storage" ]; then
- fail "${name}: storage changed across the switch: $(cat "${SCRATCH}/${name}.storage") -> ${storage}"
- return 1
- fi
- printf '%s' "$storage" >"${SCRATCH}/${name}.storage"
- db="$(store_path "$name")" || { fail "${name}: no store-opened log line (reason above)"; return 1; }
- [ "$db" = "${path}/state.db" ] || { fail "${name}: mcp-host opened ${db}, expected ${path}/state.db"; return 1; }
- if [ -s "${SCRATCH}/${name}.dbpath" ] && [ "$(cat "${SCRATCH}/${name}.dbpath")" != "$db" ]; then
- fail "${name}: store changed across the switch: $(cat "${SCRATCH}/${name}.dbpath") -> ${db}"
- return 1
- fi
- printf '%s' "$db" >"${SCRATCH}/${name}.dbpath"
- ok "${name}: Pod ${POD_UID:0:8} stateless=${stateless}, root layout at ${path}, merge exit 0, store ${db}"
-}
-
-# store_path NAME: the database the running mcp-host opened, read from its one
-# "[ConversationStore] opened" log line (kubectl logs, no exec). mcp-host
-# writes the line after the worker's first answer, which can come after Pod
-# Ready, so the log is re-read for up to STORE_LOG_TIMEOUT seconds. Callers
-# run it inside $(...), so the reason goes to stderr and the caller records
-# the FAIL.
-store_path() {
- local pod lines deadline=$((SECONDS + STORE_LOG_TIMEOUT))
- pod="$(jq -r '.metadata.name' <<<"$POD_JSON")"
- while :; do
- lines="$(kctl logs "$pod" -n "$HOST_NS" -c mcp-host |
- jq -Rr 'fromjson? | select(.msg == "[ConversationStore] opened") | .dbPath')" ||
- { echo "${1}: could not read the mcp-host log of ${pod}" >&2; return 1; }
- if [ -n "$lines" ]; then
- [ "$(wc -l <<<"$lines" | tr -d ' ')" = 1 ] ||
- { echo "${1}: expected one store-opened line in ${pod}, got: ${lines}" >&2; return 1; }
- printf '%s' "$lines"
- return 0
- fi
- [ "$SECONDS" -lt "$deadline" ] ||
- { echo "${1}: expected one store-opened line in ${pod}, got: <none> after ${STORE_LOG_TIMEOUT}s" >&2; return 1; }
- sleep 1
- done
-}
+# shellcheck source=scripts/e2e/e2e-825-lib.sh
+source "${SCRIPT_DIR}/e2e-825-lib.sh"
# switch_and_witness NAME STATELESS: request the switch, then require a new
# Pod with the requested lifecycle and the same storage as before.
@@ -330,191 +111,6 @@ switch_and_witness() {
assert_pod "$name" "$stateless" /workspace
}
-# ─── Conversations and the history oracle ─────────────────────────────
-# history_digest MESSAGES_JSON → "<turns> <sha256>". The oracle refuses an
-# empty catalog: zero turns is never a match.
-history_digest() {
- local turns sum
- turns="$(jq -r '.turns | length' <<<"$1")"
- sum="$(jq -cS '[.turns[] | {number, user_input, response}]' <<<"$1" | shasum -a 256 | cut -d' ' -f1)"
- printf '%s %s' "$turns" "$sum"
-}
-history_matches() { # EXPECTED_DIGEST MESSAGES_JSON
- local actual
- actual="$(history_digest "$2")"
- [ "${actual%% *}" -gt 0 ] && [ "$actual" = "$1" ]
-}
-session_listed() { # SESSIONS_JSON HOST THREAD
- jq -e --arg t "$3" --arg a "$2" '[.items[]? | select(.chatId == $t and .agent == $a)] | length == 1' <<<"$1" >/dev/null
-}
-# chat_set SESSIONS_JSON HOST → "<n>:<sorted chatIds>" of the Host's sessions.
-chat_set() {
- jq -r --arg a "$2" '[.items[]? | select(.agent == $a) | .chatId] | unique | "\(length):\(join(","))"' <<<"$1"
-}
-# chats_match BEFORE AFTER: the same set on both sides, and not an empty one.
-chats_match() { [ "${1%%:*}" -gt 0 ] && [ "$1" = "$2" ]; }
-
-# fetch_messages records nothing; read_messages counts a non-200 as a failure.
-fetch_messages() { # HOST THREAD → HTTP_CODE, HTTP_BODY
- rpc GET "${RPC_BASE}/api/v1/rpc/hosts/$1/sessions/$1/$2/messages?limit=100" || true
-}
-read_messages() { # HOST THREAD → HTTP_BODY
- fetch_messages "$1" "$2"
- [ "$HTTP_CODE" = 200 ] || { fail "$1/$2: messages returned ${HTTP_CODE}"; return 1; }
-}
-
-# send_turn HOST THREAD CONTENT: a suspended Host answers 503 host_waking
-# while it wakes; retry only that, bounded by TURN_TIMEOUT. The turn counts
-# only once it is readable with its marker.
-send_turn() {
- local host=$1 thread=$2 content=$3 elapsed=0 before
- fetch_messages "$host" "$thread"
- case "$HTTP_CODE" in
- 200) before="$(jq -r '.turns | length' <<<"$HTTP_BODY")" ;;
- 404) before=0 ;;
- *) fail "${host}/${thread}: messages returned ${HTTP_CODE} before the turn"; return 1 ;;
- esac
- while :; do
- rpc POST "${RPC_BASE}/api/v1/rpc/hosts/${host}/messages" \
- "$(jq -cn --arg c "$content" --arg t "$thread" '{content:$c,threadId:$t}')" "$TURN_TIMEOUT" || true
- [[ "$HTTP_CODE" =~ ^2 ]] && break
- if [ "$HTTP_CODE" = 503 ] && jq -e '.code == "host_waking"' <<<"$HTTP_BODY" >/dev/null 2>&1 &&
- [ "$elapsed" -lt "$TURN_TIMEOUT" ]; then
- sleep 5
- elapsed=$((elapsed + 5))
- continue
- fi
- fail "${host}/${thread}: turn returned ${HTTP_CODE}: ${HTTP_BODY:0:300}"
- return 1
- done
- elapsed=0
- while [ "$elapsed" -lt "$TURN_TIMEOUT" ]; do
- fetch_messages "$host" "$thread"
- if [ "$HTTP_CODE" = 200 ] && jq -e --argjson n "$((before + 1))" --arg c "$content" \
- '(.turns | length) == $n and (.turns[-1].user_input | contains($c))' <<<"$HTTP_BODY" >/dev/null; then
- return 0
- fi
- sleep 3
- elapsed=$((elapsed + 3))
- done
- fail "${host}/${thread}: turn ${before}+1 not persisted within ${TURN_TIMEOUT}s"
- return 1
-}
-
-# seed_history HOST THREADS TURNS: writes conversations and records digests.
-seed_history() {
- local host=$1 threads=$2 turns=$3 i j thread
- : >"${SCRATCH}/${host}.threads"
- for i in $(seq 1 "$threads"); do
- thread="e2e-825-${host}-t${i}"
- for j in $(seq 1 "$turns"); do
- send_turn "$host" "$thread" "e2e-825 ${RUN} ${host} thread ${i} turn ${j}: reply with one short sentence." || return 1
- done
- read_messages "$host" "$thread" || return 1
- printf '%s %s\n' "$thread" "$(history_digest "$HTTP_BODY")" >>"${SCRATCH}/${host}.threads"
- done
- rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
- [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
- chat_set "$HTTP_BODY" "$host" >"${SCRATCH}/${host}.chats"
- ok "${host}: ${threads} conversation(s) x ${turns} turn(s) written"
-}
-
-# assert_history HOST TRANSITION: the Host lists the same chatId set as before
-# the transition; every recorded conversation is unchanged; then a resume turn
-# is persisted with the next turn number.
-assert_history() {
- local host=$1 transition=$2 thread digest sessions before after next=''
- rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
- [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
- sessions="$HTTP_BODY"
- before="$(cat "${SCRATCH}/${host}.chats")"
- after="$(chat_set "$sessions" "$host")"
- chats_match "$before" "$after" ||
- { fail "${host} ${transition}: chatId set before=${before} after=${after}"; return 1; }
- echo "E2E_825_CONTINUITY ${host} ${transition} before=${before} after=${after} dbPath=$(cat "${SCRATCH}/${host}.dbpath")"
- while read -r thread digest; do
- session_listed "$sessions" "$host" "$thread" || { fail "${host}/${thread}: not in the session list"; return 1; }
- read_messages "$host" "$thread" || return 1
- history_matches "$digest" "$HTTP_BODY" ||
- { fail "${host}/${thread}: history changed (expected ${digest}, got $(history_digest "$HTTP_BODY"))"; return 1; }
- send_turn "$host" "$thread" "e2e-825 ${RUN} resume ${thread}: reply with one short sentence." || return 1
- read_messages "$host" "$thread" || return 1
- next+="${thread} $(history_digest "$HTTP_BODY")"$'\n'
- done <"${SCRATCH}/${host}.threads"
- printf '%s' "$next" >"${SCRATCH}/${host}.threads"
- ok "${host}: history listed, unchanged and resumed ($(wc -l <"${SCRATCH}/${host}.threads" | tr -d ' ') conversation(s))"
-}
-
-# ─── FAULT:pvc-fixture ────────────────────────────────────────────────
-# pvc_fixture HOST ro|rw LABEL SCRIPT [ARGS...]: runs SCRIPT with the PVC root
-# as $1 in a Pod on the Host's last node. The Pod output goes to stdout; the
-# log and failure lines go to stderr so a caller capturing stdout reads only it.
-pvc_fixture() {
- local host=$1 mode=$2 label=$3 script=$4 pod node ro body elapsed=0 phase
- shift 4
- pod="$(truncate_rfc1123 "e2e825-${label}-${host}")"
- node="$(cat "${SCRATCH}/${host}.node")"
- ro=false
- [ "$mode" = ro ] && ro=true
- log "FAULT:pvc-fixture ${label} on ${host}-workspace (${mode})" >&2
- body="$(jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$FIXTURE_IMAGE" \
- --arg claim "${host}-workspace" --argjson ro "$ro" --arg script "$script" \
- --args '{
- apiVersion:"v1", kind:"Pod",
- metadata:{name:$pod, namespace:$ns, labels:{purpose:"e2e-825-pvc-fixture"}},
- spec:{
- restartPolicy:"Never", automountServiceAccountToken:false, enableServiceLinks:false,
- terminationGracePeriodSeconds:0, nodeName:$node,
- securityContext:{runAsUser:1001, runAsGroup:1001, runAsNonRoot:true, seccompProfile:{type:"RuntimeDefault"}},
- containers:[{
- name:"fixture", image:$image, imagePullPolicy:"IfNotPresent",
- command:(["/bin/sh","-c",$script,"fixture","/mnt/pvc"] + $ARGS.positional),
- resources:{requests:{cpu:"10m",memory:"32Mi"}, limits:{cpu:"200m",memory:"128Mi"}},
- securityContext:{allowPrivilegeEscalation:false, readOnlyRootFilesystem:true, capabilities:{drop:["ALL"]}},
- volumeMounts:[{name:"pvc", mountPath:"/mnt/pvc", readOnly:$ro}]
- }],
- volumes:[{name:"pvc", persistentVolumeClaim:{claimName:$claim, readOnly:$ro}}]
- }}' "$@")"
- # Callers usually run this function inside $(...), so an array append would
- # be lost with the subshell; the registry file is what cleanup() reads.
- # Registered before the create so a create that times out after the API
- # accepted it is still cleaned up.
- printf '%s\n' "$pod" >>"$FIXTURE_POD_REGISTRY"
- kctl create -f - <<<"$body" >/dev/null || { fail "${label}: fixture Pod create failed" >&2; return 1; }
- while [ "$elapsed" -lt 180 ]; do
- phase="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
- case "$phase" in Succeeded | Failed) break ;; esac
- sleep 2
- elapsed=$((elapsed + 2))
- done
- kctl logs "$pod" -n "$HOST_NS" >"${SCRATCH}/${pod}.log" 2>&1 || true
- # A Pod left behind keeps the RWO claim attached to its node and blocks the
- # next fixture or Host Pod on another node, so a failed delete fails the step.
- if ! kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then
- fail "${label}: fixture Pod ${pod} delete failed" >&2
- return 1
- fi
- if [ "$phase" != Succeeded ]; then
- fail "${label}: fixture Pod ended ${phase:-<timeout>}: $(tail -n 5 "${SCRATCH}/${pod}.log")" >&2
- return 1
- fi
- cat "${SCRATCH}/${pod}.log"
-}
-
-# Listing of the PVC root: `<type> <path>`, plus ` <target>` for a symlink,
-# lost+found pruned. The fixture image has busybox find, which lacks -printf.
-# shellcheck disable=SC2016 # expanded inside the fixture Pod
-LIST_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -print | while IFS= read -r p; do
- if [ -L "$p" ]; then printf "l %s %s\n" "$p" "$(readlink "$p")"
- elif [ -d "$p" ]; then printf "d %s\n" "$p"
- else printf "f %s\n" "$p"; fi
-done | LC_ALL=C sort'
-# shellcheck disable=SC2016
-HASH_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum'
-# shellcheck disable=SC2016
-WRITE_FILE_SCRIPT='mkdir -p "$1/e2e-825" && printf "%s\n" "$2" > "$1/e2e-825/$3"'
-# shellcheck disable=SC2016
-READ_FILE_SCRIPT='cat "$1/e2e-825/$2"'
# The inventoried dev split shape: DB family and a backup under state/, every
# other root entry under workspace/, plus an agent-made symlink (A7.16).
# shellcheck disable=SC2016
@@ -533,103 +129,6 @@ ls -la state workspace'
SEED_TWO_DB_SCRIPT='set -eu; cd "$1"; mkdir state workspace; cp state.db state/state.db
printf "notes\n" > state/notes.txt; printf "x\n" > workspace/x; ls -la . state workspace'
-read_seed_file() { # HOST NAME EXPECTED
- local got
- got="$(pvc_fixture "$1" ro read-file "$READ_FILE_SCRIPT" "$2")" || return 1
- [ "$got" = "$3" ] || { fail "$1: seeded file e2e-825/$2 reads '${got}', expected '$3'"; return 1; }
- ok "$1: seeded file e2e-825/$2 intact at the PVC root"
-}
-
-# ─── Operator script ──────────────────────────────────────────────────
-# layout_row inventory|verify HOST EXITS: runs deploy/scripts/host-layout-merge.sh
-# on this profile and prints the result of the Host's PVC row. EXITS is a case
-# pattern; the fleet verdict is left alone because other Hosts on the profile
-# decide it, and exit 2 (read failure) never matches.
-layout_row() {
- local cmd=$1 host=$2 exits=$3 rc=0 out row
- out="$(mktemp "${SCRATCH}/layout-${cmd}-${host}.XXXXXX")"
- log "host-layout-merge.sh ${cmd} for ${HOST_NS}/${host}-workspace" >&2
- bash "$LAYOUT_SCRIPT" "$cmd" --context "$E2E_KUBECONTEXT" >"$out" 2>"${out}.err" || rc=$?
- # shellcheck disable=SC2254 # EXITS is a case pattern on purpose
- case "$rc" in
- $exits) ;;
- *)
- fail "${host}: host-layout-merge.sh ${cmd} exited ${rc} (expected ${exits}): $(tail -n 3 "${out}.err" | tr '\n' ' ') $(tail -n 3 "$out" | tr '\n' ' ')" >&2
- return 1
- ;;
- esac
- [ "$rc" != 2 ] || { fail "${host}: host-layout-merge.sh ${cmd} could not read the cluster" >&2; return 1; }
- row="$(grep -F "PVC ${HOST_NS}/${host}-workspace " "$out")" ||
- { fail "${host}: host-layout-merge.sh ${cmd} printed no row for ${host}-workspace" >&2; return 1; }
- printf '%s\n' "${row#*result=}"
-}
-
-# ─── Suspend and wake ─────────────────────────────────────────────────
-wait_suspended() { # HOST: state suspended, replicas 0, zero Pods (Terminating included)
- local host=$1 elapsed=0 state reps pods
- log "FAULT:hcc-cadence waiting for ${host} to suspend"
- while [ "$elapsed" -lt "$SUSPEND_TIMEOUT" ]; do
- state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"
- reps="$(kctl get deployment "$host" -n "$HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
- pods="$(host_pods "$host" | jq '.items | length')" || pods=unknown
- if [ "$state" = suspended ] && [ "$reps" = 0 ] && [ "$pods" = 0 ]; then
- ok "${host}: suspended with zero Pods"
- POD_UID=''
- return 0
- fi
- sleep 5
- elapsed=$((elapsed + 5))
- done
- fail "${host}: not suspended within ${SUSPEND_TIMEOUT}s (state=${state:-<none>} replicas=${reps:-?} pods=${pods})"
- return 1
-}
-
-post_wake() { # HOST
- rpc POST "${RPC_BASE}/api/v1/rpc/hosts/$1/wake" || true
- case "$HTTP_CODE" in 200 | 202 | 204) log "wake requested for $1" ;; *)
- fail "$1: wake returned ${HTTP_CODE}: ${HTTP_BODY}"
- return 1
- ;;
- esac
-}
-
-set_hcc_cadence() {
- local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES
- CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS) k
- HCC_ENV_SAVED=''
- for k in "${keys[@]}"; do
- HCC_ENV_SAVED+="${k}=$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
- -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}")"$'\n'
- done
- log 'FAULT:hcc-cadence idle 1 min, drain 20 s, heartbeat poll 5 s'
- HCC_ENV_CHANGED=1
- kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
- CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=1 CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=1 \
- CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=20000 CONTEXT_MAPPER_HEARTBEAT_POLL_MS=5000 >/dev/null ||
- { fail 'hcc-cadence: kubectl set env failed'; return 1; }
- kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||
- { fail 'hcc-cadence: HCC rollout did not settle'; return 1; }
-}
-
-restore_hcc_cadence() {
- local args=() key val actual
- [ "$HCC_ENV_CHANGED" = 1 ] || return 0
- while IFS='=' read -r key val; do
- [ -n "$key" ] || continue
- if [ -n "$val" ]; then args+=("${key}=${val}"); else args+=("${key}-"); fi
- done <<<"$HCC_ENV_SAVED"
- kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null || return 1
- kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null || return 1
- while IFS='=' read -r key val; do
- [ -n "$key" ] || continue
- actual="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
- -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${key}')].value}")"
- [ "$actual" = "$val" ] || { echo "HCC ${key} restored to '${actual}', expected '${val}'" >&2; return 1; }
- done <<<"$HCC_ENV_SAVED"
- HCC_ENV_CHANGED=0
- log 'FAULT:hcc-cadence restored'
-}
-
# A new stateless Host with history, a seeded file, then suspended.
suspended_host_with_history() { # NAME
create_host "$1" true || return 1
diff --git a/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs b/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs
index d8901d587..57ac45c5c 100644
--- a/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs
+++ b/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs
@@ -6,11 +6,16 @@ import path from 'node:path'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
-// Runs single helpers of e2e-hcc-storage-layout-switch.sh under bash with
-// stubbed kubectl/curl, so their failure paths are checked without a cluster.
+// Runs single helpers of the #825 live gates (the switch script, the upgrade
+// lane and their shared e2e-825-lib.sh) under bash with stubbed kubectl/curl,
+// so their failure paths are checked without a cluster.
-const SCRIPT = fileURLToPath(new URL('./e2e-hcc-storage-layout-switch.sh', import.meta.url))
-const SOURCE = readFileSync(SCRIPT, 'utf8')
+const SCRIPTS = [
+ './e2e-hcc-storage-layout-switch.sh',
+ './e2e-825-lib.sh',
+ './e2e-hcc-layout-upgrade.sh',
+].map(rel => fileURLToPath(new URL(rel, import.meta.url)))
+const SOURCE = SCRIPTS.map(file => readFileSync(file, 'utf8')).join('\n')
function helper(name) {
const lines = SOURCE.split('\n')
@@ -113,7 +118,74 @@ admin GET http://127.0.0.1:1/admin && rpc GET http://127.0.0.1:2/rpc && echo "CO
}
})
-test('no curl call in the script puts a Cookie or Authorization header in argv', () => {
+test('main_evidence_ok accepts only a passing main log of the same HEAD', () => {
+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-825-upgrade-evidence-'))
+ try {
+ const head = 'a'.repeat(40)
+ const write = (name, lines) => {
+ const file = path.join(dir, name)
+ writeFileSync(file, `${lines.join('\n')}\n`)
+ return file
+ }
+ const cases = [
+ ['pass', write('pass', ['PASS: x', 'E2E_UPGRADE_HEAD=' + head, 'E2E_UPGRADE_MAIN=PASS']), 0],
+ ['fail', write('fail', ['E2E_UPGRADE_HEAD=' + head, 'E2E_UPGRADE_MAIN=FAIL']), 1],
+ [
+ 'other head',
+ write('other', ['E2E_UPGRADE_HEAD=' + 'b'.repeat(40), 'E2E_UPGRADE_MAIN=PASS']),
+ 1,
+ ],
+ [
+ 'head prefix only',
+ write('prefix', ['E2E_UPGRADE_HEAD=' + head.slice(0, 9), 'E2E_UPGRADE_MAIN=PASS']),
+ 1,
+ ],
+ ['missing file', path.join(dir, 'absent'), 1],
+ ]
+ const body = `
+${helper('main_evidence_ok')}
+for f in "$@"; do main_evidence_ok "$f" "$HEAD" && echo 0 || echo 1; done
+`
+ const run = spawnSync('bash', ['-c', body, 'bash', ...cases.map(c => c[1])], {
+ encoding: 'utf8',
+ env: { PATH: process.env.PATH, HEAD: head },
+ })
+ assert.equal(run.status, 0, run.stderr)
+ assert.deepEqual(
+ run.stdout.trim().split('\n').map(Number),
+ cases.map(c => c[2]),
+ cases.map(c => c[0]).join(', ')
+ )
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+})
+
+function runUpgrade(args, env) {
+ return spawnSync('bash', [SCRIPTS[2], ...args], {
+ encoding: 'utf8',
+ env: { PATH: process.env.PATH, HOME: process.env.HOME, ...env },
+ timeout: 30_000,
+ })
+}
+
+test('the upgrade lane refuses an unknown mode before touching the cluster', () => {
+ const run = runUpgrade(['bogus'], { KUBECONTEXT: 'clerum-test-upgrade-lane-0123abcd' })
+ assert.equal(run.status, 1, run.stdout + run.stderr)
+ assert.match(run.stdout + run.stderr, /usage: e2e-hcc-layout-upgrade\.sh main\|control/)
+ // Witness: a valid mode passes the mode check and stops at a later precondition.
+ const valid = runUpgrade(['main'], { KUBECONTEXT: 'minikube' })
+ assert.equal(valid.status, 1, valid.stdout + valid.stderr)
+ assert.doesNotMatch(valid.stdout + valid.stderr, /usage:/)
+})
+
+test('the upgrade lane refuses a context that is not a branch profile', () => {
+ const run = runUpgrade(['main'], { KUBECONTEXT: 'minikube' })
+ assert.equal(run.status, 1, run.stdout + run.stderr)
+ assert.match(run.stdout + run.stderr, /select an explicit branch-scoped Minikube context/)
+})
+
+test('no curl call in the #825 scripts puts a Cookie or Authorization header in argv', () => {
assert.doesNotMatch(SOURCE, /-H\s+["']?(Cookie|Authorization):/i)
// Witness: the credentials are still sent, through auth_header.
const uses = SOURCE.match(/auth_header (admin|session|rpc) /g) ?? []