Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): upgrade from v0.11.0 to this head
Alfredo Lopez228ab974fc55
diff --git a/Makefile b/Makefileindex 850fcc2e6..456eddc65 100644--- a/Makefile+++ b/Makefile@@ -1406,6 +1406,11 @@ test-e2e-hcc-storage-layout-switch: ## Run the #825 storage-layout gate across l@echo "Running HCC storage-layout switch E2E gate..."E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" bash scripts/e2e/e2e-hcc-storage-layout-switch.sh+.PHONY: test-e2e-hcc-layout-upgrade+test-e2e-hcc-layout-upgrade: ## Run the #825 upgrade lane from v0.11.0 to this head (UPGRADE_MODE=main|control; fresh owned branch profile, E2E_PROFILE_PORTS_ENV, E2E_DEV_LOGIN_EMAIL, E2E_UPGRADE_EVIDENCE_DIR)+ @echo "Running HCC layout upgrade E2E lane ($(UPGRADE_MODE))..."+ E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" bash scripts/e2e/e2e-hcc-layout-upgrade.sh "$(UPGRADE_MODE)"+.PHONY: test-e2e-825-desktop-continuitytest-e2e-825-desktop-continuity: ## Run the #825 Desktop journey across stateful/stateless switches (owned branch profile, forwards held, real model turns)@set -eu; \diff --git a/scripts/e2e/e2e-825-lib.sh b/scripts/e2e/e2e-825-lib.shnew file mode 100644index 000000000..0fe55ab2f--- /dev/null+++ b/scripts/e2e/e2e-825-lib.sh@@ -0,0 +1,512 @@+# shellcheck shell=bash+# Helpers shared by the #825 live gates (e2e-hcc-storage-layout-switch.sh and+# e2e-hcc-layout-upgrade.sh). Sourced after e2e-lib.sh; defines functions and+# fixture scripts only. The caller sets HOST_NS, HCC_NS, HCC_DEPLOY, CAPI,+# EXT_BASE, RPC_BASE, ADMIN_USERNAME, ADMIN_PASSWORD, DEV_EMAIL, DEV_PASSWORD,+# SOURCE_HOST_JSON, SCRATCH, FIXTURE_POD_REGISTRY, FIXTURE_IMAGE, RUN,+# LAYOUT_SCRIPT, the timeouts, and the state below.++# ─── HTTP ─────────────────────────────────────────────────────────────+# http METHOD URL [BODY] [MAX_SECONDS] → HTTP_CODE, HTTP_BODY. A transport+# failure is HTTP_CODE=000 and a non-zero return. The body goes to curl on+# stdin, never in argv, because login bodies carry passwords.+http() {+ local method=$1 url=$2 body=${3:-} max=${4:-60} resp+ local args=(-sS -m "$max" -w '\n%{http_code}' -X "$method" "$url")+ [ ${#HTTP_HEADERS[@]} -eq 0 ] || args+=("${HTTP_HEADERS[@]}")+ [ -z "$body" ] || args+=(-H 'Content-Type: application/json' --data-binary @-)+ if ! resp="$(printf '%s' "$body" | curl "${args[@]}")"; then+ HTTP_CODE=000 HTTP_BODY=''+ return 1+ fi+ HTTP_CODE="$(tail -n1 <<<"$resp")"+ HTTP_BODY="$(sed '$d' <<<"$resp")"+}+# auth_header KIND LINE: write one header line to a 0600 file under $SCRATCH+# and print its path. Credentials reach curl through `-H @file`, never argv,+# where any local user can read them with ps.+auth_header() {+ local file="${SCRATCH}/${1}.header"+ (umask 077 && printf '%s\n' "$2" >"$file")+ printf '%s' "$file"+}+admin() { HTTP_HEADERS=(-H "@$(auth_header admin "Cookie: control_ui_admin_session=${ADMIN_COOKIE}")"); http "$@"; }+rpc() { HTTP_HEADERS=(-H "@$(auth_header rpc "Authorization: Bearer ${RPC_TOKEN}")"); http "$@"; }++admin_login() {+ local headers="${SCRATCH}/admin-login.headers" line cookie+ HTTP_HEADERS=(-D "$headers")+ http POST "${CAPI}/admin/auth/login" \+ "$(jq -cn --arg u "$ADMIN_USERNAME" --arg p "$ADMIN_PASSWORD" '{username:$u,password:$p}')" || true+ [[ "$HTTP_CODE" =~ ^2 ]] || { fail "admin login returned ${HTTP_CODE}"; return 1; }+ while IFS= read -r line; do+ line="${line%$'\r'}"+ case "$(tr '[:upper:]' '[:lower:]' <<<"$line")" in+ set-cookie:*control_ui_admin_session=*)+ cookie="${line#*control_ui_admin_session=}"+ ADMIN_COOKIE="${cookie%%;*}"+ ;;+ esac+ done <"$headers"+ rm -f "$headers"+ [ -n "$ADMIN_COOKIE" ] || { fail 'admin login set no control_ui_admin_session cookie'; return 1; }+}++user_login() {+ HTTP_HEADERS=()+ http POST "${EXT_BASE}/api/v1/auth/password-login" \+ "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')" || true+ [ "$HTTP_CODE" = 200 ] || { fail "password login for ${DEV_EMAIL} returned ${HTTP_CODE} (a 429 means the lockout window; wait, do not retry blindly)"; return 1; }+ SESSION_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'password login returned no token'; return 1; }+}++# The rpc token names every Host this run created so far.+mint_rpc_token() {+ local refs+ refs="$(printf '%s\n' "${CREATED_HOSTS[@]}" | jq -R . | jq -cs .)"+ HTTP_HEADERS=(-H "@$(auth_header session "Authorization: Bearer ${SESSION_TOKEN}")")+ http POST "${EXT_BASE}/api/v1/rpc/token" "$(jq -cn --argjson refs "$refs" '{hostRefs:$refs,scopes:[+ "host:message:invoke","host:session:read","host:status:read","host:health:read",+ "host:wake:write","host:model:write"]}')" || true+ [[ "$HTTP_CODE" =~ ^2 ]] || { fail "rpc token mint returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }+ RPC_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'rpc token response has no token'; return 1; }+}++# ─── Host lifecycle through the admin API ─────────────────────────────+# PUT /admin/users/:id/agents filters names against the active agent set, so a+# just-created Host can lag; retry until the response echoes it back.+set_agent_grant() {+ local url=$1 name=$2 op=$3 attempt=1 names expected+ while :; do+ admin GET "$url" || true+ [ "$HTTP_CODE" = 200 ] || { fail "GET ${url} returned ${HTTP_CODE}"; return 1; }+ expected="$(jq -c '((.agentNames // []) + (.deletedAgentNames // []))' <<<"$HTTP_BODY")"+ if [ "$op" = add ]; then+ names="$(jq -c --arg a "$name" '((.agentNames // []) + [$a]) | unique' <<<"$HTTP_BODY")"+ else+ names="$(jq -c --arg a "$name" '(.agentNames // []) - [$a]' <<<"$HTTP_BODY")"+ fi+ admin PUT "$url" "$(jq -cn --argjson n "$names" --argjson e "$expected" \+ '{agentNames:$n,expectedCurrentAgentNames:$e}')" || true+ if [[ "$HTTP_CODE" =~ ^2 ]]; then+ if [ "$op" = remove ] ||+ jq -e --arg a "$name" '(.agentNames // []) | index($a) != null' <<<"$HTTP_BODY" >/dev/null; then+ return 0+ fi+ fi+ [ "$attempt" -lt 20 ] || { fail "${op} grant '${name}' on ${url} did not converge (${HTTP_CODE})"; return 1; }+ attempt=$((attempt + 1))+ sleep 3+ done+}++grant_host() {+ set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$1" add || return 1+ [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$1" add+}++# create_host NAME STATELESS(true|false): clones the seeded source Host's spec+# (context, secret, model) without desktop or channels and creates it via the API.+create_host() {+ local name=$1 stateless=$2 spec+ spec="$(jq -c --arg n "$name" --argjson s "$stateless" '+ .spec + {host:$n} | del(.channels) | del(.desktop)+ | .lifecycle = {stateless:$s}' <<<"$SOURCE_HOST_JSON")"+ admin POST "${CAPI}/admin/hosts" "$(jq -cn --arg n "$name" --argjson s "$spec" '{metadata:{name:$n},spec:$s}')" || true+ [ "$HTTP_CODE" = 201 ] || { fail "create Host ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }+ CREATED_HOSTS+=("$name")+ grant_host "$name" || return 1+ mint_rpc_token+}++# set_lifecycle NAME STATELESS(true|false): GET then PUT with the read's+# resourceVersion, echoing the full spec as Control UI does.+set_lifecycle() {+ local name=$1 stateless=$2 rv spec+ admin GET "${CAPI}/admin/hosts/${name}" || true+ [ "$HTTP_CODE" = 200 ] || { fail "GET Host ${name} returned ${HTTP_CODE}"; return 1; }+ rv="$(jq -er '.metadata.resourceVersion' <<<"$HTTP_BODY")" || { fail "Host ${name} read has no resourceVersion"; return 1; }+ spec="$(jq -c --argjson s "$stateless" '.spec | .lifecycle = {stateless:$s}' <<<"$HTTP_BODY")"+ admin PUT "${CAPI}/admin/hosts/${name}" \+ "$(jq -cn --arg rv "$rv" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true+ [ "$HTTP_CODE" = 200 ] || { fail "PUT lifecycle stateless=${stateless} on ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }+ log "lifecycle request on ${name}: stateless=${stateless}"+}++# ─── Pods and the lifecycle witness ───────────────────────────────────+host_pods() { kctl get pods -n "$HOST_NS" -l "app=$1" -o json; }++# wait_new_ready_pod NAME OLD_UID TIMEOUT: exactly one Pod exists, it is Ready,+# and its UID differs from OLD_UID. Sets POD_JSON, POD_UID.+wait_new_ready_pod() {+ local name=$1 old=$2 timeout=$3 elapsed=0 pods+ while [ "$elapsed" -lt "$timeout" ]; do+ if pods="$(host_pods "$name")" && POD_JSON="$(jq -ce --arg old "$old" '+ .items | select(length == 1) | .[0]+ | select(.metadata.deletionTimestamp == null and .metadata.uid != $old)+ | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))' <<<"$pods")"; then+ POD_UID="$(jq -r '.metadata.uid' <<<"$POD_JSON")"+ jq -r '.spec.nodeName' <<<"$POD_JSON" >"${SCRATCH}/${name}.node"+ return 0+ fi+ sleep 3+ elapsed=$((elapsed + 3))+ done+ fail "${name}: no new Ready Pod within ${timeout}s"+ kctl get pods -n "$HOST_NS" -l "app=${name}" -o wide >&2 || true+ kctl describe pods -n "$HOST_NS" -l "app=${name}" 2>&1 | tail -n 40 >&2 || true+ return 1+}++# Storage seen by the running Pod: workspace mounts, storage env, init names.+pod_storage() {+ jq -cS '{+ mounts: [.spec.containers[] | select(.name == "mcp-host") | .volumeMounts[] | select(.name == "workspace")],+ store: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_STORE") | .value] | first),+ dbDir: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_DB_DIR") | .value] | first),+ inits: [.spec.initContainers[]? | .name]+ }' <<<"$POD_JSON"+}++# assert_pod NAME STATELESS WORKSPACE_PATH: lifecycle markers of the new Pod+# match the requested lifecycle and its storage is the one root layout.+assert_pod() {+ local name=$1 stateless=$2 path=$3 env_flag priority merge_exit storage db+ env_flag="$(jq -r '[.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") | .value] | first // ""' <<<"$POD_JSON")"+ priority="$(jq -r '.spec.priorityClassName // ""' <<<"$POD_JSON")"+ if [ "$stateless" = true ]; then+ [ "$env_flag" = true ] && [ "$priority" = clerum-interactive-host ] ||+ { fail "${name}: Pod is not stateless (env=${env_flag:-<none>} priority=${priority:-<none>})"; return 1; }+ else+ [ -z "$env_flag" ] && [ "$priority" != clerum-interactive-host ] ||+ { fail "${name}: Pod is not stateful (env=${env_flag} priority=${priority})"; return 1; }+ fi+ storage="$(pod_storage)"+ jq -e --arg p "$path" '+ .mounts == [{mountPath:$p, name:"workspace"}] and .store == "sqlite"+ and .dbDir == $p and .inits == ["workspace-layout-merge"]' <<<"$storage" >/dev/null ||+ { fail "${name}: Pod storage is not the root layout: ${storage}"; return 1; }+ merge_exit="$(jq -r '[.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge") | .state.terminated.exitCode] | first // "none"' <<<"$POD_JSON")"+ [ "$merge_exit" = 0 ] || { fail "${name}: workspace-layout-merge exit ${merge_exit}"; return 1; }+ if [ -s "${SCRATCH}/${name}.storage" ] && [ "$(cat "${SCRATCH}/${name}.storage")" != "$storage" ]; then+ fail "${name}: storage changed across the switch: $(cat "${SCRATCH}/${name}.storage") -> ${storage}"+ return 1+ fi+ printf '%s' "$storage" >"${SCRATCH}/${name}.storage"+ db="$(store_path "$name")" || { fail "${name}: no store-opened log line (reason above)"; return 1; }+ [ "$db" = "${path}/state.db" ] || { fail "${name}: mcp-host opened ${db}, expected ${path}/state.db"; return 1; }+ if [ -s "${SCRATCH}/${name}.dbpath" ] && [ "$(cat "${SCRATCH}/${name}.dbpath")" != "$db" ]; then+ fail "${name}: store changed across the switch: $(cat "${SCRATCH}/${name}.dbpath") -> ${db}"+ return 1+ fi+ printf '%s' "$db" >"${SCRATCH}/${name}.dbpath"+ ok "${name}: Pod ${POD_UID:0:8} stateless=${stateless}, root layout at ${path}, merge exit 0, store ${db}"+}++# store_path NAME: the database the running mcp-host opened, read from its one+# "[ConversationStore] opened" log line (kubectl logs, no exec). mcp-host+# writes the line after the worker's first answer, which can come after Pod+# Ready, so the log is re-read for up to STORE_LOG_TIMEOUT seconds. Callers+# run it inside $(...), so the reason goes to stderr and the caller records+# the FAIL.+store_path() {+ local pod lines deadline=$((SECONDS + STORE_LOG_TIMEOUT))+ pod="$(jq -r '.metadata.name' <<<"$POD_JSON")"+ while :; do+ lines="$(kctl logs "$pod" -n "$HOST_NS" -c mcp-host |+ jq -Rr 'fromjson? | select(.msg == "[ConversationStore] opened") | .dbPath')" ||+ { echo "${1}: could not read the mcp-host log of ${pod}" >&2; return 1; }+ if [ -n "$lines" ]; then+ [ "$(wc -l <<<"$lines" | tr -d ' ')" = 1 ] ||+ { echo "${1}: expected one store-opened line in ${pod}, got: ${lines}" >&2; return 1; }+ printf '%s' "$lines"+ return 0+ fi+ [ "$SECONDS" -lt "$deadline" ] ||+ { echo "${1}: expected one store-opened line in ${pod}, got: <none> after ${STORE_LOG_TIMEOUT}s" >&2; return 1; }+ sleep 1+ done+}++# ─── Conversations and the history oracle ─────────────────────────────+# history_digest MESSAGES_JSON → "<turns> <sha256>". The oracle refuses an+# empty catalog: zero turns is never a match.+history_digest() {+ local turns sum+ turns="$(jq -r '.turns | length' <<<"$1")"+ sum="$(jq -cS '[.turns[] | {number, user_input, response}]' <<<"$1" | shasum -a 256 | cut -d' ' -f1)"+ printf '%s %s' "$turns" "$sum"+}+history_matches() { # EXPECTED_DIGEST MESSAGES_JSON+ local actual+ actual="$(history_digest "$2")"+ [ "${actual%% *}" -gt 0 ] && [ "$actual" = "$1" ]+}+session_listed() { # SESSIONS_JSON HOST THREAD+ jq -e --arg t "$3" --arg a "$2" '[.items[]? | select(.chatId == $t and .agent == $a)] | length == 1' <<<"$1" >/dev/null+}+# chat_set SESSIONS_JSON HOST → "<n>:<sorted chatIds>" of the Host's sessions.+chat_set() {+ jq -r --arg a "$2" '[.items[]? | select(.agent == $a) | .chatId] | unique | "\(length):\(join(","))"' <<<"$1"+}+# chats_match BEFORE AFTER: the same set on both sides, and not an empty one.+chats_match() { [ "${1%%:*}" -gt 0 ] && [ "$1" = "$2" ]; }++# fetch_messages records nothing; read_messages counts a non-200 as a failure.+fetch_messages() { # HOST THREAD → HTTP_CODE, HTTP_BODY+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/$1/sessions/$1/$2/messages?limit=100" || true+}+read_messages() { # HOST THREAD → HTTP_BODY+ fetch_messages "$1" "$2"+ [ "$HTTP_CODE" = 200 ] || { fail "$1/$2: messages returned ${HTTP_CODE}"; return 1; }+}++# send_turn HOST THREAD CONTENT: a suspended Host answers 503 host_waking+# while it wakes; retry only that, bounded by TURN_TIMEOUT. The turn counts+# only once it is readable with its marker.+send_turn() {+ local host=$1 thread=$2 content=$3 elapsed=0 before+ fetch_messages "$host" "$thread"+ case "$HTTP_CODE" in+ 200) before="$(jq -r '.turns | length' <<<"$HTTP_BODY")" ;;+ 404) before=0 ;;+ *) fail "${host}/${thread}: messages returned ${HTTP_CODE} before the turn"; return 1 ;;+ esac+ while :; do+ rpc POST "${RPC_BASE}/api/v1/rpc/hosts/${host}/messages" \+ "$(jq -cn --arg c "$content" --arg t "$thread" '{content:$c,threadId:$t}')" "$TURN_TIMEOUT" || true+ [[ "$HTTP_CODE" =~ ^2 ]] && break+ if [ "$HTTP_CODE" = 503 ] && jq -e '.code == "host_waking"' <<<"$HTTP_BODY" >/dev/null 2>&1 &&+ [ "$elapsed" -lt "$TURN_TIMEOUT" ]; then+ sleep 5+ elapsed=$((elapsed + 5))+ continue+ fi+ fail "${host}/${thread}: turn returned ${HTTP_CODE}: ${HTTP_BODY:0:300}"+ return 1+ done+ elapsed=0+ while [ "$elapsed" -lt "$TURN_TIMEOUT" ]; do+ fetch_messages "$host" "$thread"+ if [ "$HTTP_CODE" = 200 ] && jq -e --argjson n "$((before + 1))" --arg c "$content" \+ '(.turns | length) == $n and (.turns[-1].user_input | contains($c))' <<<"$HTTP_BODY" >/dev/null; then+ return 0+ fi+ sleep 3+ elapsed=$((elapsed + 3))+ done+ fail "${host}/${thread}: turn ${before}+1 not persisted within ${TURN_TIMEOUT}s"+ return 1+}++# seed_history HOST THREADS TURNS: writes conversations and records digests.+seed_history() {+ local host=$1 threads=$2 turns=$3 i j thread+ : >"${SCRATCH}/${host}.threads"+ for i in $(seq 1 "$threads"); do+ thread="e2e-825-${host}-t${i}"+ for j in $(seq 1 "$turns"); do+ send_turn "$host" "$thread" "e2e-825 ${RUN} ${host} thread ${i} turn ${j}: reply with one short sentence." || return 1+ done+ read_messages "$host" "$thread" || return 1+ printf '%s %s\n' "$thread" "$(history_digest "$HTTP_BODY")" >>"${SCRATCH}/${host}.threads"+ done+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }+ chat_set "$HTTP_BODY" "$host" >"${SCRATCH}/${host}.chats"+ ok "${host}: ${threads} conversation(s) x ${turns} turn(s) written"+}++# assert_history HOST TRANSITION: the Host lists the same chatId set as before+# the transition; every recorded conversation is unchanged; then a resume turn+# is persisted with the next turn number.+assert_history() {+ local host=$1 transition=$2 thread digest sessions before after next=''+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }+ sessions="$HTTP_BODY"+ before="$(cat "${SCRATCH}/${host}.chats")"+ after="$(chat_set "$sessions" "$host")"+ chats_match "$before" "$after" ||+ { fail "${host} ${transition}: chatId set before=${before} after=${after}"; return 1; }+ echo "E2E_825_CONTINUITY ${host} ${transition} before=${before} after=${after} dbPath=$(cat "${SCRATCH}/${host}.dbpath")"+ while read -r thread digest; do+ session_listed "$sessions" "$host" "$thread" || { fail "${host}/${thread}: not in the session list"; return 1; }+ read_messages "$host" "$thread" || return 1+ history_matches "$digest" "$HTTP_BODY" ||+ { fail "${host}/${thread}: history changed (expected ${digest}, got $(history_digest "$HTTP_BODY"))"; return 1; }+ send_turn "$host" "$thread" "e2e-825 ${RUN} resume ${thread}: reply with one short sentence." || return 1+ read_messages "$host" "$thread" || return 1+ next+="${thread} $(history_digest "$HTTP_BODY")"$'\n'+ done <"${SCRATCH}/${host}.threads"+ printf '%s' "$next" >"${SCRATCH}/${host}.threads"+ ok "${host}: history listed, unchanged and resumed ($(wc -l <"${SCRATCH}/${host}.threads" | tr -d ' ') conversation(s))"+}++# ─── FAULT:pvc-fixture ────────────────────────────────────────────────+# pvc_fixture HOST ro|rw LABEL SCRIPT [ARGS...]: runs SCRIPT with the PVC root+# as $1 in a Pod on the Host's last node. The Pod output goes to stdout; the+# log and failure lines go to stderr so a caller capturing stdout reads only it.+pvc_fixture() {+ local host=$1 mode=$2 label=$3 script=$4 pod node ro body elapsed=0 phase+ shift 4+ pod="$(truncate_rfc1123 "e2e825-${label}-${host}")"+ node="$(cat "${SCRATCH}/${host}.node")"+ ro=false+ [ "$mode" = ro ] && ro=true+ log "FAULT:pvc-fixture ${label} on ${host}-workspace (${mode})" >&2+ body="$(jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$FIXTURE_IMAGE" \+ --arg claim "${host}-workspace" --argjson ro "$ro" --arg script "$script" \+ --args '{+ apiVersion:"v1", kind:"Pod",+ metadata:{name:$pod, namespace:$ns, labels:{purpose:"e2e-825-pvc-fixture"}},+ spec:{+ restartPolicy:"Never", automountServiceAccountToken:false, enableServiceLinks:false,+ terminationGracePeriodSeconds:0, nodeName:$node,+ securityContext:{runAsUser:1001, runAsGroup:1001, runAsNonRoot:true, seccompProfile:{type:"RuntimeDefault"}},+ containers:[{+ name:"fixture", image:$image, imagePullPolicy:"IfNotPresent",+ command:(["/bin/sh","-c",$script,"fixture","/mnt/pvc"] + $ARGS.positional),+ resources:{requests:{cpu:"10m",memory:"32Mi"}, limits:{cpu:"200m",memory:"128Mi"}},+ securityContext:{allowPrivilegeEscalation:false, readOnlyRootFilesystem:true, capabilities:{drop:["ALL"]}},+ volumeMounts:[{name:"pvc", mountPath:"/mnt/pvc", readOnly:$ro}]+ }],+ volumes:[{name:"pvc", persistentVolumeClaim:{claimName:$claim, readOnly:$ro}}]+ }}' "$@")"+ # Callers usually run this function inside $(...), so an array append would+ # be lost with the subshell; the registry file is what cleanup() reads.+ # Registered before the create so a create that times out after the API+ # accepted it is still cleaned up.+ printf '%s\n' "$pod" >>"$FIXTURE_POD_REGISTRY"+ kctl create -f - <<<"$body" >/dev/null || { fail "${label}: fixture Pod create failed" >&2; return 1; }+ while [ "$elapsed" -lt 180 ]; do+ phase="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"+ case "$phase" in Succeeded | Failed) break ;; esac+ sleep 2+ elapsed=$((elapsed + 2))+ done+ kctl logs "$pod" -n "$HOST_NS" >"${SCRATCH}/${pod}.log" 2>&1 || true+ # A Pod left behind keeps the RWO claim attached to its node and blocks the+ # next fixture or Host Pod on another node, so a failed delete fails the step.+ if ! kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then+ fail "${label}: fixture Pod ${pod} delete failed" >&2+ return 1+ fi+ if [ "$phase" != Succeeded ]; then+ fail "${label}: fixture Pod ended ${phase:-<timeout>}: $(tail -n 5 "${SCRATCH}/${pod}.log")" >&2+ return 1+ fi+ cat "${SCRATCH}/${pod}.log"+}++# Listing of the PVC root: `<type> <path>`, plus ` <target>` for a symlink,+# lost+found pruned. The fixture image has busybox find, which lacks -printf.+# shellcheck disable=SC2016 # expanded inside the fixture Pod+LIST_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -print | while IFS= read -r p; do+ if [ -L "$p" ]; then printf "l %s %s\n" "$p" "$(readlink "$p")"+ elif [ -d "$p" ]; then printf "d %s\n" "$p"+ else printf "f %s\n" "$p"; fi+done | LC_ALL=C sort'+# shellcheck disable=SC2016+HASH_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum'+# shellcheck disable=SC2016+WRITE_FILE_SCRIPT='mkdir -p "$1/e2e-825" && printf "%s\n" "$2" > "$1/e2e-825/$3"'+# shellcheck disable=SC2016+READ_FILE_SCRIPT='cat "$1/e2e-825/$2"'++read_seed_file() { # HOST NAME EXPECTED+ local got+ got="$(pvc_fixture "$1" ro read-file "$READ_FILE_SCRIPT" "$2")" || return 1+ [ "$got" = "$3" ] || { fail "$1: seeded file e2e-825/$2 reads '${got}', expected '$3'"; return 1; }+ ok "$1: seeded file e2e-825/$2 intact at the PVC root"+}++# ─── Operator script ──────────────────────────────────────────────────+# layout_row inventory|verify HOST EXITS: runs deploy/scripts/host-layout-merge.sh+# on this profile and prints the result of the Host's PVC row. EXITS is a case+# pattern; the fleet verdict is left alone because other Hosts on the profile+# decide it, and exit 2 (read failure) never matches.+layout_row() {+ local cmd=$1 host=$2 exits=$3 rc=0 out row+ out="$(mktemp "${SCRATCH}/layout-${cmd}-${host}.XXXXXX")"+ log "host-layout-merge.sh ${cmd} for ${HOST_NS}/${host}-workspace" >&2+ bash "$LAYOUT_SCRIPT" "$cmd" --context "$E2E_KUBECONTEXT" >"$out" 2>"${out}.err" || rc=$?+ # shellcheck disable=SC2254 # EXITS is a case pattern on purpose+ case "$rc" in+ $exits) ;;+ *)+ fail "${host}: host-layout-merge.sh ${cmd} exited ${rc} (expected ${exits}): $(tail -n 3 "${out}.err" | tr '\n' ' ') $(tail -n 3 "$out" | tr '\n' ' ')" >&2+ return 1+ ;;+ esac+ [ "$rc" != 2 ] || { fail "${host}: host-layout-merge.sh ${cmd} could not read the cluster" >&2; return 1; }+ row="$(grep -F "PVC ${HOST_NS}/${host}-workspace " "$out")" ||+ { fail "${host}: host-layout-merge.sh ${cmd} printed no row for ${host}-workspace" >&2; return 1; }+ printf '%s\n' "${row#*result=}"+}++# ─── Suspend and wake ─────────────────────────────────────────────────+wait_suspended() { # HOST: state suspended, replicas 0, zero Pods (Terminating included)+ local host=$1 elapsed=0 state reps pods+ log "FAULT:hcc-cadence waiting for ${host} to suspend"+ while [ "$elapsed" -lt "$SUSPEND_TIMEOUT" ]; do+ state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"+ reps="$(kctl get deployment "$host" -n "$HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"+ pods="$(host_pods "$host" | jq '.items | length')" || pods=unknown+ if [ "$state" = suspended ] && [ "$reps" = 0 ] && [ "$pods" = 0 ]; then+ ok "${host}: suspended with zero Pods"+ POD_UID=''+ return 0+ fi+ sleep 5+ elapsed=$((elapsed + 5))+ done+ fail "${host}: not suspended within ${SUSPEND_TIMEOUT}s (state=${state:-<none>} replicas=${reps:-?} pods=${pods})"+ return 1+}++post_wake() { # HOST+ rpc POST "${RPC_BASE}/api/v1/rpc/hosts/$1/wake" || true+ case "$HTTP_CODE" in 200 | 202 | 204) log "wake requested for $1" ;; *)+ fail "$1: wake returned ${HTTP_CODE}: ${HTTP_BODY}"+ return 1+ ;;+ esac+}++set_hcc_cadence() {+ local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES+ CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS) k+ HCC_ENV_SAVED=''+ for k in "${keys[@]}"; do+ HCC_ENV_SAVED+="${k}=$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \+ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}")"$'\n'+ done+ log 'FAULT:hcc-cadence idle 1 min, drain 20 s, heartbeat poll 5 s'+ HCC_ENV_CHANGED=1+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \+ CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=1 CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=1 \+ CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=20000 CONTEXT_MAPPER_HEARTBEAT_POLL_MS=5000 >/dev/null ||+ { fail 'hcc-cadence: kubectl set env failed'; return 1; }+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||+ { fail 'hcc-cadence: HCC rollout did not settle'; return 1; }+}++restore_hcc_cadence() {+ local args=() key val actual+ [ "$HCC_ENV_CHANGED" = 1 ] || return 0+ while IFS='=' read -r key val; do+ [ -n "$key" ] || continue+ if [ -n "$val" ]; then args+=("${key}=${val}"); else args+=("${key}-"); fi+ done <<<"$HCC_ENV_SAVED"+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null || return 1+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null || return 1+ while IFS='=' read -r key val; do+ [ -n "$key" ] || continue+ actual="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \+ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${key}')].value}")"+ [ "$actual" = "$val" ] || { echo "HCC ${key} restored to '${actual}', expected '${val}'" >&2; return 1; }+ done <<<"$HCC_ENV_SAVED"+ HCC_ENV_CHANGED=0+ log 'FAULT:hcc-cadence restored'+}diff --git a/scripts/e2e/e2e-hcc-layout-upgrade.sh b/scripts/e2e/e2e-hcc-layout-upgrade.shnew file mode 100644index 000000000..b50d7a2d8--- /dev/null+++ b/scripts/e2e/e2e-hcc-layout-upgrade.sh@@ -0,0 +1,542 @@+#!/usr/bin/env bash+# Branch-owned live lane for #825: upgrade an installation from v0.11.0 to this+# head and prove the Host PVC layout and history survive it (plan A7.40).+#+# Usage: e2e-hcc-layout-upgrade.sh main|control, each on a FRESH branch+# profile (no host-context-controller yet). The launcher deletes and recreates+# the profile between the two modes.+#+# main installs v0.11.0 (published images), seeds one Host per row, runs+# deploy/scripts/host-layout-merge.sh inventory, upgrades to this+# head (images built from this tree, database kept), then checks+# verify, repairs T5 with the upgrade-note procedure and checks the+# history of every row. Rows:+# T1 stateful;+# T2 stateless, active at the upgrade;+# T3 stateless, suspended at the upgrade (woken afterwards);+# T5 created stateless, used, switched to stateful by a+# CommunicationChannel and used again (two live databases);+# T6 stateless -> stateful -> stateless.+# Prints E2E_UPGRADE_MAIN=PASS|FAIL and E2E_UPGRADE_HEAD=<sha>.+# control the negative control: CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false+# in the upgrade deploy. inventory must report+# FLAG_FALSE_WITH_SPLIT_TREES and the history check must fail on a+# T2-shaped Host. Needs E2E_UPGRADE_MAIN_EVIDENCE (the main log of+# the same HEAD). Prints E2E_UPGRADE_RESULT=PASS only when main+# passed and the control failed as expected.+#+# Install path: `make minikube-setup IMAGE_SOURCE=ghcr MINIKUBE_IMAGE_TAG=v0.11.0`+# from this tree. That is the release's install only while this tree's+# deploy/base, deploy/components, deploy/overlays and CRDs equal v0.11.0's;+# the lane checks that and refuses otherwise.+#+# Cluster writes besides the setups and the admin API: the named fixtures of+# e2e-hcc-storage-layout-switch.sh (hcc-cadence, pvc-fixture, channel,+# host-layout-merge), the T5 repair (a pvc-fixture rename plus the deletion of+# the refused Host Pod, as the upgrade note says) and, in control mode,+# FAULT:flag-false (kubectl set env on the HCC Deployment, undone in the+# trap). No kubectl exec into Host Pods, no SQLite reads or writes.+set -euo pipefail+umask 077++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"+# shellcheck source=scripts/e2e/load-dotenv.sh+source "${REPO_ROOT}/scripts/e2e/load-dotenv.sh"+dotenv_load_canonical_root "${REPO_ROOT}"+# shellcheck source=scripts/e2e/admin-credentials.sh+source "${REPO_ROOT}/scripts/e2e/admin-credentials.sh"+# shellcheck source=scripts/e2e/e2e-lib.sh+source "${SCRIPT_DIR}/e2e-lib.sh"++die() { fail "$*"; exit 1; }++MODE="${1:-}"+case "$MODE" in main | control) ;; *) die 'usage: e2e-hcc-layout-upgrade.sh main|control' ;; esac++RELEASE=v0.11.0+# main_evidence_ok FILE HEAD: the main log passed on this same HEAD.+main_evidence_ok() {+ [ -r "$1" ] && grep -qx 'E2E_UPGRADE_MAIN=PASS' "$1" && grep -qx "E2E_UPGRADE_HEAD=$2" "$1"+}++# ─── Preconditions ────────────────────────────────────────────────────+if [ -z "$E2E_KUBECONTEXT" ] || ! is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"; then+ die 'select an explicit branch-scoped Minikube context (KUBECONTEXT=<profile>)'+fi+require_safe_kube_context+[ "${E2E_ALLOW_STATELESS_CADENCE_ACCELERATION:-0}" = 1 ] ||+ die 'set E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 (T3 shortens the HCC idle window on this owned profile)'+[ -n "${E2E_PROFILE_PORTS_ENV:-}" ] && [ -r "$E2E_PROFILE_PORTS_ENV" ] ||+ die 'provide E2E_PROFILE_PORTS_ENV from the owned branch profile'+[ -n "${E2E_DEV_LOGIN_EMAIL:-}" ] || die 'set E2E_DEV_LOGIN_EMAIL to the seeded login ([email protected])'+[ -n "${E2E_UPGRADE_EVIDENCE_DIR:-}" ] && [ -d "$E2E_UPGRADE_EVIDENCE_DIR" ] ||+ die 'set E2E_UPGRADE_EVIDENCE_DIR to an existing directory for the setup logs'+HEAD_SHA="$(git -C "$REPO_ROOT" rev-parse HEAD)"+[ -z "$(git -C "$REPO_ROOT" status --porcelain)" ] ||+ die 'the worktree is dirty; the head images would not be this HEAD'+if [ "$MODE" = control ]; then+ main_evidence_ok "${E2E_UPGRADE_MAIN_EVIDENCE:-}" "$HEAD_SHA" ||+ die "E2E_UPGRADE_MAIN_EVIDENCE must be a main log with E2E_UPGRADE_MAIN=PASS and E2E_UPGRADE_HEAD=${HEAD_SHA}"+fi+git -C "$REPO_ROOT" rev-parse -q --verify "refs/tags/${RELEASE}" >/dev/null || die "tag ${RELEASE} is not in this clone"+if ! git -C "$REPO_ROOT" diff --quiet "$RELEASE" HEAD -- deploy/base deploy/components deploy/overlays charts/clerum-crds; then+ die "manifests or CRDs differ from ${RELEASE}; this tree no longer installs the release: $(git -C "$REPO_ROOT" diff --name-only "$RELEASE" HEAD -- deploy/base deploy/components deploy/overlays charts/clerum-crds | tr '\n' ' ')"+fi+profile_port_value() {+ awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "$E2E_PROFILE_PORTS_ENV"+}+CONTROL_API_BASE_URL="$(profile_port_value CONTROL_API_URL)"+CONTROL_UI_BASE_URL="$(profile_port_value CONTROL_UI_URL)"+EXTERNAL_REST_API_BASE_URL="$(profile_port_value EXTERNAL_REST_API_URL)"+RPC_PROXY_BASE_URL="$(profile_port_value RPC_PROXY_URL)"+export CONTROL_API_BASE_URL CONTROL_UI_BASE_URL EXTERNAL_REST_API_BASE_URL RPC_PROXY_BASE_URL+require_branch_profile_urls "$E2E_KUBECONTEXT" "$E2E_PROFILE_PORTS_ENV" ||+ die 'profile URLs do not match the owned ports'+kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"]==$c)' >/dev/null ||+ die 'target is not the selected Minikube profile'++HOST_NS=mcp-host CHANNEL_NS=channels+HCC_NS=control-plane HCC_DEPLOY=host-context-controller+SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm}"+CAPI="${CONTROL_API_BASE_URL%/}/api/v1"+EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"+RPC_BASE="${RPC_PROXY_BASE_URL%/}"+ADMIN_USERNAME="${ADMIN_USERNAME:-admin}"+ADMIN_PASSWORD="${ADMIN_PASSWORD:-}"+[ -n "$ADMIN_PASSWORD" ] || ADMIN_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"+[ -n "$ADMIN_PASSWORD" ] || die 'admin password could not be resolved (ADMIN_PASSWORD or the canonical .env)'+DEV_EMAIL="$E2E_DEV_LOGIN_EMAIL"+DEV_PASSWORD="${E2E_USER_PASSWORD:-$ADMIN_PASSWORD}"+RUN="$(od -An -N3 -tx1 /dev/urandom | tr -d ' \n')"+POD_READY_TIMEOUT="${E2E_825_POD_READY_TIMEOUT:-300}"+SUSPEND_TIMEOUT="${E2E_825_SUSPEND_TIMEOUT:-300}"+TURN_TIMEOUT="${E2E_825_TURN_TIMEOUT:-240}"+STORE_LOG_TIMEOUT="${E2E_825_STORE_LOG_TIMEOUT:-30}"+LAYOUT_SCRIPT="${REPO_ROOT}/deploy/scripts/host-layout-merge.sh"+[ -r "$LAYOUT_SCRIPT" ] || die "missing operator script ${LAYOUT_SCRIPT}"++ROWS=(T1 T2 T3 T5 T6)+ROWS_DONE=0 LANE_OK=0+CREATED_HOSTS=() CREATED_CHANNELS=()+HCC_ENV_SAVED='' HCC_ENV_CHANGED=0 FLAG_FALSE_SET=0+SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/e2e-825-upgrade.XXXXXX")"+FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"+: >"$FIXTURE_POD_REGISTRY"+ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID='' FIXTURE_IMAGE='' POD_UID=''+HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()++# shellcheck source=scripts/e2e/e2e-825-lib.sh+source "${SCRIPT_DIR}/e2e-825-lib.sh"++F2='workspace layout merge refused: more than one live SQLite database across'+# The upgrade note's repair for a T5 tree, keeping the chats since the switch:+# the database family under state/ is renamed, never deleted.+# shellcheck disable=SC2016 # expanded inside the fixture Pod+REPAIR_STATE_FAMILY_SCRIPT='set -eu; cd "$1/state"; [ -f state.db ]+for s in "" -wal -shm; do [ -e "state.db$s" ] || continue; mv "state.db$s" "state.db.upgrade-lane.bak$s"; done; ls -la'+# Seed files sit where the agent sees /workspace: the PVC root for a stateful+# Host, PVC/workspace for a stateless one on the old release.+# shellcheck disable=SC2016+WRITE_AT_SCRIPT='mkdir -p "$1/$2/e2e-825" && printf "%s\n" "$3" > "$1/$2/e2e-825/$4"'+# shellcheck disable=SC2016+READ_AT_SCRIPT='cat "$1/$2/e2e-825/$3"'++# run_logged LABEL CMD...: runs CMD from the repository root, its output in the+# evidence directory, and reports the command's own exit code.+run_logged() {+ local label=$1 rc=0 logf+ shift+ logf="${E2E_UPGRADE_EVIDENCE_DIR}/${MODE}-${RUN}-${label}.log"+ log "${label}: $* (log ${logf})"+ (cd "$REPO_ROOT" && "$@") >"$logf" 2>&1 || rc=$?+ echo "${label}_EXIT=${rc}" >>"$logf"+ [ "$rc" = 0 ] || { fail "${label} exited ${rc}: $(tail -n 5 "$logf" | tr '\n' ' ')"; return 1; }+ ok "${label} exit 0"+}++# Every setup restarts the services, so the forwards are re-held and probed.+hold_forwards() {+ local label=$1 key url code failures=0+ run_logged "${label}-stop-pf" make -f scripts/minikube-profiles/branch.mk branch-profile-stop-pf || return 1+ run_logged "${label}-pf" make -f scripts/minikube-profiles/branch.mk branch-profile-pf || return 1+ for key in CONTROL_API_URL CONTROL_UI_URL EXTERNAL_REST_API_URL RPC_PROXY_URL; do+ url="$(profile_port_value "$key")"+ code="$(curl -sS -o /dev/null -m 6 -w '%{http_code}' "$url" 2>/dev/null)" || code=000+ if [ "$code" = 000 ]; then+ fail "forward ${key} (${url}) did not answer"+ failures=$((failures + 1))+ else+ ok "forward ${key} answers HTTP ${code}"+ fi+ done+ [ "$failures" = 0 ]+}++login_all() {+ admin_login || return 1+ admin GET "${CAPI}/admin/users?q=$(jq -rn --arg q "$DEV_EMAIL" '$q|@uri')" || true+ USER_ID="$(jq -r --arg e "$DEV_EMAIL" '.items[]? | select((.email // "") == $e) | .id' <<<"$HTTP_BODY" | head -n1)"+ [ -n "$USER_ID" ] || { fail "user ${DEV_EMAIL} not found (${HTTP_CODE})"; return 1; }+ admin GET "${CAPI}/admin/users/${USER_ID}/teams" || true+ TEAM_ID="$(jq -r '.items[0].id // empty' <<<"$HTTP_BODY")"+ user_login || return 1+ # bash 3.2 treats "${empty[@]}" as unbound under set -u.+ [ "${#CREATED_HOSTS[@]}" -eq 0 ] || mint_rpc_token+}++hcc_image() {+ kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].image}'+}+read_fixture_image() {+ FIXTURE_IMAGE="$(kctl get deployment "$SOURCE_HOST" -n "$HOST_NS" \+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].image}')"+ [ -n "$FIXTURE_IMAGE" ] || { fail "could not read the ${SOURCE_HOST} image for the PVC fixture"; return 1; }+}++install_release() {+ [ -z "$(kctl get deployment "$HCC_DEPLOY" -n "$HCC_NS" --ignore-not-found -o name)" ] ||+ { fail "deployment/${HCC_DEPLOY} already exists; the lane needs a fresh profile"; return 1; }+ run_logged "install-${RELEASE}" make minikube-setup MINIKUBE_PROFILE="$E2E_KUBECONTEXT" \+ IMAGE_SOURCE=ghcr MINIKUBE_IMAGE_TAG="$RELEASE" || return 1+ case "$(hcc_image)" in+ *":${RELEASE}") ok "HCC runs $(hcc_image)" ;;+ *) fail "HCC runs '$(hcc_image)', expected the ${RELEASE} image"; return 1 ;;+ esac+ # Built now and loaded next to the release images, so the upgrade below is a+ # deploy of minutes and the active Hosts stay active through it.+ run_logged build-head-images make minikube-build-images MINIKUBE_PROFILE="$E2E_KUBECONTEXT" || return 1+ hold_forwards release || return 1+ login_all || return 1+ SOURCE_HOST_JSON="$(kctl get host "$SOURCE_HOST" -n "$HOST_NS" -o json)" ||+ { fail "source Host ${SOURCE_HOST} not found"; return 1; }+ # Personalization writes IDENTITY.md and friends into the workspace; the+ # clones would then collide on F3 instead of exercising the rows.+ jq -e '.spec.personalization.enabled != true' <<<"$SOURCE_HOST_JSON" >/dev/null ||+ { fail "source Host ${SOURCE_HOST} has personalization enabled"; return 1; }+ # verify's fleet verdict below must be decided by the rows' Hosts alone.+ kctl get hosts -n "$HOST_NS" -o json | jq -e '[.items[] | select(.spec.lifecycle.stateless == true)] | length == 0' >/dev/null ||+ { fail 'the fresh install already has stateless Hosts'; return 1; }+ read_fixture_image+}++upgrade_to_head() {+ run_logged upgrade-to-head make minikube-setup MINIKUBE_PROFILE="$E2E_KUBECONTEXT" \+ IMAGE_SOURCE=local REUSE_DB=true ARGS=--skip-build || return 1+ case "$(hcc_image)" in+ *":${RELEASE}") fail "HCC still runs $(hcc_image) after the upgrade"; return 1 ;;+ *) ok "HCC runs $(hcc_image)" ;;+ esac+ hold_forwards head || return 1+ login_all || return 1+ read_fixture_image+}++HOST_T1='' HOST_T2='' HOST_T3='' HOST_T5='' HOST_T6=''+new_host() { # ROW STATELESS: creates the row's Host, sets HOST_<row>+ local name+ name="$(truncate_rfc1123 "e2e-up-$(tr '[:upper:]' '[:lower:]' <<<"$1")-${RUN}")"+ printf -v "HOST_$1" '%s' "$name"+ create_host "$name" "$2" || return 1+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1+ printf '%s' "$POD_UID" >"${SCRATCH}/${name}.uid"+}++seed_file() { # HOST DIR(.|workspace)+ pvc_fixture "$1" rw write-file "$WRITE_AT_SCRIPT" "$2" "upgrade ${RUN} $1" seed.txt >/dev/null+}++# snapshot_served HOST: what the Host serves now, the history oracle's+# baseline. Every listed chat gets a digest; an empty catalog fails.+snapshot_served() {+ local host=$1 sessions chats chat digest+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }+ sessions="$HTTP_BODY"+ chat_set "$sessions" "$host" >"${SCRATCH}/${host}.chats"+ chats="$(jq -r --arg a "$host" '[.items[]? | select(.agent == $a) | .chatId] | unique | .[]' <<<"$sessions")"+ [ -n "$chats" ] || { fail "${host}: serves no conversation before the upgrade"; return 1; }+ : >"${SCRATCH}/${host}.threads"+ for chat in $chats; do+ read_messages "$host" "$chat" || return 1+ digest="$(history_digest "$HTTP_BODY")"+ [ "${digest%% *}" -gt 0 ] || { fail "${host}/${chat}: empty history"; return 1; }+ printf '%s %s\n' "$chat" "$digest" >>"${SCRATCH}/${host}.threads"+ done+ ok "${host}: serves $(cat "${SCRATCH}/${host}.chats" | cut -d: -f1) conversation(s) before the upgrade"+}++# wait_upgraded_pod HOST: the Host's Pod after the upgrade (a new UID), waking+# a stateless Host that suspended while the deploy ran.+wait_upgraded_pod() {+ local host=$1 state+ state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"+ if [ "$state" = suspended ]; then+ log "${host} suspended during the upgrade; waking it"+ post_wake "$host" || return 1+ fi+ wait_new_ready_pod "$host" "$(cat "${SCRATCH}/${host}.uid")" "$POD_READY_TIMEOUT"+}++# The head HCC renders the merge init on every Host Deployment it converged.+wait_merge_templates() {+ local host elapsed=0 pending+ while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do+ pending=''+ for host in "$@"; do+ kctl get deployment "$host" -n "$HOST_NS" -o json | jq -e '+ [.spec.template.spec.initContainers[]?.name] == ["workspace-layout-merge"]' >/dev/null || pending+="${host} "+ done+ [ -n "$pending" ] || { ok "every row's Deployment renders workspace-layout-merge"; return 0; }+ sleep 5+ elapsed=$((elapsed + 5))+ done+ fail "Deployments not re-rendered by the head HCC within ${POD_READY_TIMEOUT}s: ${pending}"+ return 1+}++# wait_refused_pod HOST → prints the Pod whose merge init exited 1.+wait_refused_pod() {+ local host=$1 elapsed=0 pod+ while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do+ pod="$(host_pods "$host" | jq -r '[.items[] | select(any(.status.initContainerStatuses[]?;+ .name == "workspace-layout-merge" and ((.state.terminated.exitCode // 0) == 1 or (.lastState.terminated.exitCode // 0) == 1)))][0].metadata.name // ""')"+ [ -z "$pod" ] || { printf '%s' "$pod"; return 0; }+ sleep 3+ elapsed=$((elapsed + 3))+ done+ echo "${host}: no Pod with workspace-layout-merge exit 1 within ${POD_READY_TIMEOUT}s" >&2+ return 1+}++# run_layout CMD: one run of the operator script; output in LAYOUT_OUT, exit in LAYOUT_RC.+run_layout() {+ LAYOUT_OUT="${SCRATCH}/layout-$1-${SECONDS}.out"+ LAYOUT_RC=0+ bash "$LAYOUT_SCRIPT" "$1" --context "$E2E_KUBECONTEXT" >"$LAYOUT_OUT" 2>&1 || LAYOUT_RC=$?+ cp "$LAYOUT_OUT" "${E2E_UPGRADE_EVIDENCE_DIR}/${MODE}-${RUN}-layout-$1-${SECONDS}.out"+}+row_of() { # HOST → the result of the Host's PVC row in LAYOUT_OUT+ local row+ row="$(grep -F "PVC ${HOST_NS}/${1}-workspace " "$LAYOUT_OUT")" || return 1+ printf '%s' "${row#*result=}"+}+expect_row() { # HOST PATTERN LABEL+ local row+ row="$(row_of "$1")" || { fail "$3: no row for ${1}-workspace"; return 1; }+ # shellcheck disable=SC2254 # PATTERN is a case pattern on purpose+ case "$row" in $2) ok "$3: ${1}-workspace is ${row%% workspace layout*}" ;; *) fail "$3: ${1}-workspace is '${row}', expected $2"; return 1 ;; esac+}++# ─── main ─────────────────────────────────────────────────────────────+seed_rows() {+ # T3 first: the shortened idle window suspends every idle stateless Host,+ # so the others are created after it is restored.+ new_host T3 true && seed_history "$HOST_T3" 3 1 && seed_file "$HOST_T3" workspace && snapshot_served "$HOST_T3" || return 1+ set_hcc_cadence && wait_suspended "$HOST_T3" && restore_hcc_cadence || return 1+ new_host T1 false && seed_history "$HOST_T1" 3 1 && seed_file "$HOST_T1" . && snapshot_served "$HOST_T1" || return 1+ # T5: the old history under state/, then a channel makes HCC run it+ # stateful on the PVC root, where mcp-host opens a second database.+ new_host T5 true && seed_history "$HOST_T5" 3 1 && seed_file "$HOST_T5" workspace || return 1+ local channel+ channel="$(truncate_rfc1123 "e2e-up-t5-${RUN}")"+ log "FAULT:channel ${CHANNEL_NS}/${channel} → ${HOST_T5}"+ printf '%s\n' 'apiVersion: clerum.io/v1alpha1' 'kind: CommunicationChannel' 'metadata:' \+ " name: ${channel}" " namespace: ${CHANNEL_NS}" 'spec:' " hostRef: ${HOST_T5}" ' telegram:' \+ " - channelId: \"e2e-up-${RUN}\"" ' chatType: "private"' ' userIds: ["000000000"]' |+ kctl create -f - >/dev/null || { fail 'T5: CommunicationChannel create failed'; return 1; }+ CREATED_CHANNELS+=("$channel")+ wait_new_ready_pod "$HOST_T5" "$(cat "${SCRATCH}/${HOST_T5}.uid")" "$POD_READY_TIMEOUT" || return 1+ printf '%s' "$POD_UID" >"${SCRATCH}/${HOST_T5}.uid"+ seed_history "$HOST_T5" 1 1 && snapshot_served "$HOST_T5" || return 1+ # T6: the release's stateless entry keeps the newer database, so what T6+ # serves afterwards is what the snapshot records.+ new_host T6 true && seed_history "$HOST_T6" 2 1 && seed_file "$HOST_T6" workspace || return 1+ local stateless+ for stateless in false true; do+ set_lifecycle "$HOST_T6" "$stateless" || return 1+ wait_new_ready_pod "$HOST_T6" "$(cat "${SCRATCH}/${HOST_T6}.uid")" "$POD_READY_TIMEOUT" || return 1+ printf '%s' "$POD_UID" >"${SCRATCH}/${HOST_T6}.uid"+ done+ seed_history "$HOST_T6" 1 1 && snapshot_served "$HOST_T6" || return 1+ # T2 last, so it is still active when the upgrade replaces its Pod.+ new_host T2 true && seed_history "$HOST_T2" 3 1 && seed_file "$HOST_T2" workspace && snapshot_served "$HOST_T2"+}++predict_rows() {+ run_layout inventory+ [ "$LAYOUT_RC" = 1 ] || { fail "inventory exited ${LAYOUT_RC}, expected 1 (T5 blocks): $(tail -n 3 "$LAYOUT_OUT" | tr '\n' ' ')"; return 1; }+ grep -qx 'UPGRADE_BLOCKERS=1' "$LAYOUT_OUT" || { fail "inventory: $(grep UPGRADE_BLOCKERS "$LAYOUT_OUT"), expected UPGRADE_BLOCKERS=1"; return 1; }+ expect_row "$HOST_T5" "WILL_REFUSE_F2 ${F2}*" 'inventory T5' &&+ expect_row "$HOST_T2" WILL_MERGE 'inventory T2' &&+ expect_row "$HOST_T3" WILL_MERGE 'inventory T3' &&+ expect_row "$HOST_T6" WILL_MERGE 'inventory T6' &&+ expect_row "$HOST_T1" NO_SPLIT_SIGNATURE 'inventory T1'+}++check_upgraded() {+ local pod logs host refused_uid+ wait_merge_templates "$HOST_T1" "$HOST_T2" "$HOST_T3" "$HOST_T5" "$HOST_T6" || return 1+ wait_upgraded_pod "$HOST_T1" && assert_pod "$HOST_T1" false /workspace || return 1+ wait_upgraded_pod "$HOST_T2" && assert_pod "$HOST_T2" true /workspace || return 1+ wait_upgraded_pod "$HOST_T6" && assert_pod "$HOST_T6" true /workspace || return 1+ pod="$(wait_refused_pod "$HOST_T5")" || { fail 'T5: the merge did not refuse'; return 1; }+ logs="$(kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge --previous 2>/dev/null ||+ kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge)"+ grep -q "$F2" <<<"$logs" || { fail "T5: merge refused without the F2 line: ${logs}"; return 1; }+ [ "$(kctl get host "$HOST_T3" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}')" = suspended ] ||+ { fail 'T3: no longer suspended before verify'; return 1; }+ run_layout verify+ [ "$LAYOUT_RC" = 1 ] || { fail "verify exited ${LAYOUT_RC}, expected 1 (T3 split, T5 refused)"; return 1; }+ for host in "$HOST_T1" "$HOST_T2" "$HOST_T6"; do expect_row "$host" POD_OK 'verify' || return 1; done+ expect_row "$HOST_T3" SPLIT 'verify T3' && expect_row "$HOST_T5" "MERGE_REFUSED ${F2}*" 'verify T5' || return 1++ # T3 merges on its natural wake.+ post_wake "$HOST_T3" && wait_new_ready_pod "$HOST_T3" '' "$POD_READY_TIMEOUT" && assert_pod "$HOST_T3" true /workspace || return 1+ # T5: the upgrade note's repair, keeping the chats since the switch.+ log "REPAIR:${HOST_T5} rename the state/ database family (upgrade note)"+ pvc_fixture "$HOST_T5" rw repair "$REPAIR_STATE_FAMILY_SCRIPT" >/dev/null || return 1+ refused_uid="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.metadata.uid}')" || { fail "T5: could not read refused Pod ${pod}"; return 1; }+ kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null || { fail "T5: delete of refused Pod ${pod} failed"; return 1; }+ wait_new_ready_pod "$HOST_T5" "$refused_uid" "$POD_READY_TIMEOUT" &&+ assert_pod "$HOST_T5" false /workspace || return 1++ run_layout verify+ if [ "$LAYOUT_RC" != 0 ] || ! grep -qx 'HOST_LAYOUT_VERIFY=PASS' "$LAYOUT_OUT"; then+ fail "verify after the repairs exited ${LAYOUT_RC}: $(tail -n 3 "$LAYOUT_OUT" | tr '\n' ' ')"+ return 1+ fi+ for host in "$HOST_T1" "$HOST_T2" "$HOST_T3" "$HOST_T5" "$HOST_T6"; do expect_row "$host" POD_OK 'verify after repair' || return 1; done+}++check_history() {+ local listing+ assert_history "$HOST_T1" "upgrade(stateful)" && read_seed_file "$HOST_T1" seed.txt "upgrade ${RUN} ${HOST_T1}" || return 1+ ROWS_DONE=$((ROWS_DONE + 1))+ assert_history "$HOST_T2" "upgrade(stateless-active)" && read_seed_file "$HOST_T2" seed.txt "upgrade ${RUN} ${HOST_T2}" || return 1+ ROWS_DONE=$((ROWS_DONE + 1))+ assert_history "$HOST_T3" "upgrade(stateless-suspended)->wake" && read_seed_file "$HOST_T3" seed.txt "upgrade ${RUN} ${HOST_T3}" || return 1+ ROWS_DONE=$((ROWS_DONE + 1))+ # T5 serves the root family it served before the upgrade; the old family is+ # kept as a backup, and its old files stay under workspace/ (no signature).+ assert_history "$HOST_T5" "upgrade(two-databases)->repair" || return 1+ listing="$(pvc_fixture "$HOST_T5" ro list-repaired "$LIST_SCRIPT")" || return 1+ grep -qx 'f ./state/state.db.upgrade-lane.bak' <<<"$listing" ||+ { fail "T5: the renamed family is missing: ${listing}"; return 1; }+ [ "$(pvc_fixture "$HOST_T5" ro read-file "$READ_AT_SCRIPT" workspace seed.txt)" = "upgrade ${RUN} ${HOST_T5}" ] ||+ { fail 'T5: the old seed file is not under workspace/'; return 1; }+ ok 'T5: old family kept as state/state.db.upgrade-lane.bak, old files under workspace/'+ ROWS_DONE=$((ROWS_DONE + 1))+ assert_history "$HOST_T6" "upgrade(stateless-stateful-stateless)" && read_seed_file "$HOST_T6" seed.txt "upgrade ${RUN} ${HOST_T6}" || return 1+ ROWS_DONE=$((ROWS_DONE + 1))+}++run_main() {+ header "Install ${RELEASE}" && install_release &&+ header 'Seed rows on the release' && seed_rows &&+ header 'Predict with inventory' && predict_rows &&+ header 'Upgrade to this head' && upgrade_to_head &&+ header 'Verify and repair' && check_upgraded &&+ header 'History of every row' && check_history+}++# ─── control ──────────────────────────────────────────────────────────+run_control() {+ local flag sessions before after+ header "Install ${RELEASE}" && install_release || return 1+ header 'Seed a T2-shaped Host'+ new_host T2 true && seed_history "$HOST_T2" 2 1 && snapshot_served "$HOST_T2" || return 1+ header 'Upgrade with the merge flag off'+ # Set on the release HCC, which ignores it; the setup's client-side apply+ # keeps an env entry it never owned, so the head HCC starts with "false".+ log "FAULT:flag-false CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false on deployment/${HCC_DEPLOY}"+ FLAG_FALSE_SET=1+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false >/dev/null ||+ { fail 'flag-false: kubectl set env failed'; return 1; }+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||+ { fail 'flag-false: HCC rollout did not complete'; return 1; }+ upgrade_to_head || return 1+ flag="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" -o json | jq -r '+ [.spec.template.spec.containers[].env[]? | select(.name == "CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE") | .value] | first // "<unset>"')"+ [ "$flag" = false ] || { fail "control: the head HCC runs with the flag '${flag}', the control did not happen"; return 1; }+ ok 'control: the head HCC runs with CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=false'+ wait_upgraded_pod "$HOST_T2" || return 1+ jq -e '.inits == [] and .mounts == [{mountPath:"/workspace", name:"workspace"}]' <<<"$(pod_storage)" >/dev/null ||+ { fail "control: T2 Pod storage is $(pod_storage), expected the root mount without the merge init"; return 1; }+ [ "$(store_path "$HOST_T2")" = /workspace/state.db ] || { fail 'control: T2 did not open /workspace/state.db'; return 1; }+ ok 'control: T2 starts on the root mount with no merge init'+ run_layout inventory+ if [ "$LAYOUT_RC" != 1 ] || ! grep -q '^FLAG_FALSE_WITH_SPLIT_TREES=' "$LAYOUT_OUT"; then+ fail "control: inventory exited ${LAYOUT_RC} without FLAG_FALSE_WITH_SPLIT_TREES"+ return 1+ fi+ ok "control: inventory reports $(grep '^FLAG_FALSE_WITH_SPLIT_TREES=' "$LAYOUT_OUT")"+ # The oracle has to run (sessions 200) and then disagree with the baseline.+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${HOST_T2}/sessions?agent=${HOST_T2}&limit=100" || true+ [ "$HTTP_CODE" = 200 ] || { fail "control: sessions returned ${HTTP_CODE}"; return 1; }+ sessions="$HTTP_BODY"+ before="$(cat "${SCRATCH}/${HOST_T2}.chats")"+ after="$(chat_set "$sessions" "$HOST_T2")"+ if chats_match "$before" "$after"; then+ fail "control: T2 still serves its history (${after}); the control did not fail"+ return 1+ fi+ echo "E2E_UPGRADE_CONTROL_DIVERGED ${HOST_T2} before=${before} after=${after}"+ ok 'control: history check fails with the flag off, as expected'+}++# ─── Cleanup ──────────────────────────────────────────────────────────+cleanup() {+ local status=$? failed=0 name pod channel+ set +eu+ while IFS= read -r pod; do+ kctl delete pod "$pod" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ done <"$FIXTURE_POD_REGISTRY"+ for channel in "${CREATED_CHANNELS[@]}"; do+ kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1+ done+ restore_hcc_cadence || { failed=1; echo "HCC cadence restore failed; check deployment/${HCC_DEPLOY}" >&2; }+ if [ "$FLAG_FALSE_SET" = 1 ]; then+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE- >/dev/null 2>&1 || failed=1+ fi+ for name in "${CREATED_HOSTS[@]}"; do+ admin DELETE "${CAPI}/admin/hosts/${name}" >/dev/null 2>&1+ case "$HTTP_CODE" in 200 | 404) ;; *) failed=1; echo "delete Host ${name} returned ${HTTP_CODE}" >&2 ;; esac+ [ -z "$USER_ID" ] || set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1+ [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1+ done+ for name in "${CREATED_HOSTS[@]}"; do+ kctl wait --for=delete "deployment/${name}" -n "$HOST_NS" --timeout=120s >/dev/null 2>&1+ kctl delete pvc "${name}-workspace" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1+ done+ rm -rf "$SCRATCH"+ [ "$failed" = 0 ] || { status=1; fail 'cleanup left resources behind (see stderr)'; }+ print_results || status=1+ echo "E2E_UPGRADE_HEAD=${HEAD_SHA}"+ if [ "$MODE" = main ]; then+ echo "E2E_UPGRADE_ROWS_EXECUTED=${ROWS_DONE} E2E_UPGRADE_ROWS_DEFINED=${#ROWS[@]}"+ if [ "$status" = 0 ] && [ "$LANE_OK" = 1 ] && [ "$ROWS_DONE" = "${#ROWS[@]}" ]; then+ echo 'E2E_UPGRADE_MAIN=PASS'+ else+ echo 'E2E_UPGRADE_MAIN=FAIL'+ status=1+ fi+ else+ if [ "$status" = 0 ] && [ "$LANE_OK" = 1 ]; then+ echo 'E2E_UPGRADE_CONTROL=FAILED_AS_EXPECTED'+ echo 'E2E_UPGRADE_RESULT=PASS'+ else+ echo 'E2E_UPGRADE_CONTROL=NOT_PROVEN'+ echo 'E2E_UPGRADE_RESULT=FAIL'+ status=1+ fi+ fi+ echo "E2E_UPGRADE_EXIT=${status}"+ exit "$status"+}+trap cleanup EXIT++header "#825 upgrade lane ${RELEASE} → ${HEAD_SHA:0:9} (${MODE}, run ${RUN}, ${E2E_KUBECONTEXT})"+if [ "$MODE" = main ]; then run_main && LANE_OK=1; else run_control && LANE_OK=1; fidiff --git a/scripts/e2e/e2e-hcc-storage-layout-switch.sh b/scripts/e2e/e2e-hcc-storage-layout-switch.shindex 619e40d09..8a5534f9c 100755--- a/scripts/e2e/e2e-hcc-storage-layout-switch.sh+++ b/scripts/e2e/e2e-hcc-storage-layout-switch.sh@@ -99,227 +99,8 @@ FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID=''HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()-# ─── HTTP ─────────────────────────────────────────────────────────────-# http METHOD URL [BODY] [MAX_SECONDS] → HTTP_CODE, HTTP_BODY. A transport-# failure is HTTP_CODE=000 and a non-zero return. The body goes to curl on-# stdin, never in argv, because login bodies carry passwords.-http() {- local method=$1 url=$2 body=${3:-} max=${4:-60} resp- local args=(-sS -m "$max" -w '\n%{http_code}' -X "$method" "$url")- [ ${#HTTP_HEADERS[@]} -eq 0 ] || args+=("${HTTP_HEADERS[@]}")- [ -z "$body" ] || args+=(-H 'Content-Type: application/json' --data-binary @-)- if ! resp="$(printf '%s' "$body" | curl "${args[@]}")"; then- HTTP_CODE=000 HTTP_BODY=''- return 1- fi- HTTP_CODE="$(tail -n1 <<<"$resp")"- HTTP_BODY="$(sed '$d' <<<"$resp")"-}-# auth_header KIND LINE: write one header line to a 0600 file under $SCRATCH-# and print its path. Credentials reach curl through `-H @file`, never argv,-# where any local user can read them with ps.-auth_header() {- local file="${SCRATCH}/${1}.header"- (umask 077 && printf '%s\n' "$2" >"$file")- printf '%s' "$file"-}-admin() { HTTP_HEADERS=(-H "@$(auth_header admin "Cookie: control_ui_admin_session=${ADMIN_COOKIE}")"); http "$@"; }-rpc() { HTTP_HEADERS=(-H "@$(auth_header rpc "Authorization: Bearer ${RPC_TOKEN}")"); http "$@"; }--admin_login() {- local headers="${SCRATCH}/admin-login.headers" line cookie- HTTP_HEADERS=(-D "$headers")- http POST "${CAPI}/admin/auth/login" \- "$(jq -cn --arg u "$ADMIN_USERNAME" --arg p "$ADMIN_PASSWORD" '{username:$u,password:$p}')" || true- [[ "$HTTP_CODE" =~ ^2 ]] || { fail "admin login returned ${HTTP_CODE}"; return 1; }- while IFS= read -r line; do- line="${line%$'\r'}"- case "$(tr '[:upper:]' '[:lower:]' <<<"$line")" in- set-cookie:*control_ui_admin_session=*)- cookie="${line#*control_ui_admin_session=}"- ADMIN_COOKIE="${cookie%%;*}"- ;;- esac- done <"$headers"- rm -f "$headers"- [ -n "$ADMIN_COOKIE" ] || { fail 'admin login set no control_ui_admin_session cookie'; return 1; }-}--user_login() {- HTTP_HEADERS=()- http POST "${EXT_BASE}/api/v1/auth/password-login" \- "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')" || true- [ "$HTTP_CODE" = 200 ] || { fail "password login for ${DEV_EMAIL} returned ${HTTP_CODE} (a 429 means the lockout window; wait, do not retry blindly)"; return 1; }- SESSION_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'password login returned no token'; return 1; }-}--# The rpc token names every Host this run created so far.-mint_rpc_token() {- local refs- refs="$(printf '%s\n' "${CREATED_HOSTS[@]}" | jq -R . | jq -cs .)"- HTTP_HEADERS=(-H "@$(auth_header session "Authorization: Bearer ${SESSION_TOKEN}")")- http POST "${EXT_BASE}/api/v1/rpc/token" "$(jq -cn --argjson refs "$refs" '{hostRefs:$refs,scopes:[- "host:message:invoke","host:session:read","host:status:read","host:health:read",- "host:wake:write","host:model:write"]}')" || true- [[ "$HTTP_CODE" =~ ^2 ]] || { fail "rpc token mint returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }- RPC_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'rpc token response has no token'; return 1; }-}--# ─── Host lifecycle through the admin API ─────────────────────────────-# PUT /admin/users/:id/agents filters names against the active agent set, so a-# just-created Host can lag; retry until the response echoes it back.-set_agent_grant() {- local url=$1 name=$2 op=$3 attempt=1 names expected- while :; do- admin GET "$url" || true- [ "$HTTP_CODE" = 200 ] || { fail "GET ${url} returned ${HTTP_CODE}"; return 1; }- expected="$(jq -c '((.agentNames // []) + (.deletedAgentNames // []))' <<<"$HTTP_BODY")"- if [ "$op" = add ]; then- names="$(jq -c --arg a "$name" '((.agentNames // []) + [$a]) | unique' <<<"$HTTP_BODY")"- else- names="$(jq -c --arg a "$name" '(.agentNames // []) - [$a]' <<<"$HTTP_BODY")"- fi- admin PUT "$url" "$(jq -cn --argjson n "$names" --argjson e "$expected" \- '{agentNames:$n,expectedCurrentAgentNames:$e}')" || true- if [[ "$HTTP_CODE" =~ ^2 ]]; then- if [ "$op" = remove ] ||- jq -e --arg a "$name" '(.agentNames // []) | index($a) != null' <<<"$HTTP_BODY" >/dev/null; then- return 0- fi- fi- [ "$attempt" -lt 20 ] || { fail "${op} grant '${name}' on ${url} did not converge (${HTTP_CODE})"; return 1; }- attempt=$((attempt + 1))- sleep 3- done-}--grant_host() {- set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$1" add || return 1- [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$1" add-}--# create_host NAME STATELESS(true|false): clones the seeded source Host's spec-# (context, secret, model) without desktop or channels and creates it via the API.-create_host() {- local name=$1 stateless=$2 spec- spec="$(jq -c --arg n "$name" --argjson s "$stateless" '- .spec + {host:$n} | del(.channels) | del(.desktop)- | .lifecycle = {stateless:$s}' <<<"$SOURCE_HOST_JSON")"- admin POST "${CAPI}/admin/hosts" "$(jq -cn --arg n "$name" --argjson s "$spec" '{metadata:{name:$n},spec:$s}')" || true- [ "$HTTP_CODE" = 201 ] || { fail "create Host ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }- CREATED_HOSTS+=("$name")- grant_host "$name" || return 1- mint_rpc_token-}--# set_lifecycle NAME STATELESS(true|false): GET then PUT with the read's-# resourceVersion, echoing the full spec as Control UI does.-set_lifecycle() {- local name=$1 stateless=$2 rv spec- admin GET "${CAPI}/admin/hosts/${name}" || true- [ "$HTTP_CODE" = 200 ] || { fail "GET Host ${name} returned ${HTTP_CODE}"; return 1; }- rv="$(jq -er '.metadata.resourceVersion' <<<"$HTTP_BODY")" || { fail "Host ${name} read has no resourceVersion"; return 1; }- spec="$(jq -c --argjson s "$stateless" '.spec | .lifecycle = {stateless:$s}' <<<"$HTTP_BODY")"- admin PUT "${CAPI}/admin/hosts/${name}" \- "$(jq -cn --arg rv "$rv" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true- [ "$HTTP_CODE" = 200 ] || { fail "PUT lifecycle stateless=${stateless} on ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }- log "lifecycle request on ${name}: stateless=${stateless}"-}--# ─── Pods and the lifecycle witness ───────────────────────────────────-host_pods() { kctl get pods -n "$HOST_NS" -l "app=$1" -o json; }--# wait_new_ready_pod NAME OLD_UID TIMEOUT: exactly one Pod exists, it is Ready,-# and its UID differs from OLD_UID. Sets POD_JSON, POD_UID.-wait_new_ready_pod() {- local name=$1 old=$2 timeout=$3 elapsed=0 pods- while [ "$elapsed" -lt "$timeout" ]; do- if pods="$(host_pods "$name")" && POD_JSON="$(jq -ce --arg old "$old" '- .items | select(length == 1) | .[0]- | select(.metadata.deletionTimestamp == null and .metadata.uid != $old)- | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))' <<<"$pods")"; then- POD_UID="$(jq -r '.metadata.uid' <<<"$POD_JSON")"- jq -r '.spec.nodeName' <<<"$POD_JSON" >"${SCRATCH}/${name}.node"- return 0- fi- sleep 3- elapsed=$((elapsed + 3))- done- fail "${name}: no new Ready Pod within ${timeout}s"- kctl get pods -n "$HOST_NS" -l "app=${name}" -o wide >&2 || true- kctl describe pods -n "$HOST_NS" -l "app=${name}" 2>&1 | tail -n 40 >&2 || true- return 1-}--# Storage seen by the running Pod: workspace mounts, storage env, init names.-pod_storage() {- jq -cS '{- mounts: [.spec.containers[] | select(.name == "mcp-host") | .volumeMounts[] | select(.name == "workspace")],- store: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_STORE") | .value] | first),- dbDir: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_DB_DIR") | .value] | first),- inits: [.spec.initContainers[]? | .name]- }' <<<"$POD_JSON"-}--# assert_pod NAME STATELESS WORKSPACE_PATH: lifecycle markers of the new Pod-# match the requested lifecycle and its storage is the one root layout.-assert_pod() {- local name=$1 stateless=$2 path=$3 env_flag priority merge_exit storage db- env_flag="$(jq -r '[.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") | .value] | first // ""' <<<"$POD_JSON")"- priority="$(jq -r '.spec.priorityClassName // ""' <<<"$POD_JSON")"- if [ "$stateless" = true ]; then- [ "$env_flag" = true ] && [ "$priority" = clerum-interactive-host ] ||- { fail "${name}: Pod is not stateless (env=${env_flag:-<none>} priority=${priority:-<none>})"; return 1; }- else- [ -z "$env_flag" ] && [ "$priority" != clerum-interactive-host ] ||- { fail "${name}: Pod is not stateful (env=${env_flag} priority=${priority})"; return 1; }- fi- storage="$(pod_storage)"- jq -e --arg p "$path" '- .mounts == [{mountPath:$p, name:"workspace"}] and .store == "sqlite"- and .dbDir == $p and .inits == ["workspace-layout-merge"]' <<<"$storage" >/dev/null ||- { fail "${name}: Pod storage is not the root layout: ${storage}"; return 1; }- merge_exit="$(jq -r '[.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge") | .state.terminated.exitCode] | first // "none"' <<<"$POD_JSON")"- [ "$merge_exit" = 0 ] || { fail "${name}: workspace-layout-merge exit ${merge_exit}"; return 1; }- if [ -s "${SCRATCH}/${name}.storage" ] && [ "$(cat "${SCRATCH}/${name}.storage")" != "$storage" ]; then- fail "${name}: storage changed across the switch: $(cat "${SCRATCH}/${name}.storage") -> ${storage}"- return 1- fi- printf '%s' "$storage" >"${SCRATCH}/${name}.storage"- db="$(store_path "$name")" || { fail "${name}: no store-opened log line (reason above)"; return 1; }- [ "$db" = "${path}/state.db" ] || { fail "${name}: mcp-host opened ${db}, expected ${path}/state.db"; return 1; }- if [ -s "${SCRATCH}/${name}.dbpath" ] && [ "$(cat "${SCRATCH}/${name}.dbpath")" != "$db" ]; then- fail "${name}: store changed across the switch: $(cat "${SCRATCH}/${name}.dbpath") -> ${db}"- return 1- fi- printf '%s' "$db" >"${SCRATCH}/${name}.dbpath"- ok "${name}: Pod ${POD_UID:0:8} stateless=${stateless}, root layout at ${path}, merge exit 0, store ${db}"-}--# store_path NAME: the database the running mcp-host opened, read from its one-# "[ConversationStore] opened" log line (kubectl logs, no exec). mcp-host-# writes the line after the worker's first answer, which can come after Pod-# Ready, so the log is re-read for up to STORE_LOG_TIMEOUT seconds. Callers-# run it inside $(...), so the reason goes to stderr and the caller records-# the FAIL.-store_path() {- local pod lines deadline=$((SECONDS + STORE_LOG_TIMEOUT))- pod="$(jq -r '.metadata.name' <<<"$POD_JSON")"- while :; do- lines="$(kctl logs "$pod" -n "$HOST_NS" -c mcp-host |- jq -Rr 'fromjson? | select(.msg == "[ConversationStore] opened") | .dbPath')" ||- { echo "${1}: could not read the mcp-host log of ${pod}" >&2; return 1; }- if [ -n "$lines" ]; then- [ "$(wc -l <<<"$lines" | tr -d ' ')" = 1 ] ||- { echo "${1}: expected one store-opened line in ${pod}, got: ${lines}" >&2; return 1; }- printf '%s' "$lines"- return 0- fi- [ "$SECONDS" -lt "$deadline" ] ||- { echo "${1}: expected one store-opened line in ${pod}, got: <none> after ${STORE_LOG_TIMEOUT}s" >&2; return 1; }- sleep 1- done-}+# shellcheck source=scripts/e2e/e2e-825-lib.sh+source "${SCRIPT_DIR}/e2e-825-lib.sh"# switch_and_witness NAME STATELESS: request the switch, then require a new# Pod with the requested lifecycle and the same storage as before.@@ -330,191 +111,6 @@ switch_and_witness() {assert_pod "$name" "$stateless" /workspace}-# ─── Conversations and the history oracle ─────────────────────────────-# history_digest MESSAGES_JSON → "<turns> <sha256>". The oracle refuses an-# empty catalog: zero turns is never a match.-history_digest() {- local turns sum- turns="$(jq -r '.turns | length' <<<"$1")"- sum="$(jq -cS '[.turns[] | {number, user_input, response}]' <<<"$1" | shasum -a 256 | cut -d' ' -f1)"- printf '%s %s' "$turns" "$sum"-}-history_matches() { # EXPECTED_DIGEST MESSAGES_JSON- local actual- actual="$(history_digest "$2")"- [ "${actual%% *}" -gt 0 ] && [ "$actual" = "$1" ]-}-session_listed() { # SESSIONS_JSON HOST THREAD- jq -e --arg t "$3" --arg a "$2" '[.items[]? | select(.chatId == $t and .agent == $a)] | length == 1' <<<"$1" >/dev/null-}-# chat_set SESSIONS_JSON HOST → "<n>:<sorted chatIds>" of the Host's sessions.-chat_set() {- jq -r --arg a "$2" '[.items[]? | select(.agent == $a) | .chatId] | unique | "\(length):\(join(","))"' <<<"$1"-}-# chats_match BEFORE AFTER: the same set on both sides, and not an empty one.-chats_match() { [ "${1%%:*}" -gt 0 ] && [ "$1" = "$2" ]; }--# fetch_messages records nothing; read_messages counts a non-200 as a failure.-fetch_messages() { # HOST THREAD → HTTP_CODE, HTTP_BODY- rpc GET "${RPC_BASE}/api/v1/rpc/hosts/$1/sessions/$1/$2/messages?limit=100" || true-}-read_messages() { # HOST THREAD → HTTP_BODY- fetch_messages "$1" "$2"- [ "$HTTP_CODE" = 200 ] || { fail "$1/$2: messages returned ${HTTP_CODE}"; return 1; }-}--# send_turn HOST THREAD CONTENT: a suspended Host answers 503 host_waking-# while it wakes; retry only that, bounded by TURN_TIMEOUT. The turn counts-# only once it is readable with its marker.-send_turn() {- local host=$1 thread=$2 content=$3 elapsed=0 before- fetch_messages "$host" "$thread"- case "$HTTP_CODE" in- 200) before="$(jq -r '.turns | length' <<<"$HTTP_BODY")" ;;- 404) before=0 ;;- *) fail "${host}/${thread}: messages returned ${HTTP_CODE} before the turn"; return 1 ;;- esac- while :; do- rpc POST "${RPC_BASE}/api/v1/rpc/hosts/${host}/messages" \- "$(jq -cn --arg c "$content" --arg t "$thread" '{content:$c,threadId:$t}')" "$TURN_TIMEOUT" || true- [[ "$HTTP_CODE" =~ ^2 ]] && break- if [ "$HTTP_CODE" = 503 ] && jq -e '.code == "host_waking"' <<<"$HTTP_BODY" >/dev/null 2>&1 &&- [ "$elapsed" -lt "$TURN_TIMEOUT" ]; then- sleep 5- elapsed=$((elapsed + 5))- continue- fi- fail "${host}/${thread}: turn returned ${HTTP_CODE}: ${HTTP_BODY:0:300}"- return 1- done- elapsed=0- while [ "$elapsed" -lt "$TURN_TIMEOUT" ]; do- fetch_messages "$host" "$thread"- if [ "$HTTP_CODE" = 200 ] && jq -e --argjson n "$((before + 1))" --arg c "$content" \- '(.turns | length) == $n and (.turns[-1].user_input | contains($c))' <<<"$HTTP_BODY" >/dev/null; then- return 0- fi- sleep 3- elapsed=$((elapsed + 3))- done- fail "${host}/${thread}: turn ${before}+1 not persisted within ${TURN_TIMEOUT}s"- return 1-}--# seed_history HOST THREADS TURNS: writes conversations and records digests.-seed_history() {- local host=$1 threads=$2 turns=$3 i j thread- : >"${SCRATCH}/${host}.threads"- for i in $(seq 1 "$threads"); do- thread="e2e-825-${host}-t${i}"- for j in $(seq 1 "$turns"); do- send_turn "$host" "$thread" "e2e-825 ${RUN} ${host} thread ${i} turn ${j}: reply with one short sentence." || return 1- done- read_messages "$host" "$thread" || return 1- printf '%s %s\n' "$thread" "$(history_digest "$HTTP_BODY")" >>"${SCRATCH}/${host}.threads"- done- rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true- [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }- chat_set "$HTTP_BODY" "$host" >"${SCRATCH}/${host}.chats"- ok "${host}: ${threads} conversation(s) x ${turns} turn(s) written"-}--# assert_history HOST TRANSITION: the Host lists the same chatId set as before-# the transition; every recorded conversation is unchanged; then a resume turn-# is persisted with the next turn number.-assert_history() {- local host=$1 transition=$2 thread digest sessions before after next=''- rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true- [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }- sessions="$HTTP_BODY"- before="$(cat "${SCRATCH}/${host}.chats")"- after="$(chat_set "$sessions" "$host")"- chats_match "$before" "$after" ||- { fail "${host} ${transition}: chatId set before=${before} after=${after}"; return 1; }- echo "E2E_825_CONTINUITY ${host} ${transition} before=${before} after=${after} dbPath=$(cat "${SCRATCH}/${host}.dbpath")"- while read -r thread digest; do- session_listed "$sessions" "$host" "$thread" || { fail "${host}/${thread}: not in the session list"; return 1; }- read_messages "$host" "$thread" || return 1- history_matches "$digest" "$HTTP_BODY" ||- { fail "${host}/${thread}: history changed (expected ${digest}, got $(history_digest "$HTTP_BODY"))"; return 1; }- send_turn "$host" "$thread" "e2e-825 ${RUN} resume ${thread}: reply with one short sentence." || return 1- read_messages "$host" "$thread" || return 1- next+="${thread} $(history_digest "$HTTP_BODY")"$'\n'- done <"${SCRATCH}/${host}.threads"- printf '%s' "$next" >"${SCRATCH}/${host}.threads"- ok "${host}: history listed, unchanged and resumed ($(wc -l <"${SCRATCH}/${host}.threads" | tr -d ' ') conversation(s))"-}--# ─── FAULT:pvc-fixture ────────────────────────────────────────────────-# pvc_fixture HOST ro|rw LABEL SCRIPT [ARGS...]: runs SCRIPT with the PVC root-# as $1 in a Pod on the Host's last node. The Pod output goes to stdout; the-# log and failure lines go to stderr so a caller capturing stdout reads only it.-pvc_fixture() {- local host=$1 mode=$2 label=$3 script=$4 pod node ro body elapsed=0 phase- shift 4- pod="$(truncate_rfc1123 "e2e825-${label}-${host}")"- node="$(cat "${SCRATCH}/${host}.node")"- ro=false- [ "$mode" = ro ] && ro=true- log "FAULT:pvc-fixture ${label} on ${host}-workspace (${mode})" >&2- body="$(jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$FIXTURE_IMAGE" \- --arg claim "${host}-workspace" --argjson ro "$ro" --arg script "$script" \- --args '{- apiVersion:"v1", kind:"Pod",- metadata:{name:$pod, namespace:$ns, labels:{purpose:"e2e-825-pvc-fixture"}},- spec:{- restartPolicy:"Never", automountServiceAccountToken:false, enableServiceLinks:false,- terminationGracePeriodSeconds:0, nodeName:$node,- securityContext:{runAsUser:1001, runAsGroup:1001, runAsNonRoot:true, seccompProfile:{type:"RuntimeDefault"}},- containers:[{- name:"fixture", image:$image, imagePullPolicy:"IfNotPresent",- command:(["/bin/sh","-c",$script,"fixture","/mnt/pvc"] + $ARGS.positional),- resources:{requests:{cpu:"10m",memory:"32Mi"}, limits:{cpu:"200m",memory:"128Mi"}},- securityContext:{allowPrivilegeEscalation:false, readOnlyRootFilesystem:true, capabilities:{drop:["ALL"]}},- volumeMounts:[{name:"pvc", mountPath:"/mnt/pvc", readOnly:$ro}]- }],- volumes:[{name:"pvc", persistentVolumeClaim:{claimName:$claim, readOnly:$ro}}]- }}' "$@")"- # Callers usually run this function inside $(...), so an array append would- # be lost with the subshell; the registry file is what cleanup() reads.- # Registered before the create so a create that times out after the API- # accepted it is still cleaned up.- printf '%s\n' "$pod" >>"$FIXTURE_POD_REGISTRY"- kctl create -f - <<<"$body" >/dev/null || { fail "${label}: fixture Pod create failed" >&2; return 1; }- while [ "$elapsed" -lt 180 ]; do- phase="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"- case "$phase" in Succeeded | Failed) break ;; esac- sleep 2- elapsed=$((elapsed + 2))- done- kctl logs "$pod" -n "$HOST_NS" >"${SCRATCH}/${pod}.log" 2>&1 || true- # A Pod left behind keeps the RWO claim attached to its node and blocks the- # next fixture or Host Pod on another node, so a failed delete fails the step.- if ! kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then- fail "${label}: fixture Pod ${pod} delete failed" >&2- return 1- fi- if [ "$phase" != Succeeded ]; then- fail "${label}: fixture Pod ended ${phase:-<timeout>}: $(tail -n 5 "${SCRATCH}/${pod}.log")" >&2- return 1- fi- cat "${SCRATCH}/${pod}.log"-}--# Listing of the PVC root: `<type> <path>`, plus ` <target>` for a symlink,-# lost+found pruned. The fixture image has busybox find, which lacks -printf.-# shellcheck disable=SC2016 # expanded inside the fixture Pod-LIST_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -print | while IFS= read -r p; do- if [ -L "$p" ]; then printf "l %s %s\n" "$p" "$(readlink "$p")"- elif [ -d "$p" ]; then printf "d %s\n" "$p"- else printf "f %s\n" "$p"; fi-done | LC_ALL=C sort'-# shellcheck disable=SC2016-HASH_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum'-# shellcheck disable=SC2016-WRITE_FILE_SCRIPT='mkdir -p "$1/e2e-825" && printf "%s\n" "$2" > "$1/e2e-825/$3"'-# shellcheck disable=SC2016-READ_FILE_SCRIPT='cat "$1/e2e-825/$2"'# The inventoried dev split shape: DB family and a backup under state/, every# other root entry under workspace/, plus an agent-made symlink (A7.16).# shellcheck disable=SC2016@@ -533,103 +129,6 @@ ls -la state workspace'SEED_TWO_DB_SCRIPT='set -eu; cd "$1"; mkdir state workspace; cp state.db state/state.dbprintf "notes\n" > state/notes.txt; printf "x\n" > workspace/x; ls -la . state workspace'-read_seed_file() { # HOST NAME EXPECTED- local got- got="$(pvc_fixture "$1" ro read-file "$READ_FILE_SCRIPT" "$2")" || return 1- [ "$got" = "$3" ] || { fail "$1: seeded file e2e-825/$2 reads '${got}', expected '$3'"; return 1; }- ok "$1: seeded file e2e-825/$2 intact at the PVC root"-}--# ─── Operator script ──────────────────────────────────────────────────-# layout_row inventory|verify HOST EXITS: runs deploy/scripts/host-layout-merge.sh-# on this profile and prints the result of the Host's PVC row. EXITS is a case-# pattern; the fleet verdict is left alone because other Hosts on the profile-# decide it, and exit 2 (read failure) never matches.-layout_row() {- local cmd=$1 host=$2 exits=$3 rc=0 out row- out="$(mktemp "${SCRATCH}/layout-${cmd}-${host}.XXXXXX")"- log "host-layout-merge.sh ${cmd} for ${HOST_NS}/${host}-workspace" >&2- bash "$LAYOUT_SCRIPT" "$cmd" --context "$E2E_KUBECONTEXT" >"$out" 2>"${out}.err" || rc=$?- # shellcheck disable=SC2254 # EXITS is a case pattern on purpose- case "$rc" in- $exits) ;;- *)- fail "${host}: host-layout-merge.sh ${cmd} exited ${rc} (expected ${exits}): $(tail -n 3 "${out}.err" | tr '\n' ' ') $(tail -n 3 "$out" | tr '\n' ' ')" >&2- return 1- ;;- esac- [ "$rc" != 2 ] || { fail "${host}: host-layout-merge.sh ${cmd} could not read the cluster" >&2; return 1; }- row="$(grep -F "PVC ${HOST_NS}/${host}-workspace " "$out")" ||- { fail "${host}: host-layout-merge.sh ${cmd} printed no row for ${host}-workspace" >&2; return 1; }- printf '%s\n' "${row#*result=}"-}--# ─── Suspend and wake ─────────────────────────────────────────────────-wait_suspended() { # HOST: state suspended, replicas 0, zero Pods (Terminating included)- local host=$1 elapsed=0 state reps pods- log "FAULT:hcc-cadence waiting for ${host} to suspend"- while [ "$elapsed" -lt "$SUSPEND_TIMEOUT" ]; do- state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"- reps="$(kctl get deployment "$host" -n "$HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"- pods="$(host_pods "$host" | jq '.items | length')" || pods=unknown- if [ "$state" = suspended ] && [ "$reps" = 0 ] && [ "$pods" = 0 ]; then- ok "${host}: suspended with zero Pods"- POD_UID=''- return 0- fi- sleep 5- elapsed=$((elapsed + 5))- done- fail "${host}: not suspended within ${SUSPEND_TIMEOUT}s (state=${state:-<none>} replicas=${reps:-?} pods=${pods})"- return 1-}--post_wake() { # HOST- rpc POST "${RPC_BASE}/api/v1/rpc/hosts/$1/wake" || true- case "$HTTP_CODE" in 200 | 202 | 204) log "wake requested for $1" ;; *)- fail "$1: wake returned ${HTTP_CODE}: ${HTTP_BODY}"- return 1- ;;- esac-}--set_hcc_cadence() {- local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES- CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS) k- HCC_ENV_SAVED=''- for k in "${keys[@]}"; do- HCC_ENV_SAVED+="${k}=$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \- -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}")"$'\n'- done- log 'FAULT:hcc-cadence idle 1 min, drain 20 s, heartbeat poll 5 s'- HCC_ENV_CHANGED=1- kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \- CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=1 CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=1 \- CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=20000 CONTEXT_MAPPER_HEARTBEAT_POLL_MS=5000 >/dev/null ||- { fail 'hcc-cadence: kubectl set env failed'; return 1; }- kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||- { fail 'hcc-cadence: HCC rollout did not settle'; return 1; }-}--restore_hcc_cadence() {- local args=() key val actual- [ "$HCC_ENV_CHANGED" = 1 ] || return 0- while IFS='=' read -r key val; do- [ -n "$key" ] || continue- if [ -n "$val" ]; then args+=("${key}=${val}"); else args+=("${key}-"); fi- done <<<"$HCC_ENV_SAVED"- kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null || return 1- kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null || return 1- while IFS='=' read -r key val; do- [ -n "$key" ] || continue- actual="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \- -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${key}')].value}")"- [ "$actual" = "$val" ] || { echo "HCC ${key} restored to '${actual}', expected '${val}'" >&2; return 1; }- done <<<"$HCC_ENV_SAVED"- HCC_ENV_CHANGED=0- log 'FAULT:hcc-cadence restored'-}-# A new stateless Host with history, a seeded file, then suspended.suspended_host_with_history() { # NAMEcreate_host "$1" true || return 1diff --git a/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs b/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjsindex d8901d587..57ac45c5c 100644--- a/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs+++ b/scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs@@ -6,11 +6,16 @@ import path from 'node:path'import test from 'node:test'import { fileURLToPath } from 'node:url'-// Runs single helpers of e2e-hcc-storage-layout-switch.sh under bash with-// stubbed kubectl/curl, so their failure paths are checked without a cluster.+// Runs single helpers of the #825 live gates (the switch script, the upgrade+// lane and their shared e2e-825-lib.sh) under bash with stubbed kubectl/curl,+// so their failure paths are checked without a cluster.-const SCRIPT = fileURLToPath(new URL('./e2e-hcc-storage-layout-switch.sh', import.meta.url))-const SOURCE = readFileSync(SCRIPT, 'utf8')+const SCRIPTS = [+ './e2e-hcc-storage-layout-switch.sh',+ './e2e-825-lib.sh',+ './e2e-hcc-layout-upgrade.sh',+].map(rel => fileURLToPath(new URL(rel, import.meta.url)))+const SOURCE = SCRIPTS.map(file => readFileSync(file, 'utf8')).join('\n')function helper(name) {const lines = SOURCE.split('\n')@@ -113,7 +118,74 @@ admin GET http://127.0.0.1:1/admin && rpc GET http://127.0.0.1:2/rpc && echo "CO}})-test('no curl call in the script puts a Cookie or Authorization header in argv', () => {+test('main_evidence_ok accepts only a passing main log of the same HEAD', () => {+ const dir = mkdtempSync(path.join(tmpdir(), 'e2e-825-upgrade-evidence-'))+ try {+ const head = 'a'.repeat(40)+ const write = (name, lines) => {+ const file = path.join(dir, name)+ writeFileSync(file, `${lines.join('\n')}\n`)+ return file+ }+ const cases = [+ ['pass', write('pass', ['PASS: x', 'E2E_UPGRADE_HEAD=' + head, 'E2E_UPGRADE_MAIN=PASS']), 0],+ ['fail', write('fail', ['E2E_UPGRADE_HEAD=' + head, 'E2E_UPGRADE_MAIN=FAIL']), 1],+ [+ 'other head',+ write('other', ['E2E_UPGRADE_HEAD=' + 'b'.repeat(40), 'E2E_UPGRADE_MAIN=PASS']),+ 1,+ ],+ [+ 'head prefix only',+ write('prefix', ['E2E_UPGRADE_HEAD=' + head.slice(0, 9), 'E2E_UPGRADE_MAIN=PASS']),+ 1,+ ],+ ['missing file', path.join(dir, 'absent'), 1],+ ]+ const body = `+${helper('main_evidence_ok')}+for f in "$@"; do main_evidence_ok "$f" "$HEAD" && echo 0 || echo 1; done+`+ const run = spawnSync('bash', ['-c', body, 'bash', ...cases.map(c => c[1])], {+ encoding: 'utf8',+ env: { PATH: process.env.PATH, HEAD: head },+ })+ assert.equal(run.status, 0, run.stderr)+ assert.deepEqual(+ run.stdout.trim().split('\n').map(Number),+ cases.map(c => c[2]),+ cases.map(c => c[0]).join(', ')+ )+ } finally {+ rmSync(dir, { recursive: true, force: true })+ }+})++function runUpgrade(args, env) {+ return spawnSync('bash', [SCRIPTS[2], ...args], {+ encoding: 'utf8',+ env: { PATH: process.env.PATH, HOME: process.env.HOME, ...env },+ timeout: 30_000,+ })+}++test('the upgrade lane refuses an unknown mode before touching the cluster', () => {+ const run = runUpgrade(['bogus'], { KUBECONTEXT: 'clerum-test-upgrade-lane-0123abcd' })+ assert.equal(run.status, 1, run.stdout + run.stderr)+ assert.match(run.stdout + run.stderr, /usage: e2e-hcc-layout-upgrade\.sh main\|control/)+ // Witness: a valid mode passes the mode check and stops at a later precondition.+ const valid = runUpgrade(['main'], { KUBECONTEXT: 'minikube' })+ assert.equal(valid.status, 1, valid.stdout + valid.stderr)+ assert.doesNotMatch(valid.stdout + valid.stderr, /usage:/)+})++test('the upgrade lane refuses a context that is not a branch profile', () => {+ const run = runUpgrade(['main'], { KUBECONTEXT: 'minikube' })+ assert.equal(run.status, 1, run.stdout + run.stderr)+ assert.match(run.stdout + run.stderr, /select an explicit branch-scoped Minikube context/)+})++test('no curl call in the #825 scripts puts a Cookie or Authorization header in argv', () => {assert.doesNotMatch(SOURCE, /-H\s+["']?(Cookie|Authorization):/i)// Witness: the credentials are still sent, through auth_header.const uses = SOURCE.match(/auth_header (admin|session|rpc) /g) ?? []