Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): storage layout across lifecycle switches and fix the stateless oracle

Alfredo Lopez8774c8a73c7f
diff --git a/Makefile b/Makefile
index b74d68007..6691f37fe 100644
--- a/Makefile
+++ b/Makefile
@@ -1401,6 +1401,11 @@ test-e2e-stateless-suspend-wake: ## Run stateless suspend/wake E2E gate (Phase 1
@echo "Running stateless suspend/wake E2E gate..."
E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-stateless-suspend-wake.sh
+.PHONY: test-e2e-hcc-storage-layout-switch
+test-e2e-hcc-storage-layout-switch: ## Run the #825 storage-layout gate across lifecycle switches (needs an owned branch profile, E2E_PROFILE_PORTS_ENV, E2E_DEV_LOGIN_EMAIL, forwards held)
+ @echo "Running HCC storage-layout switch E2E gate..."
+ E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-hcc-storage-layout-switch.sh
+
.PHONY: test-e2e-stateless-wake-recovery
test-e2e-stateless-wake-recovery: ## Run stateless wake-recovery latency gate (R1 warm draining / R2 cold suspended / R3 drained-window p95 budgets; needs port-forwards + seeded chatllm-stateless)
@echo "Running stateless wake-recovery latency gate..."
diff --git a/scripts/e2e/e2e-hcc-storage-layout-switch.sh b/scripts/e2e/e2e-hcc-storage-layout-switch.sh
new file mode 100755
index 000000000..c9d131e24
--- /dev/null
+++ b/scripts/e2e/e2e-hcc-storage-layout-switch.sh
@@ -0,0 +1,851 @@
+#!/usr/bin/env bash
+# Branch-owned live gate for #825: one storage layout across lifecycle switches.
+#
+# Every Host mounts its workspace PVC root at the workspace path and keeps its
+# conversation database there (CLERUM_SESSION_DB_DIR=<workspacePath>), whatever
+# its lifecycle. A `workspace-layout-merge` init container moves a PVC split by
+# the earlier stateless template back to the root before mcp-host starts.
+#
+# Each scenario creates its own Hosts through the control-api admin API and
+# changes them only through that API. The trap deletes them. Named fixtures
+# (logged as FAULT:<name>) are the only other cluster writes:
+# - hcc-cadence: shortens the HCC idle window so stateless Hosts suspend in
+# about a minute (opt-in E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1),
+# restored in the trap;
+# - pvc-fixture: a short-lived Pod (the Host image, uid 1001, no token) that
+# mounts a Host PVC to list it or, on a suspended Host, to seed a tree;
+# - channel: a CommunicationChannel CR that makes HCC run a stateless Host
+# as stateful;
+# - pod-replace: deleting a Host Pod so its Deployment starts a new one.
+# No kubectl exec into Host Pods, no SQLite writes, no apply/patch of Hosts or
+# their Deployments.
+#
+# Scenarios: S1-S7 (plan A7.10), RACE (A7.13), NEG_RV (stale resourceVersion),
+# NEG_CTRL (the history oracle rejects an empty catalog). The final line is
+# E2E_825_RESULT=PASS only when every defined scenario ran and passed.
+set -euo pipefail
+umask 077
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+# shellcheck source=scripts/e2e/load-dotenv.sh
+source "${REPO_ROOT}/scripts/e2e/load-dotenv.sh"
+dotenv_load_canonical_root "${REPO_ROOT}"
+# shellcheck source=scripts/e2e/admin-credentials.sh
+source "${REPO_ROOT}/scripts/e2e/admin-credentials.sh"
+# shellcheck source=scripts/e2e/e2e-lib.sh
+source "${SCRIPT_DIR}/e2e-lib.sh"
+
+die() { fail "$*"; exit 1; }
+
+# ─── Preconditions ────────────────────────────────────────────────────
+if [ -z "$E2E_KUBECONTEXT" ] || ! is_branch_scoped_e2e_context "$E2E_KUBECONTEXT"; then
+ die 'select an explicit branch-scoped Minikube context (KUBECONTEXT=<profile>)'
+fi
+require_safe_kube_context
+[ "${E2E_ALLOW_STATELESS_CADENCE_ACCELERATION:-0}" = 1 ] ||
+ die 'set E2E_ALLOW_STATELESS_CADENCE_ACCELERATION=1 (S4-S7 shorten the HCC idle window on this owned profile)'
+[ -n "${E2E_PROFILE_PORTS_ENV:-}" ] && [ -r "$E2E_PROFILE_PORTS_ENV" ] ||
+ die 'provide E2E_PROFILE_PORTS_ENV from the owned branch profile'
+[ -n "${E2E_DEV_LOGIN_EMAIL:-}" ] || die 'set E2E_DEV_LOGIN_EMAIL to the seeded login ([email protected] on a T2 profile)'
+profile_port_value() {
+ awk -F= -v key="$1" '$1 == key { sub(/^[^=]*=/, ""); print; exit }' "$E2E_PROFILE_PORTS_ENV"
+}
+CONTROL_API_BASE_URL="$(profile_port_value CONTROL_API_URL)"
+CONTROL_UI_BASE_URL="$(profile_port_value CONTROL_UI_URL)"
+EXTERNAL_REST_API_BASE_URL="$(profile_port_value EXTERNAL_REST_API_URL)"
+RPC_PROXY_BASE_URL="$(profile_port_value RPC_PROXY_URL)"
+export CONTROL_API_BASE_URL CONTROL_UI_BASE_URL EXTERNAL_REST_API_BASE_URL RPC_PROXY_BASE_URL
+require_branch_profile_urls "$E2E_KUBECONTEXT" "$E2E_PROFILE_PORTS_ENV" ||
+ die 'profile URLs do not match the owned ports'
+kctl get nodes -o json | jq -e --arg c "$E2E_KUBECONTEXT" \
+ 'any(.items[]; .metadata.labels["minikube.k8s.io/name"]==$c)' >/dev/null ||
+ die 'target is not the selected Minikube profile'
+
+HOST_NS=mcp-host CHANNEL_NS=channels
+HCC_NS=control-plane HCC_DEPLOY=host-context-controller
+SOURCE_HOST="${E2E_SOURCE_HOST_REF:-chatllm}"
+CAPI="${CONTROL_API_BASE_URL%/}/api/v1"
+EXT_BASE="${EXTERNAL_REST_API_BASE_URL%/}"
+RPC_BASE="${RPC_PROXY_BASE_URL%/}"
+ADMIN_USERNAME="${ADMIN_USERNAME:-admin}"
+ADMIN_PASSWORD="${ADMIN_PASSWORD:-}"
+[ -n "$ADMIN_PASSWORD" ] || ADMIN_PASSWORD="$(e2e_resolve_admin_password "$REPO_ROOT" || true)"
+[ -n "$ADMIN_PASSWORD" ] || die 'admin password could not be resolved (ADMIN_PASSWORD or the canonical .env)'
+DEV_EMAIL="$E2E_DEV_LOGIN_EMAIL"
+DEV_PASSWORD="${E2E_USER_PASSWORD:-$ADMIN_PASSWORD}"
+# od reads a fixed byte count; `tr </dev/urandom | head` dies of SIGPIPE under pipefail.
+RUN="$(od -An -N3 -tx1 /dev/urandom | tr -d ' \n')"
+POD_READY_TIMEOUT="${E2E_825_POD_READY_TIMEOUT:-300}"
+SUSPEND_TIMEOUT="${E2E_825_SUSPEND_TIMEOUT:-300}"
+TURN_TIMEOUT="${E2E_825_TURN_TIMEOUT:-240}"
+BASE_INIT_FIXTURE="${REPO_ROOT}/host-context-controller/test/__fixtures__/base-workspace-layout-init.sh"
+[ -r "$BASE_INIT_FIXTURE" ] || die "missing base init fixture ${BASE_INIT_FIXTURE}"
+
+DEFINED_SCENARIOS=(NEG_CTRL NEG_RV S1 S2 S3 RACE S4 S5 S6 S7)
+EXECUTED=0 SCENARIO_FAILURES=0 SCENARIO_LOG=''
+CREATED_HOSTS=() CREATED_CHANNELS=() FIXTURE_PODS=()
+HCC_ENV_SAVED='' HCC_ENV_CHANGED=0
+SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/e2e-825.XXXXXX")"
+ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID=''
+HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()
+
+# ─── HTTP ─────────────────────────────────────────────────────────────
+# http METHOD URL [BODY] [MAX_SECONDS] → HTTP_CODE, HTTP_BODY. A transport
+# failure is HTTP_CODE=000 and a non-zero return.
+http() {
+ local method=$1 url=$2 body=${3:-} max=${4:-60} resp
+ local args=(-sS -m "$max" -w '\n%{http_code}' -X "$method" "$url")
+ [ ${#HTTP_HEADERS[@]} -eq 0 ] || args+=("${HTTP_HEADERS[@]}")
+ [ -z "$body" ] || args+=(-H 'Content-Type: application/json' -d "$body")
+ if ! resp="$(curl "${args[@]}")"; then
+ HTTP_CODE=000 HTTP_BODY=''
+ return 1
+ fi
+ HTTP_CODE="$(tail -n1 <<<"$resp")"
+ HTTP_BODY="$(sed '$d' <<<"$resp")"
+}
+admin() { HTTP_HEADERS=(-H "Cookie: control_ui_admin_session=${ADMIN_COOKIE}"); http "$@"; }
+rpc() { HTTP_HEADERS=(-H "Authorization: Bearer ${RPC_TOKEN}"); http "$@"; }
+
+admin_login() {
+ local headers="${SCRATCH}/admin-login.headers" line cookie
+ HTTP_HEADERS=(-D "$headers")
+ http POST "${CAPI}/admin/auth/login" \
+ "$(jq -cn --arg u "$ADMIN_USERNAME" --arg p "$ADMIN_PASSWORD" '{username:$u,password:$p}')" || true
+ [[ "$HTTP_CODE" =~ ^2 ]] || { fail "admin login returned ${HTTP_CODE}"; return 1; }
+ while IFS= read -r line; do
+ line="${line%$'\r'}"
+ case "$(tr '[:upper:]' '[:lower:]' <<<"$line")" in
+ set-cookie:*control_ui_admin_session=*)
+ cookie="${line#*control_ui_admin_session=}"
+ ADMIN_COOKIE="${cookie%%;*}"
+ ;;
+ esac
+ done <"$headers"
+ rm -f "$headers"
+ [ -n "$ADMIN_COOKIE" ] || { fail 'admin login set no control_ui_admin_session cookie'; return 1; }
+}
+
+user_login() {
+ HTTP_HEADERS=()
+ http POST "${EXT_BASE}/api/v1/auth/password-login" \
+ "$(jq -cn --arg e "$DEV_EMAIL" --arg p "$DEV_PASSWORD" '{email:$e,password:$p}')" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "password login for ${DEV_EMAIL} returned ${HTTP_CODE} (a 429 means the lockout window; wait, do not retry blindly)"; return 1; }
+ SESSION_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'password login returned no token'; return 1; }
+}
+
+# The rpc token names every Host this run created so far.
+mint_rpc_token() {
+ local refs
+ refs="$(printf '%s\n' "${CREATED_HOSTS[@]}" | jq -R . | jq -cs .)"
+ HTTP_HEADERS=(-H "Authorization: Bearer ${SESSION_TOKEN}")
+ http POST "${EXT_BASE}/api/v1/rpc/token" "$(jq -cn --argjson refs "$refs" '{hostRefs:$refs,scopes:[
+ "host:message:invoke","host:session:read","host:status:read","host:health:read",
+ "host:wake:write","host:model:write"]}')" || true
+ [[ "$HTTP_CODE" =~ ^2 ]] || { fail "rpc token mint returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ RPC_TOKEN="$(jq -er '.token' <<<"$HTTP_BODY")" || { fail 'rpc token response has no token'; return 1; }
+}
+
+# ─── Host lifecycle through the admin API ─────────────────────────────
+# PUT /admin/users/:id/agents filters names against the active agent set, so a
+# just-created Host can lag; retry until the response echoes it back.
+set_agent_grant() {
+ local url=$1 name=$2 op=$3 attempt=1 names expected
+ while :; do
+ admin GET "$url" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "GET ${url} returned ${HTTP_CODE}"; return 1; }
+ expected="$(jq -c '((.agentNames // []) + (.deletedAgentNames // []))' <<<"$HTTP_BODY")"
+ if [ "$op" = add ]; then
+ names="$(jq -c --arg a "$name" '((.agentNames // []) + [$a]) | unique' <<<"$HTTP_BODY")"
+ else
+ names="$(jq -c --arg a "$name" '(.agentNames // []) - [$a]' <<<"$HTTP_BODY")"
+ fi
+ admin PUT "$url" "$(jq -cn --argjson n "$names" --argjson e "$expected" \
+ '{agentNames:$n,expectedCurrentAgentNames:$e}')" || true
+ if [[ "$HTTP_CODE" =~ ^2 ]]; then
+ if [ "$op" = remove ] ||
+ jq -e --arg a "$name" '(.agentNames // []) | index($a) != null' <<<"$HTTP_BODY" >/dev/null; then
+ return 0
+ fi
+ fi
+ [ "$attempt" -lt 20 ] || { fail "${op} grant '${name}' on ${url} did not converge (${HTTP_CODE})"; return 1; }
+ attempt=$((attempt + 1))
+ sleep 3
+ done
+}
+
+grant_host() {
+ set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$1" add || return 1
+ [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$1" add
+}
+
+# create_host NAME STATELESS(true|false): clones the seeded source Host's spec
+# (context, secret, model) without desktop or channels and creates it via the API.
+create_host() {
+ local name=$1 stateless=$2 spec
+ spec="$(jq -c --arg n "$name" --argjson s "$stateless" '
+ .spec + {host:$n} | del(.channels) | del(.desktop)
+ | .lifecycle = {stateless:$s}' <<<"$SOURCE_HOST_JSON")"
+ admin POST "${CAPI}/admin/hosts" "$(jq -cn --arg n "$name" --argjson s "$spec" '{metadata:{name:$n},spec:$s}')" || true
+ [ "$HTTP_CODE" = 201 ] || { fail "create Host ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ CREATED_HOSTS+=("$name")
+ grant_host "$name" || return 1
+ mint_rpc_token
+}
+
+# set_lifecycle NAME STATELESS(true|false): GET then PUT with the read's
+# resourceVersion, echoing the full spec as Control UI does.
+set_lifecycle() {
+ local name=$1 stateless=$2 rv spec
+ admin GET "${CAPI}/admin/hosts/${name}" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "GET Host ${name} returned ${HTTP_CODE}"; return 1; }
+ rv="$(jq -er '.metadata.resourceVersion' <<<"$HTTP_BODY")" || { fail "Host ${name} read has no resourceVersion"; return 1; }
+ spec="$(jq -c --argjson s "$stateless" '.spec | .lifecycle = {stateless:$s}' <<<"$HTTP_BODY")"
+ admin PUT "${CAPI}/admin/hosts/${name}" \
+ "$(jq -cn --arg rv "$rv" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "PUT lifecycle stateless=${stateless} on ${name} returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ log "lifecycle request on ${name}: stateless=${stateless}"
+}
+
+# ─── Pods and the lifecycle witness ───────────────────────────────────
+host_pods() { kctl get pods -n "$HOST_NS" -l "app=$1" -o json; }
+
+# wait_new_ready_pod NAME OLD_UID TIMEOUT: exactly one Pod exists, it is Ready,
+# and its UID differs from OLD_UID. Sets POD_JSON, POD_UID.
+wait_new_ready_pod() {
+ local name=$1 old=$2 timeout=$3 elapsed=0 pods
+ while [ "$elapsed" -lt "$timeout" ]; do
+ if pods="$(host_pods "$name")" && POD_JSON="$(jq -ce --arg old "$old" '
+ .items | select(length == 1) | .[0]
+ | select(.metadata.deletionTimestamp == null and .metadata.uid != $old)
+ | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))' <<<"$pods")"; then
+ POD_UID="$(jq -r '.metadata.uid' <<<"$POD_JSON")"
+ jq -r '.spec.nodeName' <<<"$POD_JSON" >"${SCRATCH}/${name}.node"
+ return 0
+ fi
+ sleep 3
+ elapsed=$((elapsed + 3))
+ done
+ fail "${name}: no new Ready Pod within ${timeout}s"
+ kctl get pods -n "$HOST_NS" -l "app=${name}" -o wide >&2 || true
+ kctl describe pods -n "$HOST_NS" -l "app=${name}" 2>&1 | tail -n 40 >&2 || true
+ return 1
+}
+
+# Storage seen by the running Pod: workspace mounts, storage env, init names.
+pod_storage() {
+ jq -cS '{
+ mounts: [.spec.containers[] | select(.name == "mcp-host") | .volumeMounts[] | select(.name == "workspace")],
+ store: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_STORE") | .value] | first),
+ dbDir: ([.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_SESSION_DB_DIR") | .value] | first),
+ inits: [.spec.initContainers[]? | .name]
+ }' <<<"$POD_JSON"
+}
+
+# assert_pod NAME STATELESS WORKSPACE_PATH: lifecycle markers of the new Pod
+# match the requested lifecycle and its storage is the one root layout.
+assert_pod() {
+ local name=$1 stateless=$2 path=$3 env_flag priority merge_exit storage
+ env_flag="$(jq -r '[.spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") | .value] | first // ""' <<<"$POD_JSON")"
+ priority="$(jq -r '.spec.priorityClassName // ""' <<<"$POD_JSON")"
+ if [ "$stateless" = true ]; then
+ [ "$env_flag" = true ] && [ "$priority" = clerum-interactive-host ] ||
+ { fail "${name}: Pod is not stateless (env=${env_flag:-<none>} priority=${priority:-<none>})"; return 1; }
+ else
+ [ -z "$env_flag" ] && [ "$priority" != clerum-interactive-host ] ||
+ { fail "${name}: Pod is not stateful (env=${env_flag} priority=${priority})"; return 1; }
+ fi
+ storage="$(pod_storage)"
+ jq -e --arg p "$path" '
+ .mounts == [{mountPath:$p, name:"workspace"}] and .store == "sqlite"
+ and .dbDir == $p and .inits == ["workspace-layout-merge"]' <<<"$storage" >/dev/null ||
+ { fail "${name}: Pod storage is not the root layout: ${storage}"; return 1; }
+ merge_exit="$(jq -r '[.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge") | .state.terminated.exitCode] | first // "none"' <<<"$POD_JSON")"
+ [ "$merge_exit" = 0 ] || { fail "${name}: workspace-layout-merge exit ${merge_exit}"; return 1; }
+ if [ -s "${SCRATCH}/${name}.storage" ] && [ "$(cat "${SCRATCH}/${name}.storage")" != "$storage" ]; then
+ fail "${name}: storage changed across the switch: $(cat "${SCRATCH}/${name}.storage") -> ${storage}"
+ return 1
+ fi
+ printf '%s' "$storage" >"${SCRATCH}/${name}.storage"
+ ok "${name}: Pod ${POD_UID:0:8} stateless=${stateless}, root layout at ${path}, merge exit 0"
+}
+
+# switch_and_witness NAME STATELESS: request the switch, then require a new
+# Pod with the requested lifecycle and the same storage as before.
+switch_and_witness() {
+ local name=$1 stateless=$2 old=$POD_UID
+ set_lifecycle "$name" "$stateless" || return 1
+ wait_new_ready_pod "$name" "$old" "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" "$stateless" /workspace
+}
+
+# ─── Conversations and the history oracle ─────────────────────────────
+# history_digest MESSAGES_JSON → "<turns> <sha256>". The oracle refuses an
+# empty catalog: zero turns is never a match.
+history_digest() {
+ local turns sum
+ turns="$(jq -r '.turns | length' <<<"$1")"
+ sum="$(jq -cS '[.turns[] | {number, user_input, response}]' <<<"$1" | shasum -a 256 | cut -d' ' -f1)"
+ printf '%s %s' "$turns" "$sum"
+}
+history_matches() { # EXPECTED_DIGEST MESSAGES_JSON
+ local actual
+ actual="$(history_digest "$2")"
+ [ "${actual%% *}" -gt 0 ] && [ "$actual" = "$1" ]
+}
+session_listed() { # SESSIONS_JSON HOST THREAD
+ jq -e --arg t "$3" --arg a "$2" '[.items[]? | select(.chatId == $t and .agent == $a)] | length == 1' <<<"$1" >/dev/null
+}
+
+# fetch_messages records nothing; read_messages counts a non-200 as a failure.
+fetch_messages() { # HOST THREAD → HTTP_CODE, HTTP_BODY
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/$1/sessions/$1/$2/messages?limit=100" || true
+}
+read_messages() { # HOST THREAD → HTTP_BODY
+ fetch_messages "$1" "$2"
+ [ "$HTTP_CODE" = 200 ] || { fail "$1/$2: messages returned ${HTTP_CODE}"; return 1; }
+}
+
+# send_turn HOST THREAD CONTENT: a suspended Host answers 503 host_waking
+# while it wakes; retry only that, bounded by TURN_TIMEOUT. The turn counts
+# only once it is readable with its marker.
+send_turn() {
+ local host=$1 thread=$2 content=$3 elapsed=0 before
+ fetch_messages "$host" "$thread"
+ case "$HTTP_CODE" in
+ 200) before="$(jq -r '.turns | length' <<<"$HTTP_BODY")" ;;
+ 404) before=0 ;;
+ *) fail "${host}/${thread}: messages returned ${HTTP_CODE} before the turn"; return 1 ;;
+ esac
+ while :; do
+ rpc POST "${RPC_BASE}/api/v1/rpc/hosts/${host}/messages" \
+ "$(jq -cn --arg c "$content" --arg t "$thread" '{content:$c,threadId:$t}')" "$TURN_TIMEOUT" || true
+ [[ "$HTTP_CODE" =~ ^2 ]] && break
+ if [ "$HTTP_CODE" = 503 ] && jq -e '.code == "host_waking"' <<<"$HTTP_BODY" >/dev/null 2>&1 &&
+ [ "$elapsed" -lt "$TURN_TIMEOUT" ]; then
+ sleep 5
+ elapsed=$((elapsed + 5))
+ continue
+ fi
+ fail "${host}/${thread}: turn returned ${HTTP_CODE}: ${HTTP_BODY:0:300}"
+ return 1
+ done
+ elapsed=0
+ while [ "$elapsed" -lt "$TURN_TIMEOUT" ]; do
+ fetch_messages "$host" "$thread"
+ if [ "$HTTP_CODE" = 200 ] && jq -e --argjson n "$((before + 1))" --arg c "$content" \
+ '(.turns | length) == $n and (.turns[-1].user_input | contains($c))' <<<"$HTTP_BODY" >/dev/null; then
+ return 0
+ fi
+ sleep 3
+ elapsed=$((elapsed + 3))
+ done
+ fail "${host}/${thread}: turn ${before}+1 not persisted within ${TURN_TIMEOUT}s"
+ return 1
+}
+
+# seed_history HOST THREADS TURNS: writes conversations and records digests.
+seed_history() {
+ local host=$1 threads=$2 turns=$3 i j thread
+ : >"${SCRATCH}/${host}.threads"
+ for i in $(seq 1 "$threads"); do
+ thread="e2e-825-${host}-t${i}"
+ for j in $(seq 1 "$turns"); do
+ send_turn "$host" "$thread" "e2e-825 ${RUN} ${host} thread ${i} turn ${j}: reply with one short sentence." || return 1
+ done
+ read_messages "$host" "$thread" || return 1
+ printf '%s %s\n' "$thread" "$(history_digest "$HTTP_BODY")" >>"${SCRATCH}/${host}.threads"
+ done
+ ok "${host}: ${threads} conversation(s) x ${turns} turn(s) written"
+}
+
+# assert_history HOST: every recorded conversation is listed and unchanged;
+# then a resume turn is persisted with the next turn number.
+assert_history() {
+ local host=$1 thread digest sessions next=''
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${host}/sessions?agent=${host}&limit=100" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "${host}: sessions returned ${HTTP_CODE}"; return 1; }
+ sessions="$HTTP_BODY"
+ while read -r thread digest; do
+ session_listed "$sessions" "$host" "$thread" || { fail "${host}/${thread}: not in the session list"; return 1; }
+ read_messages "$host" "$thread" || return 1
+ history_matches "$digest" "$HTTP_BODY" ||
+ { fail "${host}/${thread}: history changed (expected ${digest}, got $(history_digest "$HTTP_BODY"))"; return 1; }
+ send_turn "$host" "$thread" "e2e-825 ${RUN} resume ${thread}: reply with one short sentence." || return 1
+ read_messages "$host" "$thread" || return 1
+ next+="${thread} $(history_digest "$HTTP_BODY")"$'\n'
+ done <"${SCRATCH}/${host}.threads"
+ printf '%s' "$next" >"${SCRATCH}/${host}.threads"
+ ok "${host}: history listed, unchanged and resumed ($(wc -l <"${SCRATCH}/${host}.threads" | tr -d ' ') conversation(s))"
+}
+
+# ─── FAULT:pvc-fixture ────────────────────────────────────────────────
+# pvc_fixture HOST ro|rw LABEL SCRIPT [ARGS...]: runs SCRIPT with the PVC root
+# as $1 in a Pod on the Host's last node. The Pod output goes to stdout; the
+# log and failure lines go to stderr so a caller capturing stdout reads only it.
+pvc_fixture() {
+ local host=$1 mode=$2 label=$3 script=$4 pod node ro body elapsed=0 phase
+ shift 4
+ pod="$(truncate_rfc1123 "e2e825-${label}-${host}")"
+ node="$(cat "${SCRATCH}/${host}.node")"
+ ro=false
+ [ "$mode" = ro ] && ro=true
+ log "FAULT:pvc-fixture ${label} on ${host}-workspace (${mode})" >&2
+ body="$(jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$FIXTURE_IMAGE" \
+ --arg claim "${host}-workspace" --argjson ro "$ro" --arg script "$script" \
+ --args '{
+ apiVersion:"v1", kind:"Pod",
+ metadata:{name:$pod, namespace:$ns, labels:{purpose:"e2e-825-pvc-fixture"}},
+ spec:{
+ restartPolicy:"Never", automountServiceAccountToken:false, enableServiceLinks:false,
+ terminationGracePeriodSeconds:0, nodeName:$node,
+ securityContext:{runAsUser:1001, runAsGroup:1001, runAsNonRoot:true, seccompProfile:{type:"RuntimeDefault"}},
+ containers:[{
+ name:"fixture", image:$image, imagePullPolicy:"IfNotPresent",
+ command:(["/bin/sh","-c",$script,"fixture","/mnt/pvc"] + $ARGS.positional),
+ resources:{requests:{cpu:"10m",memory:"32Mi"}, limits:{cpu:"200m",memory:"128Mi"}},
+ securityContext:{allowPrivilegeEscalation:false, readOnlyRootFilesystem:true, capabilities:{drop:["ALL"]}},
+ volumeMounts:[{name:"pvc", mountPath:"/mnt/pvc", readOnly:$ro}]
+ }],
+ volumes:[{name:"pvc", persistentVolumeClaim:{claimName:$claim, readOnly:$ro}}]
+ }}' "$@")"
+ kctl create -f - <<<"$body" >/dev/null || { fail "${label}: fixture Pod create failed" >&2; return 1; }
+ FIXTURE_PODS+=("$pod")
+ while [ "$elapsed" -lt 180 ]; do
+ phase="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+ case "$phase" in Succeeded | Failed) break ;; esac
+ sleep 2
+ elapsed=$((elapsed + 2))
+ done
+ kctl logs "$pod" -n "$HOST_NS" >"${SCRATCH}/${pod}.log" 2>&1 || true
+ kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1 || true
+ if [ "$phase" != Succeeded ]; then
+ fail "${label}: fixture Pod ended ${phase:-<timeout>}: $(tail -n 5 "${SCRATCH}/${pod}.log")" >&2
+ return 1
+ fi
+ cat "${SCRATCH}/${pod}.log"
+}
+
+# Listing of the PVC root: `<type> <path>`, plus ` <target>` for a symlink,
+# lost+found pruned. The fixture image has busybox find, which lacks -printf.
+# shellcheck disable=SC2016 # expanded inside the fixture Pod
+LIST_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -print | while IFS= read -r p; do
+ if [ -L "$p" ]; then printf "l %s %s\n" "$p" "$(readlink "$p")"
+ elif [ -d "$p" ]; then printf "d %s\n" "$p"
+ else printf "f %s\n" "$p"; fi
+done | LC_ALL=C sort'
+# shellcheck disable=SC2016
+HASH_SCRIPT='cd "$1" && find . -path ./lost+found -prune -o -type f -print0 | LC_ALL=C sort -z | xargs -0 sha256sum'
+# shellcheck disable=SC2016
+WRITE_FILE_SCRIPT='mkdir -p "$1/e2e-825" && printf "%s\n" "$2" > "$1/e2e-825/$3"'
+# shellcheck disable=SC2016
+READ_FILE_SCRIPT='cat "$1/e2e-825/$2"'
+# The inventoried dev split shape: DB family and a backup under state/, every
+# other root entry under workspace/, plus an agent-made symlink (A7.16).
+# shellcheck disable=SC2016
+SEED_SPLIT_SCRIPT='set -eu; cd "$1"; mkdir state workspace
+for e in * .[!.]*; do
+ [ -e "$e" ] || [ -L "$e" ] || continue
+ case "$e" in lost+found|state|workspace) continue ;; state.db*) mv "$e" state/ ;; *) mv "$e" workspace/ ;; esac
+done
+cp state/state.db state/state.db.pre-20261014.bak
+ln -s /tmp/e2e-825-link-target workspace/e2e-825-link
+ls -la state workspace'
+# Two live database families: the root one and a copy under state/ (F2).
+# shellcheck disable=SC2016
+SEED_TWO_DB_SCRIPT='set -eu; cd "$1"; mkdir state; cp state.db state/state.db; ls -la . state'
+
+read_seed_file() { # HOST NAME EXPECTED
+ local got
+ got="$(pvc_fixture "$1" ro read-file "$READ_FILE_SCRIPT" "$2")" || return 1
+ [ "$got" = "$3" ] || { fail "$1: seeded file e2e-825/$2 reads '${got}', expected '$3'"; return 1; }
+ ok "$1: seeded file e2e-825/$2 intact at the PVC root"
+}
+
+# ─── Suspend and wake ─────────────────────────────────────────────────
+wait_suspended() { # HOST: state suspended, replicas 0, zero Pods (Terminating included)
+ local host=$1 elapsed=0 state reps pods
+ log "FAULT:hcc-cadence waiting for ${host} to suspend"
+ while [ "$elapsed" -lt "$SUSPEND_TIMEOUT" ]; do
+ state="$(kctl get host "$host" -n "$HOST_NS" -o jsonpath='{.status.lifecycle.state}' 2>/dev/null || true)"
+ reps="$(kctl get deployment "$host" -n "$HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
+ pods="$(host_pods "$host" | jq '.items | length')" || pods=unknown
+ if [ "$state" = suspended ] && [ "$reps" = 0 ] && [ "$pods" = 0 ]; then
+ ok "${host}: suspended with zero Pods"
+ POD_UID=''
+ return 0
+ fi
+ sleep 5
+ elapsed=$((elapsed + 5))
+ done
+ fail "${host}: not suspended within ${SUSPEND_TIMEOUT}s (state=${state:-<none>} replicas=${reps:-?} pods=${pods})"
+ return 1
+}
+
+post_wake() { # HOST
+ rpc POST "${RPC_BASE}/api/v1/rpc/hosts/$1/wake" || true
+ case "$HTTP_CODE" in 200 | 202 | 204) log "wake requested for $1" ;; *)
+ fail "$1: wake returned ${HTTP_CODE}: ${HTTP_BODY}"
+ return 1
+ ;;
+ esac
+}
+
+set_hcc_cadence() {
+ local keys=(CONTEXT_MAPPER_STATELESS_IDLE_MINUTES CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES
+ CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS CONTEXT_MAPPER_HEARTBEAT_POLL_MS) k
+ HCC_ENV_SAVED=''
+ for k in "${keys[@]}"; do
+ HCC_ENV_SAVED+="${k}=$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${k}')].value}")"$'\n'
+ done
+ log 'FAULT:hcc-cadence idle 1 min, drain 20 s, heartbeat poll 5 s'
+ HCC_ENV_CHANGED=1
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ CONTEXT_MAPPER_STATELESS_IDLE_MINUTES=1 CONTEXT_MAPPER_STATELESS_IDLE_FLOOR_MINUTES=1 \
+ CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS=20000 CONTEXT_MAPPER_HEARTBEAT_POLL_MS=5000 >/dev/null ||
+ { fail 'hcc-cadence: kubectl set env failed'; return 1; }
+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null ||
+ { fail 'hcc-cadence: HCC rollout did not settle'; return 1; }
+}
+
+restore_hcc_cadence() {
+ local args=() key val actual
+ [ "$HCC_ENV_CHANGED" = 1 ] || return 0
+ while IFS='=' read -r key val; do
+ [ -n "$key" ] || continue
+ if [ -n "$val" ]; then args+=("${key}=${val}"); else args+=("${key}-"); fi
+ done <<<"$HCC_ENV_SAVED"
+ kctl set env "deployment/${HCC_DEPLOY}" -n "$HCC_NS" "${args[@]}" >/dev/null || return 1
+ kctl rollout status "deployment/${HCC_DEPLOY}" -n "$HCC_NS" --timeout=180s >/dev/null || return 1
+ while IFS='=' read -r key val; do
+ [ -n "$key" ] || continue
+ actual="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" \
+ -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name=='${key}')].value}")"
+ [ "$actual" = "$val" ] || { echo "HCC ${key} restored to '${actual}', expected '${val}'" >&2; return 1; }
+ done <<<"$HCC_ENV_SAVED"
+ HCC_ENV_CHANGED=0
+ log 'FAULT:hcc-cadence restored'
+}
+
+# A new stateless Host with history, a seeded file, then suspended.
+suspended_host_with_history() { # NAME
+ create_host "$1" true || return 1
+ wait_new_ready_pod "$1" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$1" true /workspace || return 1
+ seed_history "$1" 2 1 || return 1
+ pvc_fixture "$1" rw seed-file "$WRITE_FILE_SCRIPT" "seed ${RUN}" seed.txt >/dev/null || return 1
+ wait_suspended "$1"
+}
+
+# ─── Scenarios ────────────────────────────────────────────────────────
+scenario_NEG_CTRL() {
+ local digest empty='{"agent":"x","chatId":"y","totalTurns":0,"turns":[]}'
+ local real='{"turns":[{"number":1,"user_input":"a","response":"b"}]}'
+ digest="$(history_digest "$real")"
+ history_matches "$digest" "$real" || { fail 'NEG_CTRL witness: the oracle rejects a matching history'; return 1; }
+ if history_matches "$digest" "$empty"; then fail 'NEG_CTRL: the history oracle accepts an empty catalog'; return 1; fi
+ if history_matches "0 $(jq -cS '[]' <<<'null' | shasum -a 256 | cut -d' ' -f1)" "$empty"; then
+ fail 'NEG_CTRL: the history oracle accepts zero turns as a match'
+ return 1
+ fi
+ session_listed '{"items":[{"agent":"h","chatId":"t"}]}' h t || { fail 'NEG_CTRL witness: list oracle rejects a listed session'; return 1; }
+ if session_listed '{"items":[]}' h t; then fail 'NEG_CTRL: the list oracle accepts an empty catalog'; return 1; fi
+ ok 'NEG_CTRL: oracles reject an empty catalog and accept a real one'
+}
+
+scenario_NEG_RV() {
+ local name rv1 spec
+ name="$(truncate_rfc1123 "e2e-825-rv-${RUN}")"
+ create_host "$name" false || return 1
+ admin GET "${CAPI}/admin/hosts/${name}" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "NEG_RV: GET returned ${HTTP_CODE}"; return 1; }
+ rv1="$(jq -r '.metadata.resourceVersion' <<<"$HTTP_BODY")"
+ spec="$(jq -c '.spec | .lifecycle = {stateless:true}' <<<"$HTTP_BODY")"
+ admin PUT "${CAPI}/admin/hosts/${name}" "$(jq -cn --arg rv "$rv1" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
+ [ "$HTTP_CODE" = 200 ] || { fail "NEG_RV witness: fresh PUT returned ${HTTP_CODE}"; return 1; }
+ spec="$(jq -c '.lifecycle = {stateless:false}' <<<"$spec")"
+ admin PUT "${CAPI}/admin/hosts/${name}" "$(jq -cn --arg rv "$rv1" --argjson s "$spec" '{metadata:{resourceVersion:$rv},spec:$s}')" || true
+ if [ "$HTTP_CODE" != 409 ] || ! jq -e '.reason == "resource_changed"' <<<"$HTTP_BODY" >/dev/null; then
+ fail "NEG_RV: stale PUT returned ${HTTP_CODE}: ${HTTP_BODY}"
+ return 1
+ fi
+ [ "$(kctl get host "$name" -n "$HOST_NS" -o jsonpath='{.spec.lifecycle.stateless}')" = true ] ||
+ { fail 'NEG_RV: the stale PUT changed the lifecycle'; return 1; }
+ ok 'NEG_RV: stale resourceVersion → 409 resource_changed, lifecycle unchanged; fresh PUT applied'
+}
+
+# S1: stateful → stateless → stateful, 3 conversations x 2 turns, a file.
+scenario_S1() {
+ local name
+ name="$(truncate_rfc1123 "e2e-825-s1-${RUN}")"
+ create_host "$name" false || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" false /workspace || return 1
+ seed_history "$name" 3 2 || return 1
+ pvc_fixture "$name" rw seed-file "$WRITE_FILE_SCRIPT" "s1 ${RUN}" seed.txt >/dev/null || return 1
+ switch_and_witness "$name" true || return 1
+ assert_history "$name" || return 1
+ read_seed_file "$name" seed.txt "s1 ${RUN}" || return 1
+ switch_and_witness "$name" false || return 1
+ assert_history "$name" || return 1
+ read_seed_file "$name" seed.txt "s1 ${RUN}"
+}
+
+# S2: created stateless → stateful → stateless.
+scenario_S2() {
+ local name
+ name="$(truncate_rfc1123 "e2e-825-s2-${RUN}")"
+ create_host "$name" true || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" true /workspace || return 1
+ seed_history "$name" 2 2 || return 1
+ switch_and_witness "$name" false || return 1
+ assert_history "$name" || return 1
+ switch_and_witness "$name" true || return 1
+ assert_history "$name"
+}
+
+# S3: a CommunicationChannel makes HCC run the stateless Host as stateful;
+# deleting it returns the Host to stateless.
+scenario_S3() {
+ local name channel old reason elapsed=0
+ name="$(truncate_rfc1123 "e2e-825-s3-${RUN}")"
+ channel="$(truncate_rfc1123 "e2e-825-s3-${RUN}")"
+ create_host "$name" true || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" true /workspace || return 1
+ seed_history "$name" 2 1 || return 1
+ log "FAULT:channel ${CHANNEL_NS}/${channel} → ${name}"
+ printf '%s\n' 'apiVersion: clerum.io/v1alpha1' 'kind: CommunicationChannel' 'metadata:' \
+ " name: ${channel}" " namespace: ${CHANNEL_NS}" 'spec:' " hostRef: ${name}" ' telegram:' \
+ " - channelId: \"e2e-825-${RUN}\"" ' chatType: "private"' ' userIds: ["000000000"]' |
+ kctl create -f - >/dev/null || { fail 'S3: CommunicationChannel create failed'; return 1; }
+ CREATED_CHANNELS+=("$channel")
+ old=$POD_UID
+ wait_new_ready_pod "$name" "$old" "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" false /workspace || return 1
+ reason="$(kctl get host "$name" -n "$HOST_NS" -o jsonpath='{.status.conditions[?(@.type=="StatelessEnableRejected")].status}')"
+ [ "$reason" = True ] || { fail "S3: StatelessEnableRejected is '${reason}', expected True"; return 1; }
+ assert_history "$name" || return 1
+ kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --wait=true --timeout=60s >/dev/null ||
+ { fail 'S3: CommunicationChannel delete failed'; return 1; }
+ old=$POD_UID
+ wait_new_ready_pod "$name" "$old" "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" true /workspace || return 1
+ assert_history "$name"
+}
+
+# S4: switch to stateful while suspended, then back to stateless and wake.
+scenario_S4() {
+ local name
+ name="$(truncate_rfc1123 "e2e-825-s4-${RUN}")"
+ suspended_host_with_history "$name" || return 1
+ switch_and_witness "$name" false || return 1
+ assert_history "$name" || return 1
+ read_seed_file "$name" seed.txt "seed ${RUN}" || return 1
+ switch_and_witness "$name" true || return 1
+ wait_suspended "$name" || return 1
+ post_wake "$name" || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" true /workspace || return 1
+ assert_history "$name"
+}
+
+# S5: the inventoried split tree (state/ with a backup, workspace/ with a
+# symlink) merges on wake; history, file, backup and link at the root.
+scenario_S5() {
+ local name listing
+ name="$(truncate_rfc1123 "e2e-825-s5-${RUN}")"
+ suspended_host_with_history "$name" || return 1
+ pvc_fixture "$name" rw seed-split "$SEED_SPLIT_SCRIPT" >/dev/null || return 1
+ listing="$(pvc_fixture "$name" ro list-split "$LIST_SCRIPT")" || return 1
+ if ! grep -qx 'd ./state' <<<"$listing" || ! grep -qx 'f ./state/state.db'<<<"$listing"; then
+ fail "S5: seeded tree is not split: ${listing}"
+ return 1
+ fi
+ post_wake "$name" || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" true /workspace || return 1
+ listing="$(pvc_fixture "$name" ro list-merged "$LIST_SCRIPT")" || return 1
+ if grep -Eq '^d \./(state|workspace)$' <<<"$listing"; then fail "S5: split directories remain: ${listing}"; return 1; fi
+ if ! grep -qx 'f ./state.db'<<<"$listing" ||
+ ! grep -qx 'f ./state.db.pre-20261014.bak'<<<"$listing" ||
+ ! grep -qx 'l ./e2e-825-link /tmp/e2e-825-link-target' <<<"$listing"; then
+ fail "S5: merged tree lacks the database, backup or link at the root: ${listing}"
+ return 1
+ fi
+ ok "S5: split tree merged to the root"
+ assert_history "$name" || return 1
+ read_seed_file "$name" seed.txt "seed ${RUN}"
+}
+
+# S6: two live database families: the merge refuses, the Pod stays in
+# Init:Error and the tree is unchanged.
+scenario_S6() {
+ local name before after elapsed=0 pod logs
+ name="$(truncate_rfc1123 "e2e-825-s6-${RUN}")"
+ suspended_host_with_history "$name" || return 1
+ pvc_fixture "$name" rw seed-two-db "$SEED_TWO_DB_SCRIPT" >/dev/null || return 1
+ before="$(pvc_fixture "$name" ro hash-before "$HASH_SCRIPT")" || return 1
+ grep -q ' ./state/state.db$' <<<"$before" || { fail "S6 witness: state/state.db not seeded: ${before}"; return 1; }
+ post_wake "$name" || return 1
+ while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do
+ pod="$(host_pods "$name" | jq -r '[.items[] | select(any(.status.initContainerStatuses[]?;
+ .name == "workspace-layout-merge" and ((.state.terminated.exitCode // 0) == 1 or (.lastState.terminated.exitCode // 0) == 1)))][0].metadata.name // ""')"
+ [ -n "$pod" ] && break
+ sleep 3
+ elapsed=$((elapsed + 3))
+ done
+ [ -n "$pod" ] || { fail "S6: no Pod with workspace-layout-merge exit 1 within ${POD_READY_TIMEOUT}s"; return 1; }
+ logs="$(kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge --previous 2>/dev/null ||
+ kctl logs "$pod" -n "$HOST_NS" -c workspace-layout-merge)"
+ grep -q 'workspace layout merge refused: more than one live SQLite database across' <<<"$logs" ||
+ { fail "S6: merge refused without the F2 line: ${logs}"; return 1; }
+ if host_pods "$name" | jq -e 'any(.items[]; any(.status.conditions[]?; .type == "Ready" and .status == "True"))' >/dev/null; then
+ fail 'S6: a Pod became Ready on a refused tree'
+ return 1
+ fi
+ after="$(pvc_fixture "$name" ro hash-after "$HASH_SCRIPT")" || return 1
+ [ "$before" = "$after" ] || { fail "S6: tree changed after the refusal"; diff <(echo "$before") <(echo "$after") >&2 || true; return 1; }
+ ok 'S6: two live families → Init:Error with the F2 line, tree unchanged'
+}
+
+# S7: a merged tree re-split by the previous init (an HCC rollback), then
+# woken: the merge restores the root layout and the history.
+scenario_S7() {
+ local name before listing script
+ name="$(truncate_rfc1123 "e2e-825-s7-${RUN}")"
+ suspended_host_with_history "$name" || return 1
+ before="$(pvc_fixture "$name" ro list-before "$LIST_SCRIPT")" || return 1
+ script="$(cat "$BASE_INIT_FIXTURE")"
+ pvc_fixture "$name" rw base-init "$script" >/dev/null || return 1
+ listing="$(pvc_fixture "$name" ro list-split "$LIST_SCRIPT")" || return 1
+ if ! grep -qx 'd ./state' <<<"$listing" || ! grep -qx 'd ./workspace' <<<"$listing"; then
+ fail "S7: the previous init did not split the tree: ${listing}"
+ return 1
+ fi
+ post_wake "$name" || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" true /workspace || return 1
+ listing="$(pvc_fixture "$name" ro list-after "$LIST_SCRIPT")" || return 1
+ [ "$(grep -v ' ./state.db' <<<"$before")" = "$(grep -v ' ./state.db' <<<"$listing")" ] ||
+ { fail 'S7: root entries differ from the tree before the re-split'; diff <(echo "$before") <(echo "$listing") >&2 || true; return 1; }
+ ok 'S7: re-split tree merged back to the same root entries'
+ assert_history "$name" || return 1
+ read_seed_file "$name" seed.txt "seed ${RUN}"
+}
+
+# RACE: concurrent model selections on new chats create one session each and
+# never restart the Host (A7.13).
+race_host() { # NAME STATELESS
+ local name=$1 stateless=$2 model uid restarts round i chat codes pids
+ create_host "$name" "$stateless" || return 1
+ wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1
+ assert_pod "$name" "$stateless" /workspace || return 1
+ uid=$POD_UID
+ restarts="$(jq -r '[.status.containerStatuses[] | select(.name == "mcp-host") | .restartCount] | first' <<<"$POD_JSON")"
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${name}/models" || true
+ model="$(jq -er '.hostDefault' <<<"$HTTP_BODY")" || { fail "${name}: models returned ${HTTP_CODE}: ${HTTP_BODY}"; return 1; }
+ for round in 1 2 3 4 5; do
+ chat="e2e-825-race-${RUN}-${round}"
+ pids=()
+ for i in 1 2 3 4; do
+ curl -sS -m 60 -o /dev/null -w '%{http_code}\n' -X POST "${RPC_BASE}/api/v1/rpc/hosts/${name}/model" \
+ -H "Authorization: Bearer ${RPC_TOKEN}" -H 'Content-Type: application/json' \
+ -d "$(jq -cn --arg c "$chat" --arg m "$model" '{chatId:$c,model:$m}')" >"${SCRATCH}/race-${round}-${i}" &
+ pids+=("$!")
+ done
+ for i in "${pids[@]}"; do wait "$i" || true; done
+ codes="$(cat "${SCRATCH}"/race-"${round}"-* | sort | uniq -c | tr -s ' ' | tr '\n' ';')"
+ [ "$(cat "${SCRATCH}"/race-"${round}"-* | grep -cx 200)" = 4 ] ||
+ { fail "${name}: round ${round} model selections returned ${codes}"; return 1; }
+ send_turn "$name" "$chat" "e2e-825 ${RUN} race ${round}: reply with one short sentence." || return 1
+ rpc GET "${RPC_BASE}/api/v1/rpc/hosts/${name}/sessions?agent=${name}&limit=100" || true
+ session_listed "$HTTP_BODY" "$name" "$chat" || { fail "${name}: round ${round} chat not listed exactly once"; return 1; }
+ done
+ POD_JSON="$(host_pods "$name" | jq -c '.items[0]')"
+ [ "$(jq -r '.metadata.uid' <<<"$POD_JSON")" = "$uid" ] &&
+ [ "$(jq -r '[.status.containerStatuses[] | select(.name == "mcp-host") | .restartCount] | first' <<<"$POD_JSON")" = "$restarts" ] ||
+ { fail "${name}: the Pod restarted or was replaced during the race"; return 1; }
+ ok "${name}: 5 rounds x 4 concurrent selections, one session per chat, Pod ${uid:0:8} restartCount ${restarts}"
+}
+scenario_RACE() {
+ race_host "$(truncate_rfc1123 "e2e-825-race-sf-${RUN}")" false || return 1
+ race_host "$(truncate_rfc1123 "e2e-825-race-sl-${RUN}")" true
+}
+
+run_scenario() {
+ local id=$1
+ header "Scenario ${id}"
+ EXECUTED=$((EXECUTED + 1))
+ if "scenario_${id}"; then
+ ok "SCENARIO ${id} PASS"
+ SCENARIO_LOG+="${id}=PASS "
+ else
+ fail "SCENARIO ${id} FAIL"
+ SCENARIO_LOG+="${id}=FAIL "
+ SCENARIO_FAILURES=$((SCENARIO_FAILURES + 1))
+ fi
+}
+
+# ─── Cleanup ──────────────────────────────────────────────────────────
+cleanup() {
+ local status=$? failed=0 name pod channel
+ # bash 3.2 treats "${empty[@]}" as unbound under set -u.
+ set +eu
+ for pod in "${FIXTURE_PODS[@]}"; do
+ kctl delete pod "$pod" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ done
+ for channel in "${CREATED_CHANNELS[@]}"; do
+ kctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1
+ done
+ restore_hcc_cadence || { failed=1; echo "HCC cadence restore failed; check deployment/${HCC_DEPLOY}" >&2; }
+ for name in "${CREATED_HOSTS[@]}"; do
+ admin DELETE "${CAPI}/admin/hosts/${name}" >/dev/null 2>&1
+ case "$HTTP_CODE" in 200 | 404) ;; *) failed=1; echo "delete Host ${name} returned ${HTTP_CODE}" >&2 ;; esac
+ [ -z "$USER_ID" ] || set_agent_grant "${CAPI}/admin/users/${USER_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
+ [ -z "$TEAM_ID" ] || set_agent_grant "${CAPI}/admin/teams/${TEAM_ID}/agents" "$name" remove >/dev/null 2>&1 || failed=1
+ done
+ for name in "${CREATED_HOSTS[@]}"; do
+ kctl wait --for=delete "deployment/${name}" -n "$HOST_NS" --timeout=120s >/dev/null 2>&1
+ kctl delete pvc "${name}-workspace" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || failed=1
+ done
+ rm -rf "$SCRATCH"
+ [ "$failed" = 0 ] || { status=1; fail 'cleanup left resources behind (see stderr)'; }
+ echo "E2E_825_SCENARIOS ${SCENARIO_LOG}"
+ echo "E2E_825_EXECUTED=${EXECUTED} E2E_825_DEFINED=${#DEFINED_SCENARIOS[@]}"
+ print_results || status=1
+ if [ "$status" = 0 ] && [ "$SCENARIO_FAILURES" = 0 ] && [ "$EXECUTED" = "${#DEFINED_SCENARIOS[@]}" ]; then
+ echo 'E2E_825_RESULT=PASS'
+ else
+ echo 'E2E_825_RESULT=FAIL'
+ status=1
+ fi
+ exit "$status"
+}
+trap cleanup EXIT
+
+# ─── Run ──────────────────────────────────────────────────────────────
+header "#825 storage layout across lifecycle switches (run ${RUN}, ${E2E_KUBECONTEXT})"
+SOURCE_HOST_JSON="$(kctl get host "$SOURCE_HOST" -n "$HOST_NS" -o json)" || die "source Host ${SOURCE_HOST} not found"
+FIXTURE_IMAGE="$(kctl get deployment "$SOURCE_HOST" -n "$HOST_NS" \
+ -o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].image}')"
+[ -n "$FIXTURE_IMAGE" ] || die "could not read the ${SOURCE_HOST} image for the PVC fixture"
+admin_login || die 'admin login failed'
+admin GET "${CAPI}/admin/users?q=$(jq -rn --arg q "$DEV_EMAIL" '$q|@uri')" || true
+USER_ID="$(jq -r --arg e "$DEV_EMAIL" '.items[]? | select((.email // "") == $e) | .id' <<<"$HTTP_BODY" | head -n1)"
+[ -n "$USER_ID" ] || die "user ${DEV_EMAIL} not found (${HTTP_CODE})"
+admin GET "${CAPI}/admin/users/${USER_ID}/teams" || true
+TEAM_ID="$(jq -r '.items[0].id // empty' <<<"$HTTP_BODY")"
+user_login || die 'user login failed'
+ok "logged in as ${DEV_EMAIL}; fixture image ${FIXTURE_IMAGE}"
+
+for id in NEG_CTRL NEG_RV S1 S2 S3 RACE; do run_scenario "$id"; done
+header 'Suspended-Host scenarios'
+if set_hcc_cadence; then
+ for id in S4 S5 S6 S7; do run_scenario "$id"; done
+else
+ fail 'hcc-cadence fixture failed; S4-S7 not executed'
+fi
diff --git a/scripts/e2e/e2e-stateless-suspend-wake.sh b/scripts/e2e/e2e-stateless-suspend-wake.sh
index d6cfa2980..ab61e8d49 100755
--- a/scripts/e2e/e2e-stateless-suspend-wake.sh
+++ b/scripts/e2e/e2e-stateless-suspend-wake.sh
@@ -296,6 +296,25 @@ send_turn() {
TURN_STATUS="$(echo "$resp" | tail -n1)"; TURN_BODY="$(echo "$resp" | sed '$d')"
}
+# Conversation continuity oracle (#1102). THREAD_MESSAGES holds the stored
+# history of THREAD_ID; history_prefix_digest prints "<n> <sha256>" over the
+# first n turns and refuses an empty history, so an empty catalog never matches.
+THREAD_MESSAGES=""
+read_thread_messages() {
+ local resp code
+ mint_rpc_token || return 1
+ resp=$(curl -sS -m 30 -w '\n%{http_code}' \
+ "${RPC_BASE}/api/v1/rpc/hosts/${HOST_REF}/sessions/${HOST_REF}/${THREAD_ID}/messages?limit=100" \
+ -H "Authorization: Bearer ${RPC_TOKEN}") || { echo "messages GET failed at ${RPC_BASE}" >&2; return 1; }
+ code="$(echo "$resp" | tail -n1)"; THREAD_MESSAGES="$(echo "$resp" | sed '$d')"
+ [ "$code" = "200" ] || { echo "messages GET -> HTTP ${code}: ${THREAD_MESSAGES}" >&2; return 1; }
+}
+history_prefix_digest() {
+ local json=$1 n=$2
+ [ "$n" -gt 0 ] && [ "$(echo "$json" | jq '.turns | length')" -ge "$n" ] || return 1
+ printf '%s %s' "$n" "$(echo "$json" | jq -cS --argjson n "$n" '[.turns[:$n][] | {number, user_input, response}]' | shasum -a 256 | cut -d' ' -f1)"
+}
+
# send_turn_expect_ok: single bounded deadline; the ONLY retryable is the
# documented 503 {code:"host_waking"} while rpc-proxy's wake-and-hold settles.
send_turn_expect_ok() {
@@ -403,43 +422,49 @@ wake_handled_generation() { kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonp
wake_requested_annotation() { kctl get host "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath="{.metadata.annotations.clerum\\.io/wake-requested}" 2>/dev/null || true; }
deployment_replicas() { kctl get deployment "$HOST_REF" -n "$MCP_HOST_NS" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo ""; }
-# A stateless effective lifecycle is more than replicas=1: it changes the
-# pod template. Assert both shapes so a stale Host watch event cannot leave a
-# legacy stateful Deployment behind while status says stateless is enabled.
+# Every lifecycle uses one storage layout (#825): the workspace PVC root is
+# mounted at /workspace with no subPath, the conversation database lives there
+# (CLERUM_SESSION_STORE=sqlite, CLERUM_SESSION_DB_DIR=/workspace), and the only
+# init container is workspace-layout-merge. Both template checks require it, so
+# a lifecycle switch that changes storage fails either check.
+# shellcheck disable=SC2016 # $pod is a jq variable
+ROOT_LAYOUT_JQ='
+ def root_layout($pod):
+ ([ $pod.containers[] | select(.name == "mcp-host") | .volumeMounts[]? | select(.name == "workspace") ]
+ == [ {name: "workspace", mountPath: "/workspace"} ])
+ and ([ $pod.initContainers[]? | .name ] == ["workspace-layout-merge"])
+ and ([ $pod.containers[] | select(.name == "mcp-host") | .env[]?
+ | select(.name == "CLERUM_SESSION_STORE" or .name == "CLERUM_SESSION_DB_DIR") | {(.name): .value} ] | add
+ == {CLERUM_SESSION_STORE: "sqlite", CLERUM_SESSION_DB_DIR: "/workspace"});'
+
+# The stateless effective lifecycle changes compute only: interactive priority
+# class and CLERUM_STATELESS_LIFECYCLE=true, on the root storage layout.
deployment_has_stateless_template() {
local deployment
deployment="$(kctl get deployment "$HOST_REF" -n "$MCP_HOST_NS" -o json 2>/dev/null)" || return 2
- printf '%s' "$deployment" | jq -e '
+ printf '%s' "$deployment" | jq -e "${ROOT_LAYOUT_JQ}"'
.spec.template.spec as $pod
| [
($pod.priorityClassName == "clerum-interactive-host"),
- ([ $pod.initContainers[]? | select(.name == "workspace-layout-init") ] | length == 1),
([ $pod.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE" and .value == "true") ] | length == 1),
- ([ $pod.containers[] | select(.name == "mcp-host") | .volumeMounts[]? | select(.name == "workspace" and .mountPath == "/workspace" and .subPath == "workspace") ] | length == 1),
- ([ $pod.containers[] | select(.name == "mcp-host") | .volumeMounts[]? | select(.name == "workspace" and .mountPath == "/var/lib/clerum/state" and .subPath == "state") ] | length == 1)
+ root_layout($pod)
] | all
' >/dev/null
}
-# A STATEFUL effective template is discriminated ONLY by the effective-mode
-# markers: no interactive priority class and no CLERUM_STATELESS_LIFECYCLE env.
-# The requested stateless STORAGE layout (workspace-layout-init init container +
-# the /var/lib/clerum/state and /workspace subPath mounts) is durable-
-# compatibility state that a HARD REJECTION deliberately RETAINS — exactly like
-# a spec.desktop/SFS rejection (buildDeployment gates the storage layout on the
-# spec REQUEST `usesStatelessStorageLayout`, never on effective mode, so a
-# rejection never remounts the PVC root and creates a second state.db). So the
-# storage layout must NOT be part of the stateful discriminator: this helper
-# must pass for a genuinely-stateful host (no layout) AND a channel/desktop hard-
-# rejected stateless-requesting host (layout retained). See Addendum 6.
+# The stateful effective lifecycle (requested, or a hard rejection of a
+# stateless request by a CommunicationChannel or spec.desktop) has no
+# interactive priority class and no CLERUM_STATELESS_LIFECYCLE env, on the same
+# root storage layout.
deployment_has_stateful_template() {
local deployment
deployment="$(kctl get deployment "$HOST_REF" -n "$MCP_HOST_NS" -o json 2>/dev/null)" || return 2
- printf '%s' "$deployment" | jq -e '
+ printf '%s' "$deployment" | jq -e "${ROOT_LAYOUT_JQ}"'
.spec.template.spec as $pod
| [
($pod.priorityClassName != "clerum-interactive-host"),
- ([ $pod.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") ] | length == 0)
+ ([ $pod.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE") ] | length == 0),
+ root_layout($pod)
] | all
' >/dev/null
}
@@ -834,9 +859,9 @@ wait_for_ready_pod "$POD_READY_TIMEOUT" >/dev/null || { fail "no pod after wake
# CommunicationChannel, then flipped to spec.lifecycle.stateless:true, must
# hard-reject from the FIRST stateless-requesting reconcile -- the requested
# stateless lifecycle must NEVER be transiently enabled. Direct CRD writes are
-# used deliberately: control-api blocks this flip at the admin API with a 409
-# (stateless_host_communication_channel_conflict), so the state-derived HCC
-# backstop is exactly the racing/direct-write path under test. Uses the shared
+# used deliberately: control-api does not check CommunicationChannels when the
+# lifecycle changes, so HCC's state-derived rejection is the only guard and is
+# the path under test. Uses the shared
# $HOST_REF sequentially (the suite's established pattern; test 11 also flips
# spec.lifecycle.stateless on it) and restores it via the cleanup trap.
header "Test 2d-pre -- reverse arrival: channel-bound host flipped to stateless hard-rejects"
@@ -1246,6 +1271,58 @@ fi
# Test 11 -- Kill-switch with held request
# ====================================================================== #
header "Test 11 -- kill-switch with held request completes or retryable"
+# Continuity across the flip (#1102): a Ready Pod is not enough. Seed a
+# conversation turn and a workspace file on the stateless Pod, then require
+# both on the stateful Pod and again after the switch back.
+# wait_for_switched_pod EXPECTED: a Ready, non-terminating Pod whose own spec
+# has the EXPECTED lifecycle markers on the root storage layout; prints its name.
+wait_for_switched_pod() {
+ local expected=$1 elapsed=0 name
+ while [ "$elapsed" -lt "$POD_READY_TIMEOUT" ]; do
+ name="$(kctl get pods -n "$MCP_HOST_NS" -l "app=${HOST_REF}" -o json 2>/dev/null | jq -r --arg e "$expected" "${ROOT_LAYOUT_JQ}"'
+ [ .items[]
+ | select(.metadata.deletionTimestamp == null)
+ | select(any(.status.conditions[]?; .type == "Ready" and .status == "True"))
+ | select(root_layout(.spec))
+ | select(([ .spec.containers[] | select(.name == "mcp-host") | .env[]? | select(.name == "CLERUM_STATELESS_LIFECYCLE" and .value == "true") ] | length == 1)
+ == ($e == "stateless"))
+ | .metadata.name ][0] // empty')" || name=""
+ if [ -n "$name" ]; then printf '%s\n' "$name"; return 0; fi
+ sleep "$POLL_INTERVAL"; elapsed=$((elapsed + POLL_INTERVAL))
+ done
+ return 1
+}
+SEED11_FILE="e2e-1102-${RUN_ID}.txt"
+SEED11_CONTENT="seed-1102-${RUN_ID}"
+assert_test11_continuity() {
+ local label=$1 pod=$2 digest got
+ read_thread_messages || { fail "11 ${label}: cannot read the seeded conversation"; return 1; }
+ digest="$(history_prefix_digest "$THREAD_MESSAGES" "$SEED11_TURNS")" || {
+ fail "11 ${label}: seeded conversation has fewer than ${SEED11_TURNS} turn(s): $(echo "$THREAD_MESSAGES" | jq -c '.turns | length')"; return 1; }
+ [ "$digest" = "$SEED11_DIGEST" ] || { fail "11 ${label}: seeded history changed (${SEED11_DIGEST} -> ${digest})"; return 1; }
+ got="$(kctl exec "$pod" -n "$MCP_HOST_NS" -c mcp-host -- cat "/workspace/${SEED11_FILE}" 2>&1)" || got="<unreadable: ${got}>"
+ [ "$got" = "$SEED11_CONTENT" ] || { fail "11 ${label}: workspace file reads '${got}', expected '${SEED11_CONTENT}'"; return 1; }
+ ok "11 ${label}: seeded conversation (${SEED11_TURNS} turn(s)) and workspace file unchanged on ${pod}"
+}
+# Negative control: the history oracle refuses an empty catalog.
+if history_prefix_digest '{"totalTurns":0,"turns":[]}' 1 >/dev/null; then
+ fail "11 negative control: history oracle accepted an empty catalog"; print_results; exit 1
+fi
+ok "11 negative control: history oracle rejects an empty catalog"
+send_turn_expect_ok "Reply with exactly: seed-1102-${RUN_ID}" || { fail "11: seed turn failed"; print_results; exit 1; }
+S11_POD="$(wait_for_switched_pod stateless)" || { fail "11: no Ready stateless Pod on the root layout before the flip"; pod_diagnostics; print_results; exit 1; }
+# shellcheck disable=SC2016 # expanded by the Pod shell
+kctl exec "$S11_POD" -n "$MCP_HOST_NS" -c mcp-host -- sh -c 'printf "%s\n" "$1" > "/workspace/$2"' sh "$SEED11_CONTENT" "$SEED11_FILE" \
+ || { fail "11: could not seed workspace file on ${S11_POD}"; print_results; exit 1; }
+s11_wait=0
+until read_thread_messages && echo "$THREAD_MESSAGES" | jq -e --arg m "seed-1102-${RUN_ID}" '(.turns[-1].user_input // "") | contains($m)' >/dev/null; do
+ [ "$s11_wait" -lt 60 ] || { fail "11: seed turn not persisted within 60s"; print_results; exit 1; }
+ sleep 3; s11_wait=$((s11_wait + 3))
+done
+SEED11_TURNS="$(echo "$THREAD_MESSAGES" | jq '.turns | length')"
+SEED11_DIGEST="$(history_prefix_digest "$THREAD_MESSAGES" "$SEED11_TURNS")" \
+ || { fail "11: seeded conversation is empty after a 200 turn"; print_results; exit 1; }
+assert_test11_continuity "witness on the stateless Pod" "$S11_POD" || { print_results; exit 1; }
force_idle_and_suspend || { print_results; exit 1; }
k_status_file="$(mktemp)"
MARKER11="killswitch-held-${RUN_ID}"
@@ -1282,12 +1359,25 @@ else
pod_diagnostics; print_results; exit 1
fi
wait_for_deployment_template stateful "$POD_READY_TIMEOUT" "11: stateless disabled" || { print_results; exit 1; }
-kctl patch host "$HOST_REF" -n "$MCP_HOST_NS" --type=merge -p '{"spec":{"lifecycle":{"stateless":true}}}' >/dev/null 2>&1 || true
+S11_POD="$(wait_for_switched_pod stateful)" || { fail "11: no Ready stateful Pod on the root layout after the flip"; pod_diagnostics; print_results; exit 1; }
+assert_test11_continuity "stateless -> stateful" "$S11_POD" || { print_results; exit 1; }
+kctl patch host "$HOST_REF" -n "$MCP_HOST_NS" --type=merge -p '{"spec":{"lifecycle":{"stateless":true}}}' >/dev/null \
+ || { fail "test 11 could not flip stateless:true"; print_results; exit 1; }
STATELESS_DISABLED=0
wait_for_deployment_template stateless "$POD_READY_TIMEOUT" "11: stateless re-enabled" \
|| { print_results; exit 1; }
-wait_for_ready_pod "$POD_READY_TIMEOUT" >/dev/null || { fail "host did not recover after re-enabling stateless"; print_results; exit 1; }
-ok "running stateful Host converged back to stateless and is Ready"
+S11_POD="$(wait_for_switched_pod stateless)" || { fail "host did not recover after re-enabling stateless"; pod_diagnostics; print_results; exit 1; }
+assert_test11_continuity "stateful -> stateless" "$S11_POD" || { print_results; exit 1; }
+send_turn_expect_ok "Reply with exactly: resume-1102-${RUN_ID}" || { fail "11: resume turn failed after the round trip"; print_results; exit 1; }
+s11_wait=0
+until read_thread_messages && echo "$THREAD_MESSAGES" | jq -e --argjson n "$SEED11_TURNS" --arg m "resume-1102-${RUN_ID}" \
+ '(.turns | length) > $n and ((.turns[-1].user_input // "") | contains($m))' >/dev/null; do
+ [ "$s11_wait" -lt 60 ] || { fail "11: resume turn not appended to the seeded conversation within 60s"; print_results; exit 1; }
+ sleep 3; s11_wait=$((s11_wait + 3))
+done
+kctl exec "$S11_POD" -n "$MCP_HOST_NS" -c mcp-host -- rm -f "/workspace/${SEED11_FILE}" \
+ || { fail "11: could not remove the seeded workspace file"; print_results; exit 1; }
+ok "running stateful Host converged back to stateless with its conversation and files, and resumed"
# ====================================================================== #
# Test 12 -- Re-resolution on wake