Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): harden the #825 storage layout and suspend-wake harnesses
Alfredo Lopez38e243c41d96
diff --git a/scripts/e2e/e2e-hcc-storage-layout-switch.sh b/scripts/e2e/e2e-hcc-storage-layout-switch.shindex c9d131e24..8cd2f938a 100755--- a/scripts/e2e/e2e-hcc-storage-layout-switch.sh+++ b/scripts/e2e/e2e-hcc-storage-layout-switch.sh@@ -84,9 +84,11 @@ BASE_INIT_FIXTURE="${REPO_ROOT}/host-context-controller/test/__fixtures__/base-wDEFINED_SCENARIOS=(NEG_CTRL NEG_RV S1 S2 S3 RACE S4 S5 S6 S7)EXECUTED=0 SCENARIO_FAILURES=0 SCENARIO_LOG=''-CREATED_HOSTS=() CREATED_CHANNELS=() FIXTURE_PODS=()+CREATED_HOSTS=() CREATED_CHANNELS=()HCC_ENV_SAVED='' HCC_ENV_CHANGED=0SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/e2e-825.XXXXXX")"+FIXTURE_POD_REGISTRY="${SCRATCH}/fixture-pods"+: >"$FIXTURE_POD_REGISTRY"ADMIN_COOKIE='' SESSION_TOKEN='' RPC_TOKEN='' USER_ID='' TEAM_ID=''HTTP_CODE='' HTTP_BODY='' HTTP_HEADERS=()@@ -410,8 +412,12 @@ pvc_fixture() {}],volumes:[{name:"pvc", persistentVolumeClaim:{claimName:$claim, readOnly:$ro}}]}}' "$@")"+ # Callers usually run this function inside $(...), so an array append would+ # be lost with the subshell; the registry file is what cleanup() reads.+ # Registered before the create so a create that times out after the API+ # accepted it is still cleaned up.+ printf '%s\n' "$pod" >>"$FIXTURE_POD_REGISTRY"kctl create -f - <<<"$body" >/dev/null || { fail "${label}: fixture Pod create failed" >&2; return 1; }- FIXTURE_PODS+=("$pod")while [ "$elapsed" -lt 180 ]; dophase="$(kctl get pod "$pod" -n "$HOST_NS" -o jsonpath='{.status.phase}' 2>/dev/null || true)"case "$phase" in Succeeded | Failed) break ;; esac@@ -419,7 +425,12 @@ pvc_fixture() {elapsed=$((elapsed + 2))donekctl logs "$pod" -n "$HOST_NS" >"${SCRATCH}/${pod}.log" 2>&1 || true- kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1 || true+ # A Pod left behind keeps the RWO claim attached to its node and blocks the+ # next fixture or Host Pod on another node, so a failed delete fails the step.+ if ! kctl delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then+ fail "${label}: fixture Pod ${pod} delete failed" >&2+ return 1+ fiif [ "$phase" != Succeeded ]; thenfail "${label}: fixture Pod ended ${phase:-<timeout>}: $(tail -n 5 "${SCRATCH}/${pod}.log")" >&2return 1@@ -730,7 +741,11 @@ scenario_S7() {wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1assert_pod "$name" true /workspace || return 1listing="$(pvc_fixture "$name" ro list-after "$LIST_SCRIPT")" || return 1- [ "$(grep -v ' ./state.db' <<<"$before")" = "$(grep -v ' ./state.db' <<<"$listing")" ] ||+ # Only the live writer's journals may differ: the suspended Host closed its+ # database (no -wal/-shm), the woken one holds it open. state.db itself and+ # any state.db*.bak must match, so a lost or extra backup fails here.+ grep -qx 'f ./state.db' <<<"$listing" || { fail "S7: no state.db at the PVC root after the merge: ${listing}"; return 1; }+ [ "$(grep -vxE 'f \./state\.db-(wal|shm)' <<<"$before")" = "$(grep -vxE 'f \./state\.db-(wal|shm)' <<<"$listing")" ] ||{ fail 'S7: root entries differ from the tree before the re-split'; diff <(echo "$before") <(echo "$listing") >&2 || true; return 1; }ok 'S7: re-split tree merged back to the same root entries'assert_history "$name" || return 1@@ -740,7 +755,7 @@ scenario_S7() {# RACE: concurrent model selections on new chats create one session each and# never restart the Host (A7.13).race_host() { # NAME STATELESS- local name=$1 stateless=$2 model uid restarts round i chat codes pids+ local name=$1 stateless=$2 model uid restarts round i chat codes pids podscreate_host "$name" "$stateless" || return 1wait_new_ready_pod "$name" '' "$POD_READY_TIMEOUT" || return 1assert_pod "$name" "$stateless" /workspace || return 1@@ -765,10 +780,14 @@ race_host() { # NAME STATELESSrpc GET "${RPC_BASE}/api/v1/rpc/hosts/${name}/sessions?agent=${name}&limit=100" || truesession_listed "$HTTP_BODY" "$name" "$chat" || { fail "${name}: round ${round} chat not listed exactly once"; return 1; }done- POD_JSON="$(host_pods "$name" | jq -c '.items[0]')"- [ "$(jq -r '.metadata.uid' <<<"$POD_JSON")" = "$uid" ] &&+ # The original Pod must be the only Pod, not terminating: a replacement+ # leaves the old Pod Terminating next to a new one, and items[0] could be+ # either of them.+ pods="$(host_pods "$name")" || { fail "${name}: could not list Pods after the race"; return 1; }+ POD_JSON="$(jq -c --arg uid "$uid" '[.items[] | select(.metadata.uid == $uid and .metadata.deletionTimestamp == null)] | first // empty' <<<"$pods")"+ [ "$(jq '.items | length' <<<"$pods")" = 1 ] && [ -n "$POD_JSON" ] &&[ "$(jq -r '[.status.containerStatuses[] | select(.name == "mcp-host") | .restartCount] | first' <<<"$POD_JSON")" = "$restarts" ] ||- { fail "${name}: the Pod restarted or was replaced during the race"; return 1; }+ { fail "${name}: the Pod restarted or was replaced during the race: $(jq -c '[.items[] | {uid: .metadata.uid, deleting: (.metadata.deletionTimestamp != null)}]' <<<"$pods")"; return 1; }ok "${name}: 5 rounds x 4 concurrent selections, one session per chat, Pod ${uid:0:8} restartCount ${restarts}"}scenario_RACE() {@@ -795,9 +814,9 @@ cleanup() {local status=$? failed=0 name pod channel# bash 3.2 treats "${empty[@]}" as unbound under set -u.set +eu- for pod in "${FIXTURE_PODS[@]}"; do+ while IFS= read -r pod; dokctl delete pod "$pod" -n "$HOST_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1- done+ done <"$FIXTURE_POD_REGISTRY"for channel in "${CREATED_CHANNELS[@]}"; dokctl delete communicationchannel "$channel" -n "$CHANNEL_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || failed=1done@@ -829,6 +848,16 @@ trap cleanup EXIT# ─── Run ──────────────────────────────────────────────────────────────header "#825 storage layout across lifecycle switches (run ${RUN}, ${E2E_KUBECONTEXT})"+# S5-S7 need HCC to render workspace-layout-merge; its flag defaults to true+# when unset or empty (A7.24). A literal value is the only one checkable here.+MERGE_FLAG="$(kctl get "deployment/${HCC_DEPLOY}" -n "$HCC_NS" -o json | jq -r '+ [.spec.template.spec.containers[].env[]? | select(.name == "CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE")]+ | if length == 0 then "<unset>" elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end')" ||+ die "could not read deployment/${HCC_DEPLOY}"+case "$MERGE_FLAG" in+ '<unset>' | '' | true) ok "HCC CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE=${MERGE_FLAG:-<empty>} (merge init rendered)" ;;+ *) die "HCC CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE is '${MERGE_FLAG}'; this E2E needs it unset, empty or 'true'" ;;+esacSOURCE_HOST_JSON="$(kctl get host "$SOURCE_HOST" -n "$HOST_NS" -o json)" || die "source Host ${SOURCE_HOST} not found"FIXTURE_IMAGE="$(kctl get deployment "$SOURCE_HOST" -n "$HOST_NS" \-o jsonpath='{.spec.template.spec.containers[?(@.name=="mcp-host")].image}')"diff --git a/scripts/e2e/e2e-stateless-suspend-wake.sh b/scripts/e2e/e2e-stateless-suspend-wake.shindex ab61e8d49..205a3535a 100755--- a/scripts/e2e/e2e-stateless-suspend-wake.sh+++ b/scripts/e2e/e2e-stateless-suspend-wake.sh@@ -1314,10 +1314,11 @@ S11_POD="$(wait_for_switched_pod stateless)" || { fail "11: no Ready stateless P# shellcheck disable=SC2016 # expanded by the Pod shellkctl exec "$S11_POD" -n "$MCP_HOST_NS" -c mcp-host -- sh -c 'printf "%s\n" "$1" > "/workspace/$2"' sh "$SEED11_CONTENT" "$SEED11_FILE" \|| { fail "11: could not seed workspace file on ${S11_POD}"; print_results; exit 1; }-s11_wait=0+# Wall-clock deadline: each read_thread_messages can itself take up to 30s.+s11_deadline=$((SECONDS + 60))until read_thread_messages && echo "$THREAD_MESSAGES" | jq -e --arg m "seed-1102-${RUN_ID}" '(.turns[-1].user_input // "") | contains($m)' >/dev/null; do- [ "$s11_wait" -lt 60 ] || { fail "11: seed turn not persisted within 60s"; print_results; exit 1; }- sleep 3; s11_wait=$((s11_wait + 3))+ [ "$SECONDS" -lt "$s11_deadline" ] || { fail "11: seed turn not persisted within 60s"; print_results; exit 1; }+ sleep 3doneSEED11_TURNS="$(echo "$THREAD_MESSAGES" | jq '.turns | length')"SEED11_DIGEST="$(history_prefix_digest "$THREAD_MESSAGES" "$SEED11_TURNS")" \@@ -1369,11 +1370,11 @@ wait_for_deployment_template stateless "$POD_READY_TIMEOUT" "11: stateless re-enS11_POD="$(wait_for_switched_pod stateless)" || { fail "host did not recover after re-enabling stateless"; pod_diagnostics; print_results; exit 1; }assert_test11_continuity "stateful -> stateless" "$S11_POD" || { print_results; exit 1; }send_turn_expect_ok "Reply with exactly: resume-1102-${RUN_ID}" || { fail "11: resume turn failed after the round trip"; print_results; exit 1; }-s11_wait=0+s11_deadline=$((SECONDS + 60))until read_thread_messages && echo "$THREAD_MESSAGES" | jq -e --argjson n "$SEED11_TURNS" --arg m "resume-1102-${RUN_ID}" \'(.turns | length) > $n and ((.turns[-1].user_input // "") | contains($m))' >/dev/null; do- [ "$s11_wait" -lt 60 ] || { fail "11: resume turn not appended to the seeded conversation within 60s"; print_results; exit 1; }- sleep 3; s11_wait=$((s11_wait + 3))+ [ "$SECONDS" -lt "$s11_deadline" ] || { fail "11: resume turn not appended to the seeded conversation within 60s"; print_results; exit 1; }+ sleep 3donekctl exec "$S11_POD" -n "$MCP_HOST_NS" -c mcp-host -- rm -f "/workspace/${SEED11_FILE}" \|| { fail "11: could not remove the seeded workspace file"; print_results; exit 1; }