Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(deploy): verify the workspace layout merge on any cluster
Alfredo Lopeza4c1b1c3b73b
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex a0acffe54..d7e8da5e0 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -743,12 +743,13 @@ jobs:- name: Test minikube Makefile targetsrun: bash scripts/tests/test-minikube-makefile.sh- - name: Test the #825 gate context guards, report verdict and E2E helpers+ - name: Test the #825 gate context guards, report verdict, E2E helpers and layout merge checkrun: >-node scripts/tests/run-node-test-files.mjsscripts/e2e/playwright-report-gate.test.mjsscripts/e2e/e2e-hcc-storage-layout-switch.test.mjsscripts/e2e/e2e-stateless-suspend-wake.test.mjs+ deploy/scripts/host-layout-merge.test.mjs# make test-service-matrix fails when Makefile TEST_SERVICES and the# `test` job's service matrix above drift apart.diff --git a/deploy/scripts/host-layout-merge.sh b/deploy/scripts/host-layout-merge.shnew file mode 100755index 000000000..2354a5a23--- /dev/null+++ b/deploy/scripts/host-layout-merge.sh@@ -0,0 +1,525 @@+#!/usr/bin/env bash+# Issue #825: read-only inventory and turn-off gate for the HCC workspace+# layout merge on any cluster (docs/deploy/host-pvc-layout-upgrade.md).+#+# Every Host PVC is classified with the merge init's own rules: the check+# Pods run deploy/scripts/workspace-layout-merge.sh (the script HCC renders,+# kept equal to it by a host-context-controller test) with the moves removed+# and the writability checks read from mode bits, because the PVC is mounted+# read-only. The script never opens a database and never execs into a Host.+set -euo pipefail++usage() {+ cat <<'EOF'+usage:+ host-layout-merge.sh inventory --context <kube-context> [options]+ host-layout-merge.sh verify --context <kube-context> [options]+ host-layout-merge.sh check-script [--root <path>]++inventory before an upgrade: what the merge init will do to every Host PVC.+ Exit 1 when a tree will be refused (UPGRADE_BLOCKERS > 0), the+ flag is "false" while a tree is split, the flag is invalid, or a+ PVC could not be read.+verify whether CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE can be set to+ "false": exit 0 only when every Host Deployment renders the merge+ init and no Host PVC still carries the split layout.+check-script print the check the PVC check Pods run.++options:+ --hcc-namespace <ns> HCC namespace (default control-plane)+ --hcc-deployment <name> HCC Deployment (default host-context-controller)+ --no-pvc-pods start no check Pod; PVC results are UNVERIFIED+ --check-timeout <sec> wait per check Pod (default 120)+ --root <path> check-script only: PVC root (default /mnt/workspace-root)++Reads only, except one short-lived Pod per inspected PVC: claim mounted+read-only, uid/gid 1001, no service-account token, deleted after its log is+read. Every kubectl call names --context; the current-context is never used.+Exit: 0 pass, 1 an observed blocker, 2 usage, read failure, empty scope or+check Pod cleanup failure.+EOF+}++KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"+MERGE_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/workspace-layout-merge.sh"+PVC_ROOT=/mnt/workspace-root++die() {+ echo "host-layout-merge: $*" >&2+ exit 2+}+usage_error() {+ echo "host-layout-merge: $*" >&2+ usage >&2+ exit 2+}++# ─── check script ─────────────────────────────────────────────────────+# Prints the merge script with:+# - the root replaced (check-script --root, used by the tests);+# - each refusal printed on stdout as WILL_REFUSE_<section> <line>, where+# the section is the "# F<n>" block it ran in (F1 before the first);+# - each "no signature" exit printed as NO_SPLIT_SIGNATURE;+# - `[ -w ]` replaced by a mode-bit test, since the check mount is read-only;+# - everything from the move function on replaced by WILL_MERGE.+# Each rewrite must match the expected number of lines, so a changed merge+# script stops here instead of producing a check that no longer matches it.+# The patterns and replacements below are shell text for the check Pod, so+# they are single-quoted on purpose.+# shellcheck disable=SC2016+print_check_script() {+ local root=$1 line roots=0 sets=0 echoes=0 exits=0 sections=0 noops=0 writes=0 cut=0+ case "$root" in *"'"*) usage_error "--root must not contain a single quote" ;; esac+ [ -f "$MERGE_SCRIPT" ] || die "missing $MERGE_SCRIPT"+ while IFS= read -r line || [ -n "$line" ]; do+ case "$line" in+ "root='/mnt/workspace-root'")+ printf "root='%s'\n" "$root"+ roots=$((roots + 1))+ ;;+ 'set -eu')+ printf '%s\n' "$line" 'phase=F1'+ cat <<'EOF'+# Mode-bit write test: the check Pod mounts the PVC read-only, so the+# shell's -w test would refuse every tree. Same owner/group/other rule as+# access(2).+writable() (+ set -f+ [ "$(id -u)" -ne 0 ] || exit 0+ listing="$(ls -ldn "$1")" || exit 1+ set -- $listing+ if [ "$3" = "$(id -u)" ]; then+ case "$1" in ??w*) exit 0 ;; *) exit 1 ;; esac+ fi+ for gid in $(id -G); do+ if [ "$4" = "$gid" ]; then+ case "$1" in ?????w*) exit 0 ;; *) exit 1 ;; esac+ fi+ done+ case "$1" in ????????w*) exit 0 ;; *) exit 1 ;; esac+)+EOF+ sets=$((sets + 1))+ ;;+ ' echo "workspace layout merge refused: $1" >&2')+ printf '%s\n' ' echo "WILL_REFUSE_$phase workspace layout merge refused: $1"'+ echoes=$((echoes + 1))+ ;;+ ' exit 1')+ printf '%s\n' ' exit 0'+ exits=$((exits + 1))+ ;;+ '# F1' | '# F2' | '# F3' | '# F4')+ printf 'phase=%s\n' "${line#'# '}"+ sections=$((sections + 1))+ ;;+ *'|| exit 0')+ printf '%s\n' "${line%'|| exit 0'}|| { echo NO_SPLIT_SIGNATURE; exit 0; }"+ noops=$((noops + 1))+ ;;+ ' [ -w "$dir" ] || fail_merge "not writable by uid $(id -u): $dir"')+ printf '%s\n' ' writable "$dir" || fail_merge "not writable by uid $(id -u): $dir"'+ writes=$((writes + 1))+ ;;+ ' [ -w "$entry" ] || fail_merge "not writable by uid $(id -u): $entry"')+ printf '%s\n' ' writable "$entry" || fail_merge "not writable by uid $(id -u): $entry"'+ writes=$((writes + 1))+ ;;+ 'move_to_root() {')+ printf '%s\n' 'echo WILL_MERGE'+ cut=1+ break+ ;;+ *)+ printf '%s\n' "$line"+ ;;+ esac+ done <"$MERGE_SCRIPT"+ local shape="root=$roots set=$sets echo=$echoes exit=$exits sections=$sections noops=$noops writes=$writes cut=$cut"+ [ "$shape" = "root=1 set=1 echo=1 exit=1 sections=4 noops=3 writes=2 cut=1" ] ||+ die "$MERGE_SCRIPT has an unexpected shape ($shape); update print_check_script with it"+}++# ─── arguments ────────────────────────────────────────────────────────+COMMAND="${1:-}"+[ "$#" -eq 0 ] || shift+CONTEXT=""+HCC_NAMESPACE=control-plane+HCC_DEPLOYMENT='host-context-controller'+NO_PVC_PODS=0+CHECK_TIMEOUT=120+CHECK_ROOT="$PVC_ROOT"+need_value() { [ "$2" -ge 2 ] || usage_error "$1 needs a value"; }+while [ "$#" -gt 0 ]; do+ case "$1" in+ --context) need_value "$1" "$#"; CONTEXT="$2"; shift 2 ;;+ --context=*) CONTEXT="${1#--context=}"; shift ;;+ --hcc-namespace) need_value "$1" "$#"; HCC_NAMESPACE="$2"; shift 2 ;;+ --hcc-deployment) need_value "$1" "$#"; HCC_DEPLOYMENT="$2"; shift 2 ;;+ --no-pvc-pods) NO_PVC_PODS=1; shift ;;+ --check-timeout) need_value "$1" "$#"; CHECK_TIMEOUT="$2"; shift 2 ;;+ --root) need_value "$1" "$#"; CHECK_ROOT="$2"; shift 2 ;;+ -h | --help) usage; exit 0 ;;+ *) usage_error "unknown argument: $1" ;;+ esac+done++case "$COMMAND" in+ check-script)+ print_check_script "$CHECK_ROOT"+ exit 0+ ;;+ inventory | verify) ;;+ *) usage_error "expected inventory, verify or check-script" ;;+esac+[ -n "$CONTEXT" ] || usage_error "--context is required; the current-context is never used"+[ "$CHECK_ROOT" = "$PVC_ROOT" ] || usage_error "--root applies to check-script only"+case "$CHECK_TIMEOUT" in '' | *[!0-9]*) usage_error "--check-timeout must be a whole number of seconds" ;; esac+command -v jq >/dev/null 2>&1 || die "jq is required"+CHECK_SCRIPT="$(print_check_script "$PVC_ROOT")"++kc() { "$KUBECTL_BIN" --context="$CONTEXT" "$@"; }++WORK="$(mktemp -d "${TMPDIR:-/tmp}/host-layout-merge.XXXXXX")"+CHECK_PODS="$WORK/check-pods"+: >"$CHECK_PODS"+HOST_NS=""++# A check Pod left behind keeps an RWO claim attached to its node, which can+# stop the Host from starting elsewhere, so a failed delete is exit 2.+# shellcheck disable=SC2329 # invoked by the EXIT trap+cleanup() {+ local status=$? pod+ while IFS= read -r pod; do+ [ -n "$pod" ] || continue+ if ! kc delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then+ echo "host-layout-merge: check Pod $HOST_NS/$pod was not deleted; delete it by hand" >&2+ status=2+ fi+ done <"$CHECK_PODS"+ rm -rf "$WORK"+ exit "$status"+}+trap cleanup EXIT+trap 'exit 2' INT TERM++# read_json <file> <kubectl args...>: a JSON object or exit 2 with stderr.+read_json() {+ local out=$1+ shift+ if ! kc "$@" >"$out" 2>"$out.err"; then+ die "kubectl $* failed: $(cat "$out.err")"+ fi+ jq -e 'type == "object"' "$out" >/dev/null 2>&1 || die "kubectl $* did not return a JSON object"+}++# ─── HCC ──────────────────────────────────────────────────────────────+read_json "$WORK/hcc.json" get deployment "$HCC_DEPLOYMENT" -n "$HCC_NAMESPACE" -o json+jq -e '[.spec.template.spec.containers[]? | select(.name == "host-context-controller")] | length == 1' \+ "$WORK/hcc.json" >/dev/null || die "$HCC_NAMESPACE/$HCC_DEPLOYMENT has no container named host-context-controller"+jq -e '(.spec.replicas // 1) as $r | $r > 0 and (.status.readyReplicas // 0) == $r and (.status.updatedReplicas // 0) == $r' \+ "$WORK/hcc.json" >/dev/null || die "HCC_NOT_READY: $HCC_NAMESPACE/$HCC_DEPLOYMENT is not fully rolled out and Ready"++# hcc_env <name>: <unset>, <valueFrom> or the literal value.+hcc_env() {+ jq -r --arg n "$1" '[.spec.template.spec.containers[] | select(.name == "host-context-controller") | .env[]? | select(.name == $n)]+ | if length == 0 then "<unset>" elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end' "$WORK/hcc.json"+}++FLAG_RAW="$(hcc_env CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE)"+case "$FLAG_RAW" in+ '<unset>' | '' | true) FLAG=on ;;+ false) FLAG=off ;;+ '<valueFrom>') FLAG=unresolved ;;+ *) FLAG=invalid ;;+esac+HOST_NS="$(hcc_env CONTEXT_MAPPER_HOST_NAMESPACE)"+case "$HOST_NS" in+ '<unset>') HOST_NS=mcp-host ;;+ '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_NAMESPACE ($HOST_NS) from the HCC Deployment" ;;+esac+echo "HCC_IMAGE=$(jq -r '.spec.template.spec.containers[] | select(.name == "host-context-controller") | .image // "<none>"' "$WORK/hcc.json")"+echo "HCC_FLAG=$FLAG_RAW"+echo "HOST_NAMESPACE=$HOST_NS"++# ─── Hosts, Deployments, Pods, PVCs ───────────────────────────────────+read_json "$WORK/hosts.json" get hosts.clerum.io -n "$HOST_NS" -o json+read_json "$WORK/deployments.json" get deployments -n "$HOST_NS" -l clerum.io/host -o json+read_json "$WORK/pods.json" get pods -n "$HOST_NS" -l clerum.io/host -o json+read_json "$WORK/pvcs.json" get pvc -n "$HOST_NS" -l clerum.io/managed-by=host-context-controller -o json++# One tab-separated row per Host (H) and per Host PVC (P), "-" for empty.+# A Host Deployment and its Pods carry app == clerum.io/host == <host>; the+# per-Host channel-reader carries app=channel-reader and is excluded.+jq -r -n \+ --slurpfile hosts "$WORK/hosts.json" \+ --slurpfile deps "$WORK/deployments.json" \+ --slurpfile pods "$WORK/pods.json" \+ --slurpfile pvcs "$WORK/pvcs.json" '+ def owned($h): select(.metadata.labels.app == $h and .metadata.labels["clerum.io/host"] == $h);+ def kind: if (.spec.desktop.browser // false) or (.spec.desktop.x11 // false) then "DESKTOP"+ elif (.spec.lifecycle.stateless // false) then "STATELESS" else "STATEFUL" end;+ def template: [.spec.template.spec.initContainers[]?.name] as $i+ | if any($i[]; . == "workspace-layout-init") then "LEGACY_INIT"+ elif any($i[]; . == "workspace-layout-merge") then "MERGE" else "NO_INIT" end;+ def subpath($claim): .spec.template.spec as $s+ | [$s.volumes[]? | select(.persistentVolumeClaim.claimName == $claim) | .name] as $v+ | if any(($s.containers + ($s.initContainers // []))[] | .volumeMounts[]?;+ (.name as $n | $v | index($n)) != null and (.subPath // "") != "") then "yes" else "no" end;+ def merge_status: [.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge")][0] as $s+ | if $s == null then "UNKNOWN"+ elif $s.state.terminated.exitCode == 0 then "POD_OK"+ elif $s.state.terminated.exitCode == 1 then "MERGE_REFUSED:current"+ elif $s.lastState.terminated.exitCode == 1 then "MERGE_REFUSED:previous"+ elif $s.state.terminated != null then "UNKNOWN"+ elif $s.state.running != null or $s.state.waiting != null then "MERGE_RUNNING"+ else "UNKNOWN" end;+ def worst: (map({"MERGE_REFUSED": 0, "UNKNOWN": 1, "MERGE_RUNNING": 2, "POD_OK": 3}[split(":")[0]]) | min) as $m+ | map(select({"MERGE_REFUSED": 0, "UNKNOWN": 1, "MERGE_RUNNING": 2, "POD_OK": 3}[split(":")[0]] == $m))[0];+ def dash: if . == null or . == "" then "-" else . end;+ def host_facts($h):+ [$deps[0].items[] | owned($h)] as $d+ | if ($d | length) > 1 then error("more than one Deployment for Host \($h)") else . end+ | [$pods[0].items[] | owned($h) | select(.metadata.deletionTimestamp == null)] as $p+ | ($d[0] // null) as $dep+ | ($dep | if . == null then null else [.spec.template.spec.containers[]? | select(.name == "mcp-host")][0] end) as $main+ | if $dep != null and $main == null then error("Deployment \($h) has no mcp-host container") else . end+ | {+ state: (if $dep == null then "NO_DEPLOYMENT" elif ($dep.spec.replicas // 1) == 0 then "SUSPENDED" else "ACTIVE" end),+ template: (if $dep == null then "-" else ($dep | template) end),+ subpath: (if $dep == null then "-" else ($dep | subpath("\($h)-workspace")) end),+ pods: ($p | length),+ nodes: ([$p[].spec.nodeName | select(. != null)] | unique),+ merge: (if ($p | length) == 0 then "-" else ($p | map({s: merge_status, pod: .metadata.name}) | (map(.s) | worst) as $w | map(select(.s == $w))[0] | "\(.s):\(.pod)") end),+ image: ($main.image // null),+ policy: ($main.imagePullPolicy // null),+ secrets: ($dep.spec.template.spec.imagePullSecrets // [] | map(.name) | join(","))+ };+ ($hosts[0].items | map({key: .metadata.name, value: kind}) | from_entries) as $kinds+ | ($hosts[0].items[] | .metadata.name as $h | host_facts($h) as $f+ | ["H", $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash)] | @tsv),+ ($pvcs[0].items[] | .metadata.labels["clerum.io/host"] as $h+ | if $h == null or .metadata.name != "\($h)-workspace" then+ ["I", .metadata.name] | @tsv+ elif $kinds[$h] == null then+ ["P", .metadata.name, $h, "ORPHAN", "NO_HOST", "-", "-", "0", "-", "-", "-", "-", "-"] | @tsv+ else+ host_facts($h) as $f+ | ["P", .metadata.name, $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash),+ ($f.pods | tostring), ($f.nodes | join(",") | dash), ($f.merge | dash),+ ($f.image | dash), ($f.policy | dash), ($f.secrets | dash)] | @tsv+ end)+' >"$WORK/rows.tsv" 2>"$WORK/rows.err" || die "cannot classify the Host objects: $(cat "$WORK/rows.err")"++HOSTS=0+PRE_825=no+DESKTOP_HOSTS=0+TEMPLATE_STALE=0+while IFS=$'\t' read -r type name kind state template subpath; do+ [ "$type" = H ] || continue+ HOSTS=$((HOSTS + 1))+ echo "HOST $name kind=$kind state=$state template=$template subPath=$subpath"+ [ "$kind" != DESKTOP ] || DESKTOP_HOSTS=$((DESKTOP_HOSTS + 1))+ if [ "$template" = LEGACY_INIT ] || [ "$subpath" = yes ]; then PRE_825=yes; fi+ if [ "$state" != NO_DEPLOYMENT ] && [ "$template" != MERGE ]; then TEMPLATE_STALE=$((TEMPLATE_STALE + 1)); fi+done <"$WORK/rows.tsv"+while IFS=$'\t' read -r type name _; do+ [ "$type" = I ] || continue+ echo "IGNORED_PVC $HOST_NS/$name (not <host>-workspace of a clerum.io/host label)"+done <"$WORK/rows.tsv"+PVC_ROWS="$(awk -F'\t' '$1 == "P"' "$WORK/rows.tsv" | wc -l | tr -d ' ')"+[ "$HOSTS" -gt 0 ] || [ "$PVC_ROWS" -gt 0 ] || die "no Host and no Host PVC in namespace $HOST_NS; wrong --context or namespace?"+echo "PRE_825=$PRE_825"++# ─── verdicts that need no PVC read ───────────────────────────────────+if [ "$COMMAND" = verify ]; then+ case "$FLAG" in+ invalid) echo "HOST_LAYOUT_VERIFY=FLAG_INVALID"; exit 1 ;;+ unresolved) echo "HOST_LAYOUT_VERIFY=FLAG_UNRESOLVED"; exit 1 ;;+ esac+ if [ "$PRE_825" = yes ]; then+ echo "a Host Deployment still renders workspace-layout-init or a workspace subPath: run inventory and finish the upgrade first"+ echo "HOST_LAYOUT_VERIFY=PRE_825"+ exit 1+ fi+fi++# ─── check Pods ───────────────────────────────────────────────────────+# run_check <pvc> <node|-> <image> <policy> <secrets|->: prints the result+# token (and refusal line) for that PVC.+run_check() {+ local claim=$1 node=$2 image=$3 policy=$4 secrets=$5 pod phase elapsed=0 first lines+ pod="$(printf 'host-layout-check-%s' "$claim" | cut -c1-63 | sed 's/[-.]*$//')"+ jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$image" --arg policy "$policy" \+ --arg secrets "$secrets" --arg claim "$claim" --arg script "$CHECK_SCRIPT" --arg root "$PVC_ROOT" '{+ apiVersion: "v1", kind: "Pod",+ metadata: {name: $pod, namespace: $ns, labels: {purpose: "host-layout-merge-check"}},+ spec: ({+ restartPolicy: "Never", automountServiceAccountToken: false, enableServiceLinks: false,+ terminationGracePeriodSeconds: 0,+ securityContext: {runAsUser: 1001, runAsGroup: 1001, runAsNonRoot: true, seccompProfile: {type: "RuntimeDefault"}},+ imagePullSecrets: (if $secrets == "-" then [] else ($secrets | split(",") | map({name: .})) end),+ containers: [{+ name: "check", image: $image, imagePullPolicy: $policy,+ command: ["/bin/sh", "-c", $script],+ resources: {requests: {cpu: "10m", memory: "32Mi"}, limits: {cpu: "100m", memory: "64Mi"}},+ securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: ["ALL"]}},+ volumeMounts: [{name: "pvc", mountPath: $root, readOnly: true}]+ }],+ volumes: [{name: "pvc", persistentVolumeClaim: {claimName: $claim, readOnly: true}}]+ } + (if $node == "-" then {} else {nodeName: $node} end))+ }' >"$WORK/pod.json"+ # Registered before the create, so a create that times out after the API+ # accepted it is still deleted.+ printf '%s\n' "$pod" >>"$CHECK_PODS"+ kc create -f - <"$WORK/pod.json" >/dev/null 2>"$WORK/create.err" ||+ die "check Pod $HOST_NS/$pod create failed: $(cat "$WORK/create.err")"+ while :; do+ read_json "$WORK/status.json" get pod "$pod" -n "$HOST_NS" -o json+ phase="$(jq -r '.status.phase // ""' "$WORK/status.json")"+ case "$phase" in Succeeded | Failed) break ;; esac+ [ "$elapsed" -lt "$CHECK_TIMEOUT" ] || break+ sleep 2+ elapsed=$((elapsed + 2))+ done+ : >"$WORK/check.log"+ if [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; then+ kc logs "$pod" -n "$HOST_NS" >"$WORK/check.log" 2>&1 || die "cannot read the log of check Pod $HOST_NS/$pod: $(cat "$WORK/check.log")"+ fi+ kc delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>"$WORK/delete.err" ||+ die "check Pod $HOST_NS/$pod delete failed: $(cat "$WORK/delete.err")"+ grep -vxF -- "$pod" "$CHECK_PODS" >"$CHECK_PODS.next" || true+ mv "$CHECK_PODS.next" "$CHECK_PODS"++ if [ "$phase" != Succeeded ]; then+ echo "UNREADABLE phase=${phase:-<none>} $(jq -r '[.status.containerStatuses[]?.state.waiting.reason // empty] | join(",")' "$WORK/status.json") $(tail -n 2 "$WORK/check.log" | tr '\n' ' ')"+ return+ fi+ first="$(head -n 1 "$WORK/check.log")"+ lines="$(wc -l <"$WORK/check.log" | tr -d ' ')"+ case "$first" in+ WILL_MERGE | NO_SPLIT_SIGNATURE)+ if [ "$lines" = 1 ]; then echo "$first"; else echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")"; fi+ ;;+ 'WILL_REFUSE_F'[1-4]' workspace layout merge refused: '*) tr '\n' ' ' <"$WORK/check.log" | sed 's/ $//'; echo ;;+ *) echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")" ;;+ esac+}++# The image for a PVC without a Host Deployment comes from HCC itself.+hcc_host_image() {+ local image policy secret+ image="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE)"+ policy="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY)"+ secret="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET)"+ case "$image" in '' | '<unset>' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE ($image) for a check Pod" ;; esac+ case "$policy" in '<unset>') policy=Always ;; '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY ($policy)" ;; esac+ case "$secret" in '<unset>') secret=clerum ;; '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET ($secret)" ;; esac+ printf '%s\t%s\t%s\n' "$image" "$policy" "$secret"+}++# merge_line <status:pod>: the refusal line from the merge init's log.+merge_line() {+ local which=${1%%:*} pod=${1#*:} args=()+ [ "$which" = previous ] && args=(--previous)+ kc logs "$pod" -n "$HOST_NS" -c workspace-layout-merge ${args[@]+"${args[@]}"} >"$WORK/merge.log" 2>&1 ||+ die "cannot read the workspace-layout-merge log of $HOST_NS/$pod: $(cat "$WORK/merge.log")"+ grep -m 1 '^workspace layout merge refused: ' "$WORK/merge.log" || echo "<no refusal line in the log>"+}++TOTAL=0 POD_OK=0 NO_SPLIT=0 SPLIT=0 REFUSED=0 RUNNING=0 UNREADABLE=0 UNVERIFIED=0 UNKNOWN=0+ORPHANS=0 ORPHAN_SPLIT=0 UPGRADE_BLOCKERS=0 SPLIT_TREES=0+while IFS=$'\t' read -r type pvc host kind state _template _subpath pods nodes merge image policy secrets; do+ [ "$type" = P ] || continue+ result=""+ if [ "$COMMAND" = verify ] && [ "$FLAG" = on ] && [ "$pods" -gt 0 ]; then+ case "${merge%%:*}" in+ POD_OK | MERGE_RUNNING | UNKNOWN) result="${merge%%:*}" ;;+ MERGE_REFUSED) result="MERGE_REFUSED $(merge_line "${merge#MERGE_REFUSED:}")" ;;+ *) die "unexpected merge status $merge for $pvc" ;;+ esac+ elif [ "$NO_PVC_PODS" -eq 1 ]; then+ result=UNVERIFIED+ else+ node=-+ if [ "$pods" -gt 0 ]; then+ case "$nodes" in+ - ) node=- ;;+ *,*) result="UNREADABLE Host Pods on more than one node ($nodes)" ;;+ *) node="$nodes" ;;+ esac+ fi+ if [ -z "$result" ]; then+ if [ "$image" = - ]; then+ IFS=$'\t' read -r image policy secrets <<<"$(hcc_host_image)"+ fi+ result="$(run_check "$pvc" "$node" "$image" "$policy" "$secrets" </dev/null)"+ # verify names a tree the merge will still change SPLIT.+ [ "$COMMAND" = inventory ] || [ "$result" != WILL_MERGE ] || result=SPLIT+ fi+ fi+ if [ "$kind" = ORPHAN ]; then+ echo "PVC $HOST_NS/$pvc host=$host kind=ORPHAN state=$state result=$result"+ ORPHANS=$((ORPHANS + 1))+ case "$result" in WILL_MERGE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;; esac+ continue+ fi+ echo "PVC $HOST_NS/$pvc host=$host kind=$kind state=$state result=$result"+ TOTAL=$((TOTAL + 1))+ case "$result" in+ POD_OK) POD_OK=$((POD_OK + 1)) ;;+ NO_SPLIT_SIGNATURE) NO_SPLIT=$((NO_SPLIT + 1)) ;;+ WILL_MERGE | SPLIT) SPLIT=$((SPLIT + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)) ;;+ WILL_REFUSE_*) REFUSED=$((REFUSED + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)); UPGRADE_BLOCKERS=$((UPGRADE_BLOCKERS + 1)) ;;+ MERGE_REFUSED*) REFUSED=$((REFUSED + 1)) ;;+ MERGE_RUNNING) RUNNING=$((RUNNING + 1)) ;;+ UNREADABLE*) UNREADABLE=$((UNREADABLE + 1)) ;;+ UNVERIFIED) UNVERIFIED=$((UNVERIFIED + 1)) ;;+ UNKNOWN) UNKNOWN=$((UNKNOWN + 1)) ;;+ *) die "unclassified result for $pvc: $result" ;;+ esac+done <"$WORK/rows.tsv"++SUM=$((POD_OK + NO_SPLIT + SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED + UNKNOWN))+[ "$SUM" -eq "$TOTAL" ] || die "counted $SUM results for $TOTAL Host PVCs"+echo "COUNTS total=$TOTAL pod_ok=$POD_OK no_split=$NO_SPLIT split=$SPLIT refused=$REFUSED running=$RUNNING unreadable=$UNREADABLE unverified=$UNVERIFIED unknown=$UNKNOWN template_stale=$TEMPLATE_STALE"+echo "ORPHANS=$ORPHANS ORPHAN_SPLIT=$ORPHAN_SPLIT"+if [ "$ORPHAN_SPLIT" -gt 0 ]; then+ echo "note: a Host recreated with an orphan's name after the flag is off starts on an empty database; delete the orphan PVC first or recreate the Host before turning the flag off"+fi+echo "PVC_CONTENTS=NOT_INSPECTED"+if [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; then+ echo "FLAG_FALSE_WITH_SPLIT_TREES=$SPLIT_TREES"+fi++if [ "$COMMAND" = inventory ]; then+ echo "UPGRADE_BLOCKERS=$UPGRADE_BLOCKERS"+ if [ "$DESKTOP_HOSTS" -gt 0 ]; then+ echo "DESKTOP_HOSTS=$DESKTOP_HOSTS export /workspace/state.db* from each desktop Host Pod before the first rollout of this HCC (docs/deploy/host-pvc-layout-upgrade.md)"+ else+ echo "DESKTOP_HOSTS=0"+ fi+ if [ "$UPGRADE_BLOCKERS" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] ||+ { [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; } ||+ [ "$FLAG" = invalid ] || [ "$FLAG" = unresolved ]; then+ echo "HOST_LAYOUT_INVENTORY=BLOCKED"+ exit 1+ fi+ echo "HOST_LAYOUT_INVENTORY=OK"+ exit 0+fi++if [ "$FLAG" = off ]; then+ if [ "$SPLIT_TREES" -gt 0 ]; then+ echo "HOST_LAYOUT_VERIFY=FLAG_FALSE_WITH_SPLIT_TREES"+ else+ echo "HOST_LAYOUT_VERIFY=ALREADY_OFF"+ fi+ exit 1+fi+if [ "$TOTAL" -gt 0 ] && [ "$((SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED + UNKNOWN + TEMPLATE_STALE))" -eq 0 ]; then+ echo "HOST_LAYOUT_VERIFY=PASS"+ exit 0+fi+echo "HOST_LAYOUT_VERIFY=FAIL"+exit 1diff --git a/deploy/scripts/host-layout-merge.test.mjs b/deploy/scripts/host-layout-merge.test.mjsnew file mode 100644index 000000000..68688f376--- /dev/null+++ b/deploy/scripts/host-layout-merge.test.mjs@@ -0,0 +1,531 @@+import assert from 'node:assert/strict'+import { spawnSync } from 'node:child_process'+import {+ chmodSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ rmSync,+ symlinkSync,+ writeFileSync,+} from 'node:fs'+import { tmpdir } from 'node:os'+import path from 'node:path'+import test from 'node:test'+import { fileURLToPath } from 'node:url'++// Runs host-layout-merge.sh against a fake kubectl that answers from canned+// JSON and records every call, so the inventory, the turn-off gate and the+// check Pods are exercised without a cluster. All names are synthetic.++const SCRIPT = fileURLToPath(new URL('./host-layout-merge.sh', import.meta.url))+const CONTEXT = 'test-context'+const NS = 'mcp-host'++const FAKE_KUBECTL = `#!/usr/bin/env node+const fs = require('fs')+const fx = JSON.parse(fs.readFileSync(process.env.FAKE_KUBECTL_FIXTURE, 'utf8'))+const args = process.argv.slice(2)+const stdin = args.includes('-f') ? fs.readFileSync(0, 'utf8') : ''+fs.appendFileSync(process.env.FAKE_KUBECTL_LOG, JSON.stringify({ args, stdin }) + '\\n')+const out = (s, code = 0) => { process.stdout.write(s); process.exit(code) }+if (!args.includes('--context=' + fx.context)) { process.stderr.write('missing --context'); process.exit(97) }+const rest = args.filter(a => !a.startsWith('--context=')).join(' ')+const statePath = process.env.FAKE_KUBECTL_LOG + '.pods.json'+const pods = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, 'utf8')) : {}+const save = () => fs.writeFileSync(statePath, JSON.stringify(pods))+if (rest === 'create -f -') {+ const m = JSON.parse(stdin)+ const claim = m.spec.volumes[0].persistentVolumeClaim.claimName+ pods[m.metadata.name] = claim+ save()+ out('pod/' + m.metadata.name + ' created\\n')+}+let m = rest.match(/^get pod (\\S+) -n \\S+ -o json$/)+if (m && pods[m[1]]) {+ const check = fx.checks[pods[m[1]]] || {}+ out(JSON.stringify({ status: { phase: check.phase || 'Succeeded', containerStatuses: check.containerStatuses || [] } }))+}+m = rest.match(/^logs (\\S+) -n \\S+$/)+if (m && pods[m[1]]) out(fx.checks[pods[m[1]]].output)+m = rest.match(/^delete pod (\\S+) -n \\S+ --wait=true --timeout=60s$/)+if (m && pods[m[1]]) {+ if ((fx.checks[pods[m[1]]] || {}).deleteFails) { process.stderr.write('delete refused'); process.exit(1) }+ delete pods[m[1]]+ save()+ out('pod "' + m[1] + '" deleted\\n')+}+const r = fx.responses[rest]+if (!r) { process.stderr.write('fake kubectl: unexpected call: ' + rest); process.exit(98) }+if (r.stderr) process.stderr.write(r.stderr)+out(typeof r.stdout === 'string' ? r.stdout : JSON.stringify(r.stdout), r.status || 0)+`++function hcc(env = []) {+ return {+ spec: {+ replicas: 1,+ template: {+ spec: {+ containers: [+ {+ name: 'host-context-controller',+ env: [+ { name: 'CONTEXT_MAPPER_HOST_IMAGE', value: 'registry.test/mcp-host:head' },+ ...env,+ ],+ },+ ],+ },+ },+ },+ status: { readyReplicas: 1, updatedReplicas: 1 },+ }+}++function hostCr(name, spec = {}) {+ return { metadata: { name, namespace: NS }, spec: { contextRef: 'ctx', ...spec } }+}++function hostDeployment(name, { replicas = 1, inits = ['workspace-layout-merge'], subPath, containers } = {}) {+ const mount = { name: 'workspace', mountPath: '/workspace', ...(subPath ? { subPath } : {}) }+ return {+ metadata: { name, labels: { app: name, 'clerum.io/host': name } },+ spec: {+ replicas,+ template: {+ spec: {+ initContainers: inits.map(n => ({ name: n, volumeMounts: [{ name: 'workspace', mountPath: '/mnt/workspace-root' }] })),+ containers: containers ?? [+ { name: 'mcp-host', image: `registry.test/mcp-host:${name}`, imagePullPolicy: 'IfNotPresent', volumeMounts: [mount] },+ ],+ imagePullSecrets: [{ name: 'pull-secret' }],+ volumes: [{ name: 'workspace', persistentVolumeClaim: { claimName: `${name}-workspace` } }],+ },+ },+ },+ }+}++function hostPod(host, node, mergeStatus) {+ return {+ metadata: { name: `${host}-pod`, labels: { app: host, 'clerum.io/host': host } },+ spec: { nodeName: node },+ status: {+ phase: 'Running',+ initContainerStatuses: mergeStatus === undefined ? [] : [{ name: 'workspace-layout-merge', ...mergeStatus }],+ },+ }+}++function pvc(host) {+ return {+ metadata: {+ name: `${host}-workspace`,+ labels: { 'clerum.io/managed-by': 'host-context-controller', 'clerum.io/host': host },+ },+ }+}++const EXITED_0 = { state: { terminated: { exitCode: 0 } } }+const F2_LINE = 'workspace layout merge refused: more than one live SQLite database across /mnt/workspace-root, /mnt/workspace-root/state and /mnt/workspace-root/workspace'++// A stateful active Host, a suspended stateless Host and a desktop Host whose+// mcp-host container is not the first one, plus a channel-reader Deployment+// and Pod that carry a Host label and must be ignored.+function fleet() {+ return {+ hcc: hcc(),+ hosts: [+ hostCr('alpha'),+ hostCr('beta', { lifecycle: { stateless: true } }),+ hostCr('gamma', { desktop: { browser: true } }),+ ],+ deployments: [+ hostDeployment('alpha'),+ hostDeployment('beta', { replicas: 0 }),+ hostDeployment('gamma', {+ containers: [+ { name: 'x11', image: 'registry.test/x11:1' },+ { name: 'mcp-host', image: 'registry.test/mcp-host-desktop:gamma', imagePullPolicy: 'Always', volumeMounts: [{ name: 'workspace', mountPath: '/config/workspace' }] },+ ],+ }),+ { metadata: { name: 'channel-reader-alpha', labels: { app: 'channel-reader', 'clerum.io/host': 'alpha' } }, spec: { replicas: 1, template: { spec: { containers: [{ name: 'reader' }] } } } },+ ],+ pods: [+ hostPod('alpha', 'node-a', EXITED_0),+ hostPod('gamma', 'node-b', EXITED_0),+ { metadata: { name: 'channel-reader-alpha-pod', labels: { app: 'channel-reader', 'clerum.io/host': 'alpha' } }, spec: { nodeName: 'node-c' }, status: { phase: 'Running' } },+ ],+ pvcs: [pvc('alpha'), pvc('beta'), pvc('gamma')],+ checks: {+ 'alpha-workspace': { output: 'NO_SPLIT_SIGNATURE\n' },+ 'beta-workspace': { output: 'NO_SPLIT_SIGNATURE\n' },+ 'gamma-workspace': { output: 'NO_SPLIT_SIGNATURE\n' },+ },+ extra: {},+ }+}++function responses(f) {+ const list = items => ({ stdout: { items } })+ return {+ 'get deployment host-context-controller -n control-plane -o json': { stdout: f.hcc },+ [`get hosts.clerum.io -n ${NS} -o json`]: list(f.hosts),+ [`get deployments -n ${NS} -l clerum.io/host -o json`]: list(f.deployments),+ [`get pods -n ${NS} -l clerum.io/host -o json`]: list(f.pods),+ [`get pvc -n ${NS} -l clerum.io/managed-by=host-context-controller -o json`]: list(f.pvcs),+ ...f.extra,+ }+}++function run(f, args) {+ const dir = mkdtempSync(path.join(tmpdir(), 'host-layout-merge-test-'))+ try {+ const kubectl = path.join(dir, 'kubectl')+ writeFileSync(kubectl, FAKE_KUBECTL, { mode: 0o755 })+ const fixture = path.join(dir, 'fixture.json')+ writeFileSync(fixture, JSON.stringify({ context: CONTEXT, responses: responses(f), checks: f.checks }))+ const log = path.join(dir, 'calls.jsonl')+ writeFileSync(log, '')+ const res = spawnSync('bash', [SCRIPT, ...args], {+ encoding: 'utf8',+ env: { ...process.env, KUBECTL_BIN: kubectl, FAKE_KUBECTL_FIXTURE: fixture, FAKE_KUBECTL_LOG: log },+ })+ if (res.error) throw res.error+ const calls = readFileSync(log, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l))+ return { status: res.status, stdout: res.stdout, stderr: res.stderr, calls }+ } finally {+ rmSync(dir, { recursive: true, force: true })+ }+}++const verify = (f, ...extra) => run(f, ['verify', '--context', CONTEXT, ...extra])+const inventory = (f, ...extra) => run(f, ['inventory', '--context', CONTEXT, ...extra])++function row(out, pvcName) {+ const lines = out.split('\n').filter(l => l.startsWith(`PVC ${NS}/${pvcName} `))+ assert.equal(lines.length, 1, `one row for ${pvcName} in:\n${out}`)+ return lines[0]+}++const verbs = calls => calls.map(c => c.args.filter(a => !a.startsWith('--context='))[0])+const creates = calls => calls.filter(c => verbs([c])[0] === 'create').map(c => JSON.parse(c.stdin))++function assertReadOnlyCalls(calls) {+ assert.ok(calls.length > 0, 'kubectl was called')+ for (const c of calls) assert.ok(c.args.includes(`--context=${CONTEXT}`), `call without --context: ${c.args.join(' ')}`)+ for (const verb of verbs(calls)) assert.ok(['get', 'logs'].includes(verb), `mutating call: ${verb}`)+}++function assertCheckPodsOnly(calls, claims) {+ for (const c of calls) assert.ok(c.args.includes(`--context=${CONTEXT}`), `call without --context: ${c.args.join(' ')}`)+ const mutating = verbs(calls).filter(v => !['get', 'logs'].includes(v))+ assert.deepEqual(mutating.sort(), [...claims.map(() => 'create'), ...claims.map(() => 'delete')].sort())+ assert.deepEqual(creates(calls).map(m => m.spec.volumes[0].persistentVolumeClaim.claimName).sort(), [...claims].sort())+}++test('verify passes a merged fleet and checks only the PVC without a Pod', () => {+ const r = verify(fleet())+ assert.equal(r.status, 0, r.stdout + r.stderr)+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)+ assert.match(row(r.stdout, 'alpha-workspace'), /host=alpha kind=STATEFUL state=ACTIVE result=POD_OK$/)+ assert.match(row(r.stdout, 'beta-workspace'), /host=beta kind=STATELESS state=SUSPENDED result=NO_SPLIT_SIGNATURE$/)+ assert.match(row(r.stdout, 'gamma-workspace'), /host=gamma kind=DESKTOP state=ACTIVE result=POD_OK$/)+ assert.match(r.stdout, /^COUNTS total=3 pod_ok=2 no_split=1 split=0 refused=0 running=0 unreadable=0 unverified=0 unknown=0 template_stale=0$/m)+ assert.match(r.stdout, /^PVC_CONTENTS=NOT_INSPECTED$/m)+ assertCheckPodsOnly(r.calls, ['beta-workspace'])++ const [pod] = creates(r.calls)+ assert.equal(pod.spec.nodeName, undefined)+ assert.equal(pod.spec.automountServiceAccountToken, false)+ assert.equal(pod.spec.securityContext.fsGroup, undefined)+ assert.equal(pod.spec.securityContext.runAsUser, 1001)+ assert.equal(pod.spec.securityContext.runAsGroup, 1001)+ assert.equal(pod.spec.volumes[0].persistentVolumeClaim.readOnly, true)+ assert.deepEqual(pod.spec.containers[0].volumeMounts, [{ name: 'pvc', mountPath: '/mnt/workspace-root', readOnly: true }])+ assert.equal(pod.spec.containers[0].image, 'registry.test/mcp-host:beta')+ assert.equal(pod.spec.containers[0].imagePullPolicy, 'IfNotPresent')+ assert.deepEqual(pod.spec.imagePullSecrets, [{ name: 'pull-secret' }])+ assert.deepEqual(pod.spec.containers[0].securityContext.capabilities, { drop: ['ALL'] })+ assert.doesNotMatch(pod.spec.containers[0].command[2], /\bmv |\brmdir /)+})++test('inventory reads every PVC, pinning running Hosts to their node', () => {+ const f = fleet()+ f.checks['beta-workspace'].output = 'WILL_MERGE\n'+ const r = inventory(f)+ assert.equal(r.status, 0, r.stdout + r.stderr)+ assert.match(r.stdout, /^HOST_LAYOUT_INVENTORY=OK$/m)+ assert.match(r.stdout, /^UPGRADE_BLOCKERS=0$/m)+ assert.match(row(r.stdout, 'beta-workspace'), /result=WILL_MERGE$/)+ assert.match(r.stdout, /^DESKTOP_HOSTS=1 /m)+ assert.match(r.stdout, /^PRE_825=no$/m)+ assertCheckPodsOnly(r.calls, ['alpha-workspace', 'beta-workspace', 'gamma-workspace'])+ const nodes = Object.fromEntries(creates(r.calls).map(m => [m.spec.volumes[0].persistentVolumeClaim.claimName, m.spec.nodeName]))+ assert.deepEqual(nodes, { 'alpha-workspace': 'node-a', 'beta-workspace': undefined, 'gamma-workspace': 'node-b' })+})++test('inventory reports a Host switched to stateful before the upgrade as a blocker (T5)', () => {+ const f = fleet()+ f.checks['alpha-workspace'].output = `WILL_REFUSE_F2 ${F2_LINE}\n`+ const r = inventory(f)+ assert.equal(r.status, 1, r.stdout + r.stderr)+ assert.match(r.stdout, /^UPGRADE_BLOCKERS=1$/m)+ assert.match(r.stdout, /^HOST_LAYOUT_INVENTORY=BLOCKED$/m)+ assert.ok(row(r.stdout, 'alpha-workspace').endsWith(`result=WILL_REFUSE_F2 ${F2_LINE}`))+})++test('inventory reports a legacy fleet as PRE_825 and still reads it', () => {+ const f = fleet()+ f.deployments[1] = hostDeployment('beta', { replicas: 0, inits: ['workspace-layout-init'], subPath: 'workspace' })+ f.checks['beta-workspace'].output = 'WILL_MERGE\n'+ const r = inventory(f)+ assert.equal(r.status, 0, r.stdout + r.stderr)+ assert.match(r.stdout, /^PRE_825=yes$/m)+ assert.match(r.stdout, /^HOST beta kind=STATELESS state=SUSPENDED template=LEGACY_INIT subPath=yes$/m)+ assert.match(row(r.stdout, 'beta-workspace'), /result=WILL_MERGE$/)+})++test('--no-pvc-pods starts no Pod: inventory reports UNVERIFIED, verify fails', () => {+ const inv = inventory(fleet(), '--no-pvc-pods')+ assert.equal(inv.status, 0, inv.stdout + inv.stderr)+ assert.match(row(inv.stdout, 'beta-workspace'), /result=UNVERIFIED$/)+ assertReadOnlyCalls(inv.calls)++ const ver = verify(fleet(), '--no-pvc-pods')+ assert.equal(ver.status, 1, ver.stdout + ver.stderr)+ assert.match(row(ver.stdout, 'alpha-workspace'), /result=POD_OK$/)+ assert.match(row(ver.stdout, 'beta-workspace'), /result=UNVERIFIED$/)+ assert.match(ver.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)+ assertReadOnlyCalls(ver.calls)+})++test('the merge flag decides the verdict before any PVC is read', () => {+ for (const [env, verdict] of [+ [{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'ture' }, 'FLAG_INVALID'],+ [{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', valueFrom: { configMapKeyRef: { name: 'c', key: 'k' } } }, 'FLAG_UNRESOLVED'],+ ]) {+ const f = fleet()+ f.hcc = hcc([env])+ const r = verify(f)+ assert.equal(r.status, 1, r.stdout + r.stderr)+ assert.match(r.stdout, new RegExp(`^HOST_LAYOUT_VERIFY=${verdict}$`, 'm'))+ assertReadOnlyCalls(r.calls)+ }+ const on = fleet()+ on.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'true' }])+ assert.equal(verify(on).status, 0)+})++test('flag false: ALREADY_OFF without split trees, FLAG_FALSE_WITH_SPLIT_TREES with one', () => {+ const off = fleet()+ off.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'false' }])+ const clean = verify(off)+ assert.equal(clean.status, 1, clean.stdout + clean.stderr)+ assert.match(clean.stdout, /^HOST_LAYOUT_VERIFY=ALREADY_OFF$/m)+ // With the flag off a running Pod has no merge init to read, so every PVC is read.+ assertCheckPodsOnly(clean.calls, ['alpha-workspace', 'beta-workspace', 'gamma-workspace'])++ off.checks['beta-workspace'].output = 'WILL_MERGE\n'+ for (const r of [verify(off), inventory(off)]) {+ assert.equal(r.status, 1, r.stdout + r.stderr)+ assert.match(r.stdout, /^FLAG_FALSE_WITH_SPLIT_TREES=1$/m)+ }+ assert.match(verify(off).stdout, /^HOST_LAYOUT_VERIFY=FLAG_FALSE_WITH_SPLIT_TREES$/m)+})++test('verify stops at PRE_825 for a legacy init or a workspace subPath', () => {+ for (const dep of [+ hostDeployment('alpha', { inits: ['workspace-layout-init'] }),+ hostDeployment('alpha', { subPath: 'workspace' }),+ ]) {+ const f = fleet()+ f.deployments[0] = dep+ const r = verify(f)+ assert.equal(r.status, 1, r.stdout + r.stderr)+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PRE_825$/m)+ assertReadOnlyCalls(r.calls)+ }+})++test('verify fails a Host Deployment that does not render the merge init', () => {+ const f = fleet()+ f.deployments[0] = hostDeployment('alpha', { inits: [] })+ const r = verify(f)+ assert.equal(r.status, 1, r.stdout + r.stderr)+ assert.match(r.stdout, /^HOST alpha kind=STATEFUL state=ACTIVE template=NO_INIT subPath=no$/m)+ assert.match(r.stdout, / template_stale=1$/m)+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)+})++test('verify reads the merge init status of running Pods', () => {+ const cases = [+ [{ state: { running: {} } }, 'MERGE_RUNNING', null],+ [{ state: { terminated: { exitCode: 1 } } }, `MERGE_REFUSED ${F2_LINE}`, ''],+ [{ state: { waiting: { reason: 'CrashLoopBackOff' } }, lastState: { terminated: { exitCode: 1 } } }, `MERGE_REFUSED ${F2_LINE}`, ' --previous'],+ [{ state: { terminated: { exitCode: 137 } } }, 'UNKNOWN', null],+ [undefined, 'UNKNOWN', null],+ ]+ for (const [status, result, logsFlag] of cases) {+ const f = fleet()+ f.pods[0] = hostPod('alpha', 'node-a', status)+ if (logsFlag !== null) {+ f.extra[`logs alpha-pod -n ${NS} -c workspace-layout-merge${logsFlag}`] = { stdout: `${F2_LINE}\n` }+ }+ const r = verify(f)+ assert.equal(r.status, 1, `${result}: ${r.stdout}${r.stderr}`)+ assert.ok(row(r.stdout, 'alpha-workspace').endsWith(`result=${result}`), row(r.stdout, 'alpha-workspace'))+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)+ }+ const recovered = fleet()+ recovered.pods[0] = hostPod('alpha', 'node-a', { state: { terminated: { exitCode: 0 } }, lastState: { terminated: { exitCode: 1 } } })+ const r = verify(recovered)+ assert.equal(r.status, 0, r.stdout + r.stderr)+ assert.match(row(r.stdout, 'alpha-workspace'), /result=POD_OK$/)+})++test('verify fails a suspended Host whose PVC is still split or unreadable', () => {+ const cases = [+ [{ output: 'WILL_MERGE\n' }, 'SPLIT', / split=1 /],+ [{ output: `WILL_REFUSE_F2 ${F2_LINE}\n` }, `WILL_REFUSE_F2 ${F2_LINE}`, / refused=1 /],+ [{ output: '' }, 'UNREADABLE', / unreadable=1 /],+ [{ output: 'WILL_MERGE\nNO_SPLIT_SIGNATURE\n' }, 'UNREADABLE', / unreadable=1 /],+ [{ phase: 'Failed', output: 'sh: boom\n' }, 'UNREADABLE', / unreadable=1 /],+ [{ phase: 'Pending', containerStatuses: [{ state: { waiting: { reason: 'ImagePullBackOff' } } }] }, 'UNREADABLE', / unreadable=1 /],+ ]+ for (const [check, result, count] of cases) {+ const f = fleet()+ f.checks['beta-workspace'] = check+ const r = verify(f, '--check-timeout', '0')+ assert.equal(r.status, 1, `${result}: ${r.stdout}${r.stderr}`)+ assert.ok(row(r.stdout, 'beta-workspace').includes(`result=${result}`), row(r.stdout, 'beta-workspace'))+ assert.match(r.stdout, count)+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)+ // The check Pod is deleted whatever it printed.+ assertCheckPodsOnly(r.calls, ['beta-workspace'])+ }+})++test('an orphan split PVC is reported without blocking the verdict (T9)', () => {+ const f = fleet()+ f.pvcs.push(pvc('delta'))+ f.checks['delta-workspace'] = { output: 'WILL_MERGE\n' }+ const r = verify(f)+ assert.equal(r.status, 0, r.stdout + r.stderr)+ assert.match(row(r.stdout, 'delta-workspace'), /host=delta kind=ORPHAN state=NO_HOST result=SPLIT$/)+ assert.match(r.stdout, /^ORPHANS=1 ORPHAN_SPLIT=1$/m)+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)+ const orphanPod = creates(r.calls).find(m => m.spec.volumes[0].persistentVolumeClaim.claimName === 'delta-workspace')+ assert.equal(orphanPod.spec.containers[0].image, 'registry.test/mcp-host:head')+ assert.equal(orphanPod.spec.containers[0].imagePullPolicy, 'Always')+ assert.deepEqual(orphanPod.spec.imagePullSecrets, [{ name: 'clerum' }])+})++test('read failures, an empty scope and a failed cleanup exit 2', () => {+ const empty = fleet()+ empty.hosts = []+ empty.deployments = []+ empty.pods = []+ empty.pvcs = []+ const e = verify(empty)+ assert.equal(e.status, 2, e.stdout + e.stderr)+ assert.match(e.stderr, /no Host and no Host PVC/)++ const failing = fleet()+ failing.extra[`get pods -n ${NS} -l clerum.io/host -o json`] = { stdout: { items: [] }, status: 1, stderr: 'forbidden' }+ const f = verify(failing)+ assert.equal(f.status, 2, f.stdout + f.stderr)+ assert.match(f.stderr, /forbidden/)+ assert.doesNotMatch(f.stdout, /HOST_LAYOUT_VERIFY=/)++ const notReady = fleet()+ notReady.hcc.status.readyReplicas = 0+ const n = verify(notReady)+ assert.equal(n.status, 2, n.stdout + n.stderr)+ assert.match(n.stderr, /HCC_NOT_READY/)++ const stuck = fleet()+ stuck.checks['beta-workspace'].deleteFails = true+ const s = verify(stuck)+ assert.equal(s.status, 2, s.stdout + s.stderr)+ assert.match(s.stderr, /delete/)+})++test('usage errors exit 2 without calling kubectl', () => {+ for (const args of [['verify'], ['inventory', '--context'], ['verify', '--context', CONTEXT, '--bogus'], ['frobnicate', '--context', CONTEXT], []]) {+ const r = run(fleet(), args)+ assert.equal(r.status, 2, `${args.join(' ')}: ${r.stdout}${r.stderr}`)+ assert.equal(r.calls.length, 0)+ }+})++// The check the Pods run, on real trees. Each tree's listing is asserted+// before its token so a wrong fixture cannot pass as a wrong token.+function tree(build) {+ const root = mkdtempSync(path.join(tmpdir(), 'host-layout-check-tree-'))+ build(root)+ return root+}++function check(root) {+ const printed = spawnSync('bash', [SCRIPT, 'check-script', '--root', root], { encoding: 'utf8' })+ assert.equal(printed.status, 0, printed.stderr)+ const res = spawnSync('/bin/sh', ['-c', printed.stdout], { encoding: 'utf8' })+ assert.equal(res.stderr, '')+ assert.equal(res.status, 0)+ return res.stdout+}++const file = (p, body = 'x') => writeFileSync(p, body)++test('check-script classifies split, foreign and refused trees', () => {+ const trees = [+ ['split', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); file(`${r}/workspace/SOUL.md`) }, ['state', 'workspace'], 'WILL_MERGE\n'],+ ['backup only', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db.pre-1.bak`) }, ['state', 'workspace'], 'WILL_MERGE\n'],+ ['foreign state.db name', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db.json`) }, ['state', 'workspace'], 'NO_SPLIT_SIGNATURE\n'],+ ['symlinked state', r => { mkdirSync(`${r}/elsewhere`); file(`${r}/elsewhere/state.db`); symlinkSync('elsewhere', `${r}/state`); mkdirSync(`${r}/workspace`) }, ['elsewhere', 'state', 'workspace'], 'NO_SPLIT_SIGNATURE\n'],+ ['workspace missing', r => { mkdirSync(`${r}/state`); file(`${r}/state/state.db`) }, ['state'], 'NO_SPLIT_SIGNATURE\n'],+ ['stateful root', r => { file(`${r}/state.db`); file(`${r}/SOUL.md`) }, ['SOUL.md', 'state.db'], 'NO_SPLIT_SIGNATURE\n'],+ ['T5', r => { file(`${r}/state.db`); mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`) }, ['state', 'state.db', 'workspace'], `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ROOT, ROOT/state and ROOT/workspace\n`],+ ['read-only directory to move', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); mkdirSync(`${r}/workspace/users`); chmodSync(`${r}/workspace/users`, 0o555) }, ['state', 'workspace'], 'WILL_REFUSE_F4 workspace layout merge refused: not writable by uid UID: ROOT/workspace/users\n'],+ ]+ const uid = String(process.getuid())+ for (const [name, build, listing, expected] of trees) {+ const root = tree(build)+ try {+ assert.deepEqual(listDir(root), listing, name)+ assert.equal(check(root), expected.replaceAll('ROOT', root).replaceAll('UID', uid), name)+ assert.deepEqual(listDir(root), listing, `${name}: the check changed the tree`)+ } finally {+ chmodTree(root)+ rmSync(root, { recursive: true, force: true })+ }+ }+})++test('check-script refuses an unlistable state/ like the merge init does', { skip: process.getuid() === 0 && 'root reads any mode' }, () => {+ const root = tree(r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); chmodSync(`${r}/state`, 0o300) })+ try {+ assert.equal(check(root), `WILL_REFUSE_F1 workspace layout merge refused: state directory is not readable and searchable: ${root}/state\n`)+ } finally {+ chmodTree(root)+ rmSync(root, { recursive: true, force: true })+ }+})++test('check-script ends before the moves', () => {+ const printed = spawnSync('bash', [SCRIPT, 'check-script'], { encoding: 'utf8' })+ assert.equal(printed.status, 0, printed.stderr)+ assert.match(printed.stdout, /^root='\/mnt\/workspace-root'$/m)+ assert.match(printed.stdout, /\necho WILL_MERGE\n$/)+ assert.doesNotMatch(printed.stdout, /\bmv |\brmdir |\[ -w /)+})++function listDir(root) {+ return spawnSync('ls', ['-A', root], { encoding: 'utf8' }).stdout.split('\n').filter(Boolean).sort()+}++function chmodTree(root) {+ spawnSync('chmod', ['-R', 'u+rwx', root])+}diff --git a/deploy/scripts/workspace-layout-merge.sh b/deploy/scripts/workspace-layout-merge.shnew file mode 100644index 000000000..60c33e2a0--- /dev/null+++ b/deploy/scripts/workspace-layout-merge.sh@@ -0,0 +1,148 @@+set -eu+root='/mnt/workspace-root'+fail_merge() {+ echo "workspace layout merge refused: $1" >&2+ exit 1+}+present() {+ [ -e "$1" ] || [ -L "$1" ]+}+canonical_dir() {+ (CDPATH= cd -P "$1" 2>/dev/null && pwd -P)+}+assert_managed_dir() {+ path="$1"+ label="$2"+ [ ! -L "$path" ] || fail_merge "$label directory is a symlink: $path"+ [ -d "$path" ] || fail_merge "$label path is not a directory: $path"+ resolved="$(canonical_dir "$path")" || fail_merge "cannot resolve $label directory: $path"+ case "$resolved" in+ "$root_real"/*) ;;+ *) fail_merge "$label directory escapes the workspace PVC root: $path" ;;+ esac+}+is_empty_dir() {+ [ -d "$1" ] && [ ! -L "$1" ] || return 1+ listing="$(ls -A "$1" 2>/dev/null)" || fail_merge "cannot list directory: $1"+ [ -z "$listing" ]+}+has_main() {+ present "$1/state.db"+}+has_journal() {+ present "$1/state.db-wal" || present "$1/state.db-shm"+}++[ ! -L "$root" ] || fail_merge "workspace PVC root is a symlink: $root"+[ -d "$root" ] || fail_merge "workspace PVC root is not a directory: $root"+root_real="$(canonical_dir "$root")" || fail_merge "cannot resolve workspace PVC root: $root"+[ "$root_real" != "/" ] || fail_merge "workspace PVC root resolves to /"++# Split-layout signature: real state/ and workspace/ directories, and a+# database entry under state/.+[ ! -L "$root/state" ] && [ -d "$root/state" ] || exit 0+[ ! -L "$root/workspace" ] && [ -d "$root/workspace" ] || exit 0+# The globs cannot see the entries of a state/ they cannot list or search.+[ -r "$root/state" ] && [ -x "$root/state" ] ||+ fail_merge "state directory is not readable and searchable: $root/state"+# Only the names the earlier init writes count: the database, its journals+# and the .pre-<ts>.bak copy it keeps of each of them from an older source.+signature=0+for entry in "$root/state"/state.db "$root/state"/state.db-wal "$root/state"/state.db-shm \+ "$root/state"/state.db.pre-*.bak "$root/state"/state.db-wal.pre-*.bak "$root/state"/state.db-shm.pre-*.bak; do+ if present "$entry"; then signature=1; fi+done+[ "$signature" -eq 1 ] || exit 0++# F1+assert_managed_dir "$root/state" state+assert_managed_dir "$root/workspace" workspace+# The move globs cannot see the entries of a workspace/ they cannot list.+[ -r "$root/workspace" ] && [ -x "$root/workspace" ] ||+ fail_merge "workspace directory is not readable and searchable: $root/workspace"+for dir in "$root" "$root/state" "$root/workspace"; do+ for entry in "$dir"/state.db "$dir"/state.db-wal "$dir"/state.db-shm \+ "$dir"/state.db.pre-*.bak "$dir"/state.db-wal.pre-*.bak "$dir"/state.db-shm.pre-*.bak; do+ present "$entry" || continue+ [ ! -L "$entry" ] || fail_merge "SQLite file is a symlink: $entry"+ [ -f "$entry" ] || fail_merge "SQLite file is not a regular file: $entry"+ done+done++# F2+live=0+root_live=0+workspace_live=0+if has_main "$root" || has_journal "$root"; then root_live=1; live=$((live + 1)); fi+if has_main "$root/state" || has_journal "$root/state"; then live=$((live + 1)); fi+if has_main "$root/workspace" || has_journal "$root/workspace"; then+ workspace_live=1+ live=$((live + 1))+fi+resume=0+if [ "$root_live" -eq 1 ] && ! has_main "$root" && has_main "$root/state" && [ "$workspace_live" -eq 0 ]; then+ resume=1+fi+if [ "$live" -gt 1 ] && [ "$resume" -eq 0 ]; then+ fail_merge "more than one live SQLite database across $root, $root/state and $root/workspace"+fi++# F3+! present "$root/workspace/lost+found" || fail_merge "workspace/lost+found would remain: $root/workspace/lost+found"+for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?*; do+ present "$entry" || continue+ base="${entry##*/}"+ if present "$root/$base"; then+ if is_empty_dir "$entry" && [ -d "$root/$base" ] && [ ! -L "$root/$base" ]; then continue; fi+ fail_merge "destination exists: $root/$base"+ fi+done+for entry in "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do+ present "$entry" || continue+ base="${entry##*/}"+ if present "$root/$base"; then+ if is_empty_dir "$entry" && [ -d "$root/$base" ] && [ ! -L "$root/$base" ]; then continue; fi+ fail_merge "destination exists: $root/$base"+ fi+ if present "$root/workspace/$base"; then+ fail_merge "$base exists under both state/ and workspace/"+ fi+done++# F4+for dir in "$root" "$root/state" "$root/workspace"; do+ [ -w "$dir" ] || fail_merge "not writable by uid $(id -u): $dir"+done+# Renaming a directory to a new parent rewrites its .. entry, which needs+# write permission on the directory itself.+for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?* \+ "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do+ [ -d "$entry" ] && [ ! -L "$entry" ] || continue+ ! present "$root/${entry##*/}" || continue+ [ -w "$entry" ] || fail_merge "not writable by uid $(id -u): $entry"+done++move_to_root() {+ entry="$1"+ base="${entry##*/}"+ assert_managed_dir "$root/state" state+ assert_managed_dir "$root/workspace" workspace+ if present "$root/$base"; then+ rmdir "$entry"+ else+ mv "$entry" "$root/$base"+ fi+}+for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?*; do+ present "$entry" || continue+ move_to_root "$entry"+done+for entry in "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do+ present "$entry" || continue+ [ "${entry##*/}" != state.db ] || continue+ move_to_root "$entry"+done+if present "$root/state/state.db"; then move_to_root "$root/state/state.db"; fi+rmdir "$root/workspace"+rmdir "$root/state"+syncdiff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex cd174bb0b..c75e50255 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.396",+ "version": "0.1.397","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.396",+ "version": "0.1.397","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 559c50b32..b2ec4d573 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.396",+ "version": "0.1.397","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.tsindex cc390f9ef..ee0699eb0 100644--- a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts+++ b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts@@ -6,7 +6,7 @@* uncheckpointed WAL frames, split by the base `workspace-layout-init`* (committed verbatim as a fixture), then merged by the new script.*/-import { afterEach, describe, expect, it } from 'vitest'+import { afterAll, afterEach, describe, expect, it } from 'vitest'import { spawnSync } from 'node:child_process'import { createHash } from 'node:crypto'import {@@ -65,10 +65,63 @@ function runScript(script: string, args: string[] = [], env?: NodeJS.ProcessEnv)return { status: res.status ?? -1, stderr: res.stderr ?? '' }}+// deploy/scripts/host-layout-merge.sh predicts this merge on clusters from a+// committed copy of the script, run read-only. Every merge below without a+// crash env first asks it for a prediction and then checks it against what+// the merge did, so each tree in this file is also a drift case.+const DEPLOY_SCRIPTS = join(__dirname, '..', '..', 'deploy', 'scripts')+const MERGE_SCRIPT_COPY = join(DEPLOY_SCRIPTS, 'workspace-layout-merge.sh')+const VERIFY_SCRIPT = join(DEPLOY_SCRIPTS, 'host-layout-merge.sh')+const PREDICTED_TOKENS = new Set<string>()++function predictMerge(root: string): string {+ const printed = spawnSync('bash', [VERIFY_SCRIPT, 'check-script', '--root', root], {+ encoding: 'utf8',+ })+ if (printed.error) throw printed.error+ expect(printed.status, printed.stderr).toBe(0)+ const res = spawnSync('/bin/sh', ['-c', printed.stdout], { encoding: 'utf8' })+ if (res.error) throw res.error+ expect(res, `check on ${root}`).toMatchObject({ status: 0, stderr: '' })+ return res.stdout+}++function rootListing(root: string): string {+ return readdirSync(root).sort().join('\0')+}+function merge(root: string, env?: NodeJS.ProcessEnv): RunResult {- return runScript(buildWorkspaceLayoutMergeScript(root), [], env)+ if (env) return runScript(buildWorkspaceLayoutMergeScript(root), [], env)+ const predicted = predictMerge(root)+ // A root that is not a directory is refused, so it has no listing to compare.+ const before = statSync(root).isDirectory() ? rootListing(root) : null+ const res = runScript(buildWorkspaceLayoutMergeScript(root))+ const token = predicted.split(/[ \n]/, 1)[0]+ PREDICTED_TOKENS.add(token)+ if (res.status === 1) {+ expect(token, predicted).toMatch(/^WILL_REFUSE_F[1-4]$/)+ expect(predicted.slice(token.length + 1)).toBe(res.stderr)+ } else {+ expect(res.status, res.stderr).toBe(0)+ expect(before, `${root} merged without being a directory`).not.toBeNull()+ const expected = rootListing(root) === before ? 'NO_SPLIT_SIGNATURE' : 'WILL_MERGE'+ expect(predicted).toBe(`${expected}\n`)+ }+ return res}+afterAll(() => {+ // Liveness of the drift check: every outcome was predicted at least once.+ expect([...PREDICTED_TOKENS].sort()).toEqual([+ 'NO_SPLIT_SIGNATURE',+ 'WILL_MERGE',+ 'WILL_REFUSE_F1',+ 'WILL_REFUSE_F2',+ 'WILL_REFUSE_F3',+ 'WILL_REFUSE_F4',+ ])+})+function baseInit(root: string): void {const res = runScript(readFileSync(BASE_INIT, 'utf8'), [root])expect(res, res.stderr).toMatchObject({ status: 0 })@@ -176,6 +229,24 @@ function mvCalls(env: NodeJS.ProcessEnv): number {}describe('workspace layout merge', () => {+ it('ships the rendered script as the copy the verify script reads', () => {+ expect(+ readFileSync(MERGE_SCRIPT_COPY, 'utf8'),+ 'regenerate deploy/scripts/workspace-layout-merge.sh from buildWorkspaceLayoutMergeScript()'+ ).toBe(buildWorkspaceLayoutMergeScript())+ })++ it('predicts a refusal for a Host switched to stateful before the upgrade (T5)', () => {+ // A v0.10/v0.11 stateless Host switched to stateful: new chats in the+ // root database, the older history still under state/.+ const root = splitRoot()+ cpSync(join(root, 'state', 'state.db'), join(root, 'state.db'))+ expect(predictMerge(root)).toBe(+ `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ${root}, ${root}/state and ${root}/workspace\n`+ )+ expect(merge(root).status).toBe(1)+ })+it('merges a split tree with an uncheckpointed WAL back to the PVC root', () => {const root = splitRoot()// Witness that the split tree is the real one: the database and its