Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(deploy): verify the workspace layout merge on any cluster

Alfredo Lopeza4c1b1c3b73b
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index a0acffe54..d7e8da5e0 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -743,12 +743,13 @@ jobs:
- name: Test minikube Makefile targets
run: bash scripts/tests/test-minikube-makefile.sh
- - name: Test the #825 gate context guards, report verdict and E2E helpers
+ - name: Test the #825 gate context guards, report verdict, E2E helpers and layout merge check
run: >-
node scripts/tests/run-node-test-files.mjs
scripts/e2e/playwright-report-gate.test.mjs
scripts/e2e/e2e-hcc-storage-layout-switch.test.mjs
scripts/e2e/e2e-stateless-suspend-wake.test.mjs
+ deploy/scripts/host-layout-merge.test.mjs
# make test-service-matrix fails when Makefile TEST_SERVICES and the
# `test` job's service matrix above drift apart.
diff --git a/deploy/scripts/host-layout-merge.sh b/deploy/scripts/host-layout-merge.sh
new file mode 100755
index 000000000..2354a5a23
--- /dev/null
+++ b/deploy/scripts/host-layout-merge.sh
@@ -0,0 +1,525 @@
+#!/usr/bin/env bash
+# Issue #825: read-only inventory and turn-off gate for the HCC workspace
+# layout merge on any cluster (docs/deploy/host-pvc-layout-upgrade.md).
+#
+# Every Host PVC is classified with the merge init's own rules: the check
+# Pods run deploy/scripts/workspace-layout-merge.sh (the script HCC renders,
+# kept equal to it by a host-context-controller test) with the moves removed
+# and the writability checks read from mode bits, because the PVC is mounted
+# read-only. The script never opens a database and never execs into a Host.
+set -euo pipefail
+
+usage() {
+ cat <<'EOF'
+usage:
+ host-layout-merge.sh inventory --context <kube-context> [options]
+ host-layout-merge.sh verify --context <kube-context> [options]
+ host-layout-merge.sh check-script [--root <path>]
+
+inventory before an upgrade: what the merge init will do to every Host PVC.
+ Exit 1 when a tree will be refused (UPGRADE_BLOCKERS > 0), the
+ flag is "false" while a tree is split, the flag is invalid, or a
+ PVC could not be read.
+verify whether CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE can be set to
+ "false": exit 0 only when every Host Deployment renders the merge
+ init and no Host PVC still carries the split layout.
+check-script print the check the PVC check Pods run.
+
+options:
+ --hcc-namespace <ns> HCC namespace (default control-plane)
+ --hcc-deployment <name> HCC Deployment (default host-context-controller)
+ --no-pvc-pods start no check Pod; PVC results are UNVERIFIED
+ --check-timeout <sec> wait per check Pod (default 120)
+ --root <path> check-script only: PVC root (default /mnt/workspace-root)
+
+Reads only, except one short-lived Pod per inspected PVC: claim mounted
+read-only, uid/gid 1001, no service-account token, deleted after its log is
+read. Every kubectl call names --context; the current-context is never used.
+Exit: 0 pass, 1 an observed blocker, 2 usage, read failure, empty scope or
+check Pod cleanup failure.
+EOF
+}
+
+KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
+MERGE_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/workspace-layout-merge.sh"
+PVC_ROOT=/mnt/workspace-root
+
+die() {
+ echo "host-layout-merge: $*" >&2
+ exit 2
+}
+usage_error() {
+ echo "host-layout-merge: $*" >&2
+ usage >&2
+ exit 2
+}
+
+# ─── check script ─────────────────────────────────────────────────────
+# Prints the merge script with:
+# - the root replaced (check-script --root, used by the tests);
+# - each refusal printed on stdout as WILL_REFUSE_<section> <line>, where
+# the section is the "# F<n>" block it ran in (F1 before the first);
+# - each "no signature" exit printed as NO_SPLIT_SIGNATURE;
+# - `[ -w ]` replaced by a mode-bit test, since the check mount is read-only;
+# - everything from the move function on replaced by WILL_MERGE.
+# Each rewrite must match the expected number of lines, so a changed merge
+# script stops here instead of producing a check that no longer matches it.
+# The patterns and replacements below are shell text for the check Pod, so
+# they are single-quoted on purpose.
+# shellcheck disable=SC2016
+print_check_script() {
+ local root=$1 line roots=0 sets=0 echoes=0 exits=0 sections=0 noops=0 writes=0 cut=0
+ case "$root" in *"'"*) usage_error "--root must not contain a single quote" ;; esac
+ [ -f "$MERGE_SCRIPT" ] || die "missing $MERGE_SCRIPT"
+ while IFS= read -r line || [ -n "$line" ]; do
+ case "$line" in
+ "root='/mnt/workspace-root'")
+ printf "root='%s'\n" "$root"
+ roots=$((roots + 1))
+ ;;
+ 'set -eu')
+ printf '%s\n' "$line" 'phase=F1'
+ cat <<'EOF'
+# Mode-bit write test: the check Pod mounts the PVC read-only, so the
+# shell's -w test would refuse every tree. Same owner/group/other rule as
+# access(2).
+writable() (
+ set -f
+ [ "$(id -u)" -ne 0 ] || exit 0
+ listing="$(ls -ldn "$1")" || exit 1
+ set -- $listing
+ if [ "$3" = "$(id -u)" ]; then
+ case "$1" in ??w*) exit 0 ;; *) exit 1 ;; esac
+ fi
+ for gid in $(id -G); do
+ if [ "$4" = "$gid" ]; then
+ case "$1" in ?????w*) exit 0 ;; *) exit 1 ;; esac
+ fi
+ done
+ case "$1" in ????????w*) exit 0 ;; *) exit 1 ;; esac
+)
+EOF
+ sets=$((sets + 1))
+ ;;
+ ' echo "workspace layout merge refused: $1" >&2')
+ printf '%s\n' ' echo "WILL_REFUSE_$phase workspace layout merge refused: $1"'
+ echoes=$((echoes + 1))
+ ;;
+ ' exit 1')
+ printf '%s\n' ' exit 0'
+ exits=$((exits + 1))
+ ;;
+ '# F1' | '# F2' | '# F3' | '# F4')
+ printf 'phase=%s\n' "${line#'# '}"
+ sections=$((sections + 1))
+ ;;
+ *'|| exit 0')
+ printf '%s\n' "${line%'|| exit 0'}|| { echo NO_SPLIT_SIGNATURE; exit 0; }"
+ noops=$((noops + 1))
+ ;;
+ ' [ -w "$dir" ] || fail_merge "not writable by uid $(id -u): $dir"')
+ printf '%s\n' ' writable "$dir" || fail_merge "not writable by uid $(id -u): $dir"'
+ writes=$((writes + 1))
+ ;;
+ ' [ -w "$entry" ] || fail_merge "not writable by uid $(id -u): $entry"')
+ printf '%s\n' ' writable "$entry" || fail_merge "not writable by uid $(id -u): $entry"'
+ writes=$((writes + 1))
+ ;;
+ 'move_to_root() {')
+ printf '%s\n' 'echo WILL_MERGE'
+ cut=1
+ break
+ ;;
+ *)
+ printf '%s\n' "$line"
+ ;;
+ esac
+ done <"$MERGE_SCRIPT"
+ local shape="root=$roots set=$sets echo=$echoes exit=$exits sections=$sections noops=$noops writes=$writes cut=$cut"
+ [ "$shape" = "root=1 set=1 echo=1 exit=1 sections=4 noops=3 writes=2 cut=1" ] ||
+ die "$MERGE_SCRIPT has an unexpected shape ($shape); update print_check_script with it"
+}
+
+# ─── arguments ────────────────────────────────────────────────────────
+COMMAND="${1:-}"
+[ "$#" -eq 0 ] || shift
+CONTEXT=""
+HCC_NAMESPACE=control-plane
+HCC_DEPLOYMENT='host-context-controller'
+NO_PVC_PODS=0
+CHECK_TIMEOUT=120
+CHECK_ROOT="$PVC_ROOT"
+need_value() { [ "$2" -ge 2 ] || usage_error "$1 needs a value"; }
+while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --context) need_value "$1" "$#"; CONTEXT="$2"; shift 2 ;;
+ --context=*) CONTEXT="${1#--context=}"; shift ;;
+ --hcc-namespace) need_value "$1" "$#"; HCC_NAMESPACE="$2"; shift 2 ;;
+ --hcc-deployment) need_value "$1" "$#"; HCC_DEPLOYMENT="$2"; shift 2 ;;
+ --no-pvc-pods) NO_PVC_PODS=1; shift ;;
+ --check-timeout) need_value "$1" "$#"; CHECK_TIMEOUT="$2"; shift 2 ;;
+ --root) need_value "$1" "$#"; CHECK_ROOT="$2"; shift 2 ;;
+ -h | --help) usage; exit 0 ;;
+ *) usage_error "unknown argument: $1" ;;
+ esac
+done
+
+case "$COMMAND" in
+ check-script)
+ print_check_script "$CHECK_ROOT"
+ exit 0
+ ;;
+ inventory | verify) ;;
+ *) usage_error "expected inventory, verify or check-script" ;;
+esac
+[ -n "$CONTEXT" ] || usage_error "--context is required; the current-context is never used"
+[ "$CHECK_ROOT" = "$PVC_ROOT" ] || usage_error "--root applies to check-script only"
+case "$CHECK_TIMEOUT" in '' | *[!0-9]*) usage_error "--check-timeout must be a whole number of seconds" ;; esac
+command -v jq >/dev/null 2>&1 || die "jq is required"
+CHECK_SCRIPT="$(print_check_script "$PVC_ROOT")"
+
+kc() { "$KUBECTL_BIN" --context="$CONTEXT" "$@"; }
+
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/host-layout-merge.XXXXXX")"
+CHECK_PODS="$WORK/check-pods"
+: >"$CHECK_PODS"
+HOST_NS=""
+
+# A check Pod left behind keeps an RWO claim attached to its node, which can
+# stop the Host from starting elsewhere, so a failed delete is exit 2.
+# shellcheck disable=SC2329 # invoked by the EXIT trap
+cleanup() {
+ local status=$? pod
+ while IFS= read -r pod; do
+ [ -n "$pod" ] || continue
+ if ! kc delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then
+ echo "host-layout-merge: check Pod $HOST_NS/$pod was not deleted; delete it by hand" >&2
+ status=2
+ fi
+ done <"$CHECK_PODS"
+ rm -rf "$WORK"
+ exit "$status"
+}
+trap cleanup EXIT
+trap 'exit 2' INT TERM
+
+# read_json <file> <kubectl args...>: a JSON object or exit 2 with stderr.
+read_json() {
+ local out=$1
+ shift
+ if ! kc "$@" >"$out" 2>"$out.err"; then
+ die "kubectl $* failed: $(cat "$out.err")"
+ fi
+ jq -e 'type == "object"' "$out" >/dev/null 2>&1 || die "kubectl $* did not return a JSON object"
+}
+
+# ─── HCC ──────────────────────────────────────────────────────────────
+read_json "$WORK/hcc.json" get deployment "$HCC_DEPLOYMENT" -n "$HCC_NAMESPACE" -o json
+jq -e '[.spec.template.spec.containers[]? | select(.name == "host-context-controller")] | length == 1' \
+ "$WORK/hcc.json" >/dev/null || die "$HCC_NAMESPACE/$HCC_DEPLOYMENT has no container named host-context-controller"
+jq -e '(.spec.replicas // 1) as $r | $r > 0 and (.status.readyReplicas // 0) == $r and (.status.updatedReplicas // 0) == $r' \
+ "$WORK/hcc.json" >/dev/null || die "HCC_NOT_READY: $HCC_NAMESPACE/$HCC_DEPLOYMENT is not fully rolled out and Ready"
+
+# hcc_env <name>: <unset>, <valueFrom> or the literal value.
+hcc_env() {
+ jq -r --arg n "$1" '[.spec.template.spec.containers[] | select(.name == "host-context-controller") | .env[]? | select(.name == $n)]
+ | if length == 0 then "<unset>" elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end' "$WORK/hcc.json"
+}
+
+FLAG_RAW="$(hcc_env CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE)"
+case "$FLAG_RAW" in
+ '<unset>' | '' | true) FLAG=on ;;
+ false) FLAG=off ;;
+ '<valueFrom>') FLAG=unresolved ;;
+ *) FLAG=invalid ;;
+esac
+HOST_NS="$(hcc_env CONTEXT_MAPPER_HOST_NAMESPACE)"
+case "$HOST_NS" in
+ '<unset>') HOST_NS=mcp-host ;;
+ '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_NAMESPACE ($HOST_NS) from the HCC Deployment" ;;
+esac
+echo "HCC_IMAGE=$(jq -r '.spec.template.spec.containers[] | select(.name == "host-context-controller") | .image // "<none>"' "$WORK/hcc.json")"
+echo "HCC_FLAG=$FLAG_RAW"
+echo "HOST_NAMESPACE=$HOST_NS"
+
+# ─── Hosts, Deployments, Pods, PVCs ───────────────────────────────────
+read_json "$WORK/hosts.json" get hosts.clerum.io -n "$HOST_NS" -o json
+read_json "$WORK/deployments.json" get deployments -n "$HOST_NS" -l clerum.io/host -o json
+read_json "$WORK/pods.json" get pods -n "$HOST_NS" -l clerum.io/host -o json
+read_json "$WORK/pvcs.json" get pvc -n "$HOST_NS" -l clerum.io/managed-by=host-context-controller -o json
+
+# One tab-separated row per Host (H) and per Host PVC (P), "-" for empty.
+# A Host Deployment and its Pods carry app == clerum.io/host == <host>; the
+# per-Host channel-reader carries app=channel-reader and is excluded.
+jq -r -n \
+ --slurpfile hosts "$WORK/hosts.json" \
+ --slurpfile deps "$WORK/deployments.json" \
+ --slurpfile pods "$WORK/pods.json" \
+ --slurpfile pvcs "$WORK/pvcs.json" '
+ def owned($h): select(.metadata.labels.app == $h and .metadata.labels["clerum.io/host"] == $h);
+ def kind: if (.spec.desktop.browser // false) or (.spec.desktop.x11 // false) then "DESKTOP"
+ elif (.spec.lifecycle.stateless // false) then "STATELESS" else "STATEFUL" end;
+ def template: [.spec.template.spec.initContainers[]?.name] as $i
+ | if any($i[]; . == "workspace-layout-init") then "LEGACY_INIT"
+ elif any($i[]; . == "workspace-layout-merge") then "MERGE" else "NO_INIT" end;
+ def subpath($claim): .spec.template.spec as $s
+ | [$s.volumes[]? | select(.persistentVolumeClaim.claimName == $claim) | .name] as $v
+ | if any(($s.containers + ($s.initContainers // []))[] | .volumeMounts[]?;
+ (.name as $n | $v | index($n)) != null and (.subPath // "") != "") then "yes" else "no" end;
+ def merge_status: [.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge")][0] as $s
+ | if $s == null then "UNKNOWN"
+ elif $s.state.terminated.exitCode == 0 then "POD_OK"
+ elif $s.state.terminated.exitCode == 1 then "MERGE_REFUSED:current"
+ elif $s.lastState.terminated.exitCode == 1 then "MERGE_REFUSED:previous"
+ elif $s.state.terminated != null then "UNKNOWN"
+ elif $s.state.running != null or $s.state.waiting != null then "MERGE_RUNNING"
+ else "UNKNOWN" end;
+ def worst: (map({"MERGE_REFUSED": 0, "UNKNOWN": 1, "MERGE_RUNNING": 2, "POD_OK": 3}[split(":")[0]]) | min) as $m
+ | map(select({"MERGE_REFUSED": 0, "UNKNOWN": 1, "MERGE_RUNNING": 2, "POD_OK": 3}[split(":")[0]] == $m))[0];
+ def dash: if . == null or . == "" then "-" else . end;
+ def host_facts($h):
+ [$deps[0].items[] | owned($h)] as $d
+ | if ($d | length) > 1 then error("more than one Deployment for Host \($h)") else . end
+ | [$pods[0].items[] | owned($h) | select(.metadata.deletionTimestamp == null)] as $p
+ | ($d[0] // null) as $dep
+ | ($dep | if . == null then null else [.spec.template.spec.containers[]? | select(.name == "mcp-host")][0] end) as $main
+ | if $dep != null and $main == null then error("Deployment \($h) has no mcp-host container") else . end
+ | {
+ state: (if $dep == null then "NO_DEPLOYMENT" elif ($dep.spec.replicas // 1) == 0 then "SUSPENDED" else "ACTIVE" end),
+ template: (if $dep == null then "-" else ($dep | template) end),
+ subpath: (if $dep == null then "-" else ($dep | subpath("\($h)-workspace")) end),
+ pods: ($p | length),
+ nodes: ([$p[].spec.nodeName | select(. != null)] | unique),
+ merge: (if ($p | length) == 0 then "-" else ($p | map({s: merge_status, pod: .metadata.name}) | (map(.s) | worst) as $w | map(select(.s == $w))[0] | "\(.s):\(.pod)") end),
+ image: ($main.image // null),
+ policy: ($main.imagePullPolicy // null),
+ secrets: ($dep.spec.template.spec.imagePullSecrets // [] | map(.name) | join(","))
+ };
+ ($hosts[0].items | map({key: .metadata.name, value: kind}) | from_entries) as $kinds
+ | ($hosts[0].items[] | .metadata.name as $h | host_facts($h) as $f
+ | ["H", $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash)] | @tsv),
+ ($pvcs[0].items[] | .metadata.labels["clerum.io/host"] as $h
+ | if $h == null or .metadata.name != "\($h)-workspace" then
+ ["I", .metadata.name] | @tsv
+ elif $kinds[$h] == null then
+ ["P", .metadata.name, $h, "ORPHAN", "NO_HOST", "-", "-", "0", "-", "-", "-", "-", "-"] | @tsv
+ else
+ host_facts($h) as $f
+ | ["P", .metadata.name, $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash),
+ ($f.pods | tostring), ($f.nodes | join(",") | dash), ($f.merge | dash),
+ ($f.image | dash), ($f.policy | dash), ($f.secrets | dash)] | @tsv
+ end)
+' >"$WORK/rows.tsv" 2>"$WORK/rows.err" || die "cannot classify the Host objects: $(cat "$WORK/rows.err")"
+
+HOSTS=0
+PRE_825=no
+DESKTOP_HOSTS=0
+TEMPLATE_STALE=0
+while IFS=$'\t' read -r type name kind state template subpath; do
+ [ "$type" = H ] || continue
+ HOSTS=$((HOSTS + 1))
+ echo "HOST $name kind=$kind state=$state template=$template subPath=$subpath"
+ [ "$kind" != DESKTOP ] || DESKTOP_HOSTS=$((DESKTOP_HOSTS + 1))
+ if [ "$template" = LEGACY_INIT ] || [ "$subpath" = yes ]; then PRE_825=yes; fi
+ if [ "$state" != NO_DEPLOYMENT ] && [ "$template" != MERGE ]; then TEMPLATE_STALE=$((TEMPLATE_STALE + 1)); fi
+done <"$WORK/rows.tsv"
+while IFS=$'\t' read -r type name _; do
+ [ "$type" = I ] || continue
+ echo "IGNORED_PVC $HOST_NS/$name (not <host>-workspace of a clerum.io/host label)"
+done <"$WORK/rows.tsv"
+PVC_ROWS="$(awk -F'\t' '$1 == "P"' "$WORK/rows.tsv" | wc -l | tr -d ' ')"
+[ "$HOSTS" -gt 0 ] || [ "$PVC_ROWS" -gt 0 ] || die "no Host and no Host PVC in namespace $HOST_NS; wrong --context or namespace?"
+echo "PRE_825=$PRE_825"
+
+# ─── verdicts that need no PVC read ───────────────────────────────────
+if [ "$COMMAND" = verify ]; then
+ case "$FLAG" in
+ invalid) echo "HOST_LAYOUT_VERIFY=FLAG_INVALID"; exit 1 ;;
+ unresolved) echo "HOST_LAYOUT_VERIFY=FLAG_UNRESOLVED"; exit 1 ;;
+ esac
+ if [ "$PRE_825" = yes ]; then
+ echo "a Host Deployment still renders workspace-layout-init or a workspace subPath: run inventory and finish the upgrade first"
+ echo "HOST_LAYOUT_VERIFY=PRE_825"
+ exit 1
+ fi
+fi
+
+# ─── check Pods ───────────────────────────────────────────────────────
+# run_check <pvc> <node|-> <image> <policy> <secrets|->: prints the result
+# token (and refusal line) for that PVC.
+run_check() {
+ local claim=$1 node=$2 image=$3 policy=$4 secrets=$5 pod phase elapsed=0 first lines
+ pod="$(printf 'host-layout-check-%s' "$claim" | cut -c1-63 | sed 's/[-.]*$//')"
+ jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$image" --arg policy "$policy" \
+ --arg secrets "$secrets" --arg claim "$claim" --arg script "$CHECK_SCRIPT" --arg root "$PVC_ROOT" '{
+ apiVersion: "v1", kind: "Pod",
+ metadata: {name: $pod, namespace: $ns, labels: {purpose: "host-layout-merge-check"}},
+ spec: ({
+ restartPolicy: "Never", automountServiceAccountToken: false, enableServiceLinks: false,
+ terminationGracePeriodSeconds: 0,
+ securityContext: {runAsUser: 1001, runAsGroup: 1001, runAsNonRoot: true, seccompProfile: {type: "RuntimeDefault"}},
+ imagePullSecrets: (if $secrets == "-" then [] else ($secrets | split(",") | map({name: .})) end),
+ containers: [{
+ name: "check", image: $image, imagePullPolicy: $policy,
+ command: ["/bin/sh", "-c", $script],
+ resources: {requests: {cpu: "10m", memory: "32Mi"}, limits: {cpu: "100m", memory: "64Mi"}},
+ securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: ["ALL"]}},
+ volumeMounts: [{name: "pvc", mountPath: $root, readOnly: true}]
+ }],
+ volumes: [{name: "pvc", persistentVolumeClaim: {claimName: $claim, readOnly: true}}]
+ } + (if $node == "-" then {} else {nodeName: $node} end))
+ }' >"$WORK/pod.json"
+ # Registered before the create, so a create that times out after the API
+ # accepted it is still deleted.
+ printf '%s\n' "$pod" >>"$CHECK_PODS"
+ kc create -f - <"$WORK/pod.json" >/dev/null 2>"$WORK/create.err" ||
+ die "check Pod $HOST_NS/$pod create failed: $(cat "$WORK/create.err")"
+ while :; do
+ read_json "$WORK/status.json" get pod "$pod" -n "$HOST_NS" -o json
+ phase="$(jq -r '.status.phase // ""' "$WORK/status.json")"
+ case "$phase" in Succeeded | Failed) break ;; esac
+ [ "$elapsed" -lt "$CHECK_TIMEOUT" ] || break
+ sleep 2
+ elapsed=$((elapsed + 2))
+ done
+ : >"$WORK/check.log"
+ if [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; then
+ kc logs "$pod" -n "$HOST_NS" >"$WORK/check.log" 2>&1 || die "cannot read the log of check Pod $HOST_NS/$pod: $(cat "$WORK/check.log")"
+ fi
+ kc delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>"$WORK/delete.err" ||
+ die "check Pod $HOST_NS/$pod delete failed: $(cat "$WORK/delete.err")"
+ grep -vxF -- "$pod" "$CHECK_PODS" >"$CHECK_PODS.next" || true
+ mv "$CHECK_PODS.next" "$CHECK_PODS"
+
+ if [ "$phase" != Succeeded ]; then
+ echo "UNREADABLE phase=${phase:-<none>} $(jq -r '[.status.containerStatuses[]?.state.waiting.reason // empty] | join(",")' "$WORK/status.json") $(tail -n 2 "$WORK/check.log" | tr '\n' ' ')"
+ return
+ fi
+ first="$(head -n 1 "$WORK/check.log")"
+ lines="$(wc -l <"$WORK/check.log" | tr -d ' ')"
+ case "$first" in
+ WILL_MERGE | NO_SPLIT_SIGNATURE)
+ if [ "$lines" = 1 ]; then echo "$first"; else echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")"; fi
+ ;;
+ 'WILL_REFUSE_F'[1-4]' workspace layout merge refused: '*) tr '\n' ' ' <"$WORK/check.log" | sed 's/ $//'; echo ;;
+ *) echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")" ;;
+ esac
+}
+
+# The image for a PVC without a Host Deployment comes from HCC itself.
+hcc_host_image() {
+ local image policy secret
+ image="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE)"
+ policy="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY)"
+ secret="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET)"
+ case "$image" in '' | '<unset>' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE ($image) for a check Pod" ;; esac
+ case "$policy" in '<unset>') policy=Always ;; '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY ($policy)" ;; esac
+ case "$secret" in '<unset>') secret=clerum ;; '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET ($secret)" ;; esac
+ printf '%s\t%s\t%s\n' "$image" "$policy" "$secret"
+}
+
+# merge_line <status:pod>: the refusal line from the merge init's log.
+merge_line() {
+ local which=${1%%:*} pod=${1#*:} args=()
+ [ "$which" = previous ] && args=(--previous)
+ kc logs "$pod" -n "$HOST_NS" -c workspace-layout-merge ${args[@]+"${args[@]}"} >"$WORK/merge.log" 2>&1 ||
+ die "cannot read the workspace-layout-merge log of $HOST_NS/$pod: $(cat "$WORK/merge.log")"
+ grep -m 1 '^workspace layout merge refused: ' "$WORK/merge.log" || echo "<no refusal line in the log>"
+}
+
+TOTAL=0 POD_OK=0 NO_SPLIT=0 SPLIT=0 REFUSED=0 RUNNING=0 UNREADABLE=0 UNVERIFIED=0 UNKNOWN=0
+ORPHANS=0 ORPHAN_SPLIT=0 UPGRADE_BLOCKERS=0 SPLIT_TREES=0
+while IFS=$'\t' read -r type pvc host kind state _template _subpath pods nodes merge image policy secrets; do
+ [ "$type" = P ] || continue
+ result=""
+ if [ "$COMMAND" = verify ] && [ "$FLAG" = on ] && [ "$pods" -gt 0 ]; then
+ case "${merge%%:*}" in
+ POD_OK | MERGE_RUNNING | UNKNOWN) result="${merge%%:*}" ;;
+ MERGE_REFUSED) result="MERGE_REFUSED $(merge_line "${merge#MERGE_REFUSED:}")" ;;
+ *) die "unexpected merge status $merge for $pvc" ;;
+ esac
+ elif [ "$NO_PVC_PODS" -eq 1 ]; then
+ result=UNVERIFIED
+ else
+ node=-
+ if [ "$pods" -gt 0 ]; then
+ case "$nodes" in
+ - ) node=- ;;
+ *,*) result="UNREADABLE Host Pods on more than one node ($nodes)" ;;
+ *) node="$nodes" ;;
+ esac
+ fi
+ if [ -z "$result" ]; then
+ if [ "$image" = - ]; then
+ IFS=$'\t' read -r image policy secrets <<<"$(hcc_host_image)"
+ fi
+ result="$(run_check "$pvc" "$node" "$image" "$policy" "$secrets" </dev/null)"
+ # verify names a tree the merge will still change SPLIT.
+ [ "$COMMAND" = inventory ] || [ "$result" != WILL_MERGE ] || result=SPLIT
+ fi
+ fi
+ if [ "$kind" = ORPHAN ]; then
+ echo "PVC $HOST_NS/$pvc host=$host kind=ORPHAN state=$state result=$result"
+ ORPHANS=$((ORPHANS + 1))
+ case "$result" in WILL_MERGE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;; esac
+ continue
+ fi
+ echo "PVC $HOST_NS/$pvc host=$host kind=$kind state=$state result=$result"
+ TOTAL=$((TOTAL + 1))
+ case "$result" in
+ POD_OK) POD_OK=$((POD_OK + 1)) ;;
+ NO_SPLIT_SIGNATURE) NO_SPLIT=$((NO_SPLIT + 1)) ;;
+ WILL_MERGE | SPLIT) SPLIT=$((SPLIT + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)) ;;
+ WILL_REFUSE_*) REFUSED=$((REFUSED + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)); UPGRADE_BLOCKERS=$((UPGRADE_BLOCKERS + 1)) ;;
+ MERGE_REFUSED*) REFUSED=$((REFUSED + 1)) ;;
+ MERGE_RUNNING) RUNNING=$((RUNNING + 1)) ;;
+ UNREADABLE*) UNREADABLE=$((UNREADABLE + 1)) ;;
+ UNVERIFIED) UNVERIFIED=$((UNVERIFIED + 1)) ;;
+ UNKNOWN) UNKNOWN=$((UNKNOWN + 1)) ;;
+ *) die "unclassified result for $pvc: $result" ;;
+ esac
+done <"$WORK/rows.tsv"
+
+SUM=$((POD_OK + NO_SPLIT + SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED + UNKNOWN))
+[ "$SUM" -eq "$TOTAL" ] || die "counted $SUM results for $TOTAL Host PVCs"
+echo "COUNTS total=$TOTAL pod_ok=$POD_OK no_split=$NO_SPLIT split=$SPLIT refused=$REFUSED running=$RUNNING unreadable=$UNREADABLE unverified=$UNVERIFIED unknown=$UNKNOWN template_stale=$TEMPLATE_STALE"
+echo "ORPHANS=$ORPHANS ORPHAN_SPLIT=$ORPHAN_SPLIT"
+if [ "$ORPHAN_SPLIT" -gt 0 ]; then
+ echo "note: a Host recreated with an orphan's name after the flag is off starts on an empty database; delete the orphan PVC first or recreate the Host before turning the flag off"
+fi
+echo "PVC_CONTENTS=NOT_INSPECTED"
+if [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; then
+ echo "FLAG_FALSE_WITH_SPLIT_TREES=$SPLIT_TREES"
+fi
+
+if [ "$COMMAND" = inventory ]; then
+ echo "UPGRADE_BLOCKERS=$UPGRADE_BLOCKERS"
+ if [ "$DESKTOP_HOSTS" -gt 0 ]; then
+ echo "DESKTOP_HOSTS=$DESKTOP_HOSTS export /workspace/state.db* from each desktop Host Pod before the first rollout of this HCC (docs/deploy/host-pvc-layout-upgrade.md)"
+ else
+ echo "DESKTOP_HOSTS=0"
+ fi
+ if [ "$UPGRADE_BLOCKERS" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] ||
+ { [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; } ||
+ [ "$FLAG" = invalid ] || [ "$FLAG" = unresolved ]; then
+ echo "HOST_LAYOUT_INVENTORY=BLOCKED"
+ exit 1
+ fi
+ echo "HOST_LAYOUT_INVENTORY=OK"
+ exit 0
+fi
+
+if [ "$FLAG" = off ]; then
+ if [ "$SPLIT_TREES" -gt 0 ]; then
+ echo "HOST_LAYOUT_VERIFY=FLAG_FALSE_WITH_SPLIT_TREES"
+ else
+ echo "HOST_LAYOUT_VERIFY=ALREADY_OFF"
+ fi
+ exit 1
+fi
+if [ "$TOTAL" -gt 0 ] && [ "$((SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED + UNKNOWN + TEMPLATE_STALE))" -eq 0 ]; then
+ echo "HOST_LAYOUT_VERIFY=PASS"
+ exit 0
+fi
+echo "HOST_LAYOUT_VERIFY=FAIL"
+exit 1
diff --git a/deploy/scripts/host-layout-merge.test.mjs b/deploy/scripts/host-layout-merge.test.mjs
new file mode 100644
index 000000000..68688f376
--- /dev/null
+++ b/deploy/scripts/host-layout-merge.test.mjs
@@ -0,0 +1,531 @@
+import assert from 'node:assert/strict'
+import { spawnSync } from 'node:child_process'
+import {
+ chmodSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync,
+} from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+
+// Runs host-layout-merge.sh against a fake kubectl that answers from canned
+// JSON and records every call, so the inventory, the turn-off gate and the
+// check Pods are exercised without a cluster. All names are synthetic.
+
+const SCRIPT = fileURLToPath(new URL('./host-layout-merge.sh', import.meta.url))
+const CONTEXT = 'test-context'
+const NS = 'mcp-host'
+
+const FAKE_KUBECTL = `#!/usr/bin/env node
+const fs = require('fs')
+const fx = JSON.parse(fs.readFileSync(process.env.FAKE_KUBECTL_FIXTURE, 'utf8'))
+const args = process.argv.slice(2)
+const stdin = args.includes('-f') ? fs.readFileSync(0, 'utf8') : ''
+fs.appendFileSync(process.env.FAKE_KUBECTL_LOG, JSON.stringify({ args, stdin }) + '\\n')
+const out = (s, code = 0) => { process.stdout.write(s); process.exit(code) }
+if (!args.includes('--context=' + fx.context)) { process.stderr.write('missing --context'); process.exit(97) }
+const rest = args.filter(a => !a.startsWith('--context=')).join(' ')
+const statePath = process.env.FAKE_KUBECTL_LOG + '.pods.json'
+const pods = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, 'utf8')) : {}
+const save = () => fs.writeFileSync(statePath, JSON.stringify(pods))
+if (rest === 'create -f -') {
+ const m = JSON.parse(stdin)
+ const claim = m.spec.volumes[0].persistentVolumeClaim.claimName
+ pods[m.metadata.name] = claim
+ save()
+ out('pod/' + m.metadata.name + ' created\\n')
+}
+let m = rest.match(/^get pod (\\S+) -n \\S+ -o json$/)
+if (m && pods[m[1]]) {
+ const check = fx.checks[pods[m[1]]] || {}
+ out(JSON.stringify({ status: { phase: check.phase || 'Succeeded', containerStatuses: check.containerStatuses || [] } }))
+}
+m = rest.match(/^logs (\\S+) -n \\S+$/)
+if (m && pods[m[1]]) out(fx.checks[pods[m[1]]].output)
+m = rest.match(/^delete pod (\\S+) -n \\S+ --wait=true --timeout=60s$/)
+if (m && pods[m[1]]) {
+ if ((fx.checks[pods[m[1]]] || {}).deleteFails) { process.stderr.write('delete refused'); process.exit(1) }
+ delete pods[m[1]]
+ save()
+ out('pod "' + m[1] + '" deleted\\n')
+}
+const r = fx.responses[rest]
+if (!r) { process.stderr.write('fake kubectl: unexpected call: ' + rest); process.exit(98) }
+if (r.stderr) process.stderr.write(r.stderr)
+out(typeof r.stdout === 'string' ? r.stdout : JSON.stringify(r.stdout), r.status || 0)
+`
+
+function hcc(env = []) {
+ return {
+ spec: {
+ replicas: 1,
+ template: {
+ spec: {
+ containers: [
+ {
+ name: 'host-context-controller',
+ env: [
+ { name: 'CONTEXT_MAPPER_HOST_IMAGE', value: 'registry.test/mcp-host:head' },
+ ...env,
+ ],
+ },
+ ],
+ },
+ },
+ },
+ status: { readyReplicas: 1, updatedReplicas: 1 },
+ }
+}
+
+function hostCr(name, spec = {}) {
+ return { metadata: { name, namespace: NS }, spec: { contextRef: 'ctx', ...spec } }
+}
+
+function hostDeployment(name, { replicas = 1, inits = ['workspace-layout-merge'], subPath, containers } = {}) {
+ const mount = { name: 'workspace', mountPath: '/workspace', ...(subPath ? { subPath } : {}) }
+ return {
+ metadata: { name, labels: { app: name, 'clerum.io/host': name } },
+ spec: {
+ replicas,
+ template: {
+ spec: {
+ initContainers: inits.map(n => ({ name: n, volumeMounts: [{ name: 'workspace', mountPath: '/mnt/workspace-root' }] })),
+ containers: containers ?? [
+ { name: 'mcp-host', image: `registry.test/mcp-host:${name}`, imagePullPolicy: 'IfNotPresent', volumeMounts: [mount] },
+ ],
+ imagePullSecrets: [{ name: 'pull-secret' }],
+ volumes: [{ name: 'workspace', persistentVolumeClaim: { claimName: `${name}-workspace` } }],
+ },
+ },
+ },
+ }
+}
+
+function hostPod(host, node, mergeStatus) {
+ return {
+ metadata: { name: `${host}-pod`, labels: { app: host, 'clerum.io/host': host } },
+ spec: { nodeName: node },
+ status: {
+ phase: 'Running',
+ initContainerStatuses: mergeStatus === undefined ? [] : [{ name: 'workspace-layout-merge', ...mergeStatus }],
+ },
+ }
+}
+
+function pvc(host) {
+ return {
+ metadata: {
+ name: `${host}-workspace`,
+ labels: { 'clerum.io/managed-by': 'host-context-controller', 'clerum.io/host': host },
+ },
+ }
+}
+
+const EXITED_0 = { state: { terminated: { exitCode: 0 } } }
+const F2_LINE = 'workspace layout merge refused: more than one live SQLite database across /mnt/workspace-root, /mnt/workspace-root/state and /mnt/workspace-root/workspace'
+
+// A stateful active Host, a suspended stateless Host and a desktop Host whose
+// mcp-host container is not the first one, plus a channel-reader Deployment
+// and Pod that carry a Host label and must be ignored.
+function fleet() {
+ return {
+ hcc: hcc(),
+ hosts: [
+ hostCr('alpha'),
+ hostCr('beta', { lifecycle: { stateless: true } }),
+ hostCr('gamma', { desktop: { browser: true } }),
+ ],
+ deployments: [
+ hostDeployment('alpha'),
+ hostDeployment('beta', { replicas: 0 }),
+ hostDeployment('gamma', {
+ containers: [
+ { name: 'x11', image: 'registry.test/x11:1' },
+ { name: 'mcp-host', image: 'registry.test/mcp-host-desktop:gamma', imagePullPolicy: 'Always', volumeMounts: [{ name: 'workspace', mountPath: '/config/workspace' }] },
+ ],
+ }),
+ { metadata: { name: 'channel-reader-alpha', labels: { app: 'channel-reader', 'clerum.io/host': 'alpha' } }, spec: { replicas: 1, template: { spec: { containers: [{ name: 'reader' }] } } } },
+ ],
+ pods: [
+ hostPod('alpha', 'node-a', EXITED_0),
+ hostPod('gamma', 'node-b', EXITED_0),
+ { metadata: { name: 'channel-reader-alpha-pod', labels: { app: 'channel-reader', 'clerum.io/host': 'alpha' } }, spec: { nodeName: 'node-c' }, status: { phase: 'Running' } },
+ ],
+ pvcs: [pvc('alpha'), pvc('beta'), pvc('gamma')],
+ checks: {
+ 'alpha-workspace': { output: 'NO_SPLIT_SIGNATURE\n' },
+ 'beta-workspace': { output: 'NO_SPLIT_SIGNATURE\n' },
+ 'gamma-workspace': { output: 'NO_SPLIT_SIGNATURE\n' },
+ },
+ extra: {},
+ }
+}
+
+function responses(f) {
+ const list = items => ({ stdout: { items } })
+ return {
+ 'get deployment host-context-controller -n control-plane -o json': { stdout: f.hcc },
+ [`get hosts.clerum.io -n ${NS} -o json`]: list(f.hosts),
+ [`get deployments -n ${NS} -l clerum.io/host -o json`]: list(f.deployments),
+ [`get pods -n ${NS} -l clerum.io/host -o json`]: list(f.pods),
+ [`get pvc -n ${NS} -l clerum.io/managed-by=host-context-controller -o json`]: list(f.pvcs),
+ ...f.extra,
+ }
+}
+
+function run(f, args) {
+ const dir = mkdtempSync(path.join(tmpdir(), 'host-layout-merge-test-'))
+ try {
+ const kubectl = path.join(dir, 'kubectl')
+ writeFileSync(kubectl, FAKE_KUBECTL, { mode: 0o755 })
+ const fixture = path.join(dir, 'fixture.json')
+ writeFileSync(fixture, JSON.stringify({ context: CONTEXT, responses: responses(f), checks: f.checks }))
+ const log = path.join(dir, 'calls.jsonl')
+ writeFileSync(log, '')
+ const res = spawnSync('bash', [SCRIPT, ...args], {
+ encoding: 'utf8',
+ env: { ...process.env, KUBECTL_BIN: kubectl, FAKE_KUBECTL_FIXTURE: fixture, FAKE_KUBECTL_LOG: log },
+ })
+ if (res.error) throw res.error
+ const calls = readFileSync(log, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l))
+ return { status: res.status, stdout: res.stdout, stderr: res.stderr, calls }
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+}
+
+const verify = (f, ...extra) => run(f, ['verify', '--context', CONTEXT, ...extra])
+const inventory = (f, ...extra) => run(f, ['inventory', '--context', CONTEXT, ...extra])
+
+function row(out, pvcName) {
+ const lines = out.split('\n').filter(l => l.startsWith(`PVC ${NS}/${pvcName} `))
+ assert.equal(lines.length, 1, `one row for ${pvcName} in:\n${out}`)
+ return lines[0]
+}
+
+const verbs = calls => calls.map(c => c.args.filter(a => !a.startsWith('--context='))[0])
+const creates = calls => calls.filter(c => verbs([c])[0] === 'create').map(c => JSON.parse(c.stdin))
+
+function assertReadOnlyCalls(calls) {
+ assert.ok(calls.length > 0, 'kubectl was called')
+ for (const c of calls) assert.ok(c.args.includes(`--context=${CONTEXT}`), `call without --context: ${c.args.join(' ')}`)
+ for (const verb of verbs(calls)) assert.ok(['get', 'logs'].includes(verb), `mutating call: ${verb}`)
+}
+
+function assertCheckPodsOnly(calls, claims) {
+ for (const c of calls) assert.ok(c.args.includes(`--context=${CONTEXT}`), `call without --context: ${c.args.join(' ')}`)
+ const mutating = verbs(calls).filter(v => !['get', 'logs'].includes(v))
+ assert.deepEqual(mutating.sort(), [...claims.map(() => 'create'), ...claims.map(() => 'delete')].sort())
+ assert.deepEqual(creates(calls).map(m => m.spec.volumes[0].persistentVolumeClaim.claimName).sort(), [...claims].sort())
+}
+
+test('verify passes a merged fleet and checks only the PVC without a Pod', () => {
+ const r = verify(fleet())
+ assert.equal(r.status, 0, r.stdout + r.stderr)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)
+ assert.match(row(r.stdout, 'alpha-workspace'), /host=alpha kind=STATEFUL state=ACTIVE result=POD_OK$/)
+ assert.match(row(r.stdout, 'beta-workspace'), /host=beta kind=STATELESS state=SUSPENDED result=NO_SPLIT_SIGNATURE$/)
+ assert.match(row(r.stdout, 'gamma-workspace'), /host=gamma kind=DESKTOP state=ACTIVE result=POD_OK$/)
+ assert.match(r.stdout, /^COUNTS total=3 pod_ok=2 no_split=1 split=0 refused=0 running=0 unreadable=0 unverified=0 unknown=0 template_stale=0$/m)
+ assert.match(r.stdout, /^PVC_CONTENTS=NOT_INSPECTED$/m)
+ assertCheckPodsOnly(r.calls, ['beta-workspace'])
+
+ const [pod] = creates(r.calls)
+ assert.equal(pod.spec.nodeName, undefined)
+ assert.equal(pod.spec.automountServiceAccountToken, false)
+ assert.equal(pod.spec.securityContext.fsGroup, undefined)
+ assert.equal(pod.spec.securityContext.runAsUser, 1001)
+ assert.equal(pod.spec.securityContext.runAsGroup, 1001)
+ assert.equal(pod.spec.volumes[0].persistentVolumeClaim.readOnly, true)
+ assert.deepEqual(pod.spec.containers[0].volumeMounts, [{ name: 'pvc', mountPath: '/mnt/workspace-root', readOnly: true }])
+ assert.equal(pod.spec.containers[0].image, 'registry.test/mcp-host:beta')
+ assert.equal(pod.spec.containers[0].imagePullPolicy, 'IfNotPresent')
+ assert.deepEqual(pod.spec.imagePullSecrets, [{ name: 'pull-secret' }])
+ assert.deepEqual(pod.spec.containers[0].securityContext.capabilities, { drop: ['ALL'] })
+ assert.doesNotMatch(pod.spec.containers[0].command[2], /\bmv |\brmdir /)
+})
+
+test('inventory reads every PVC, pinning running Hosts to their node', () => {
+ const f = fleet()
+ f.checks['beta-workspace'].output = 'WILL_MERGE\n'
+ const r = inventory(f)
+ assert.equal(r.status, 0, r.stdout + r.stderr)
+ assert.match(r.stdout, /^HOST_LAYOUT_INVENTORY=OK$/m)
+ assert.match(r.stdout, /^UPGRADE_BLOCKERS=0$/m)
+ assert.match(row(r.stdout, 'beta-workspace'), /result=WILL_MERGE$/)
+ assert.match(r.stdout, /^DESKTOP_HOSTS=1 /m)
+ assert.match(r.stdout, /^PRE_825=no$/m)
+ assertCheckPodsOnly(r.calls, ['alpha-workspace', 'beta-workspace', 'gamma-workspace'])
+ const nodes = Object.fromEntries(creates(r.calls).map(m => [m.spec.volumes[0].persistentVolumeClaim.claimName, m.spec.nodeName]))
+ assert.deepEqual(nodes, { 'alpha-workspace': 'node-a', 'beta-workspace': undefined, 'gamma-workspace': 'node-b' })
+})
+
+test('inventory reports a Host switched to stateful before the upgrade as a blocker (T5)', () => {
+ const f = fleet()
+ f.checks['alpha-workspace'].output = `WILL_REFUSE_F2 ${F2_LINE}\n`
+ const r = inventory(f)
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(r.stdout, /^UPGRADE_BLOCKERS=1$/m)
+ assert.match(r.stdout, /^HOST_LAYOUT_INVENTORY=BLOCKED$/m)
+ assert.ok(row(r.stdout, 'alpha-workspace').endsWith(`result=WILL_REFUSE_F2 ${F2_LINE}`))
+})
+
+test('inventory reports a legacy fleet as PRE_825 and still reads it', () => {
+ const f = fleet()
+ f.deployments[1] = hostDeployment('beta', { replicas: 0, inits: ['workspace-layout-init'], subPath: 'workspace' })
+ f.checks['beta-workspace'].output = 'WILL_MERGE\n'
+ const r = inventory(f)
+ assert.equal(r.status, 0, r.stdout + r.stderr)
+ assert.match(r.stdout, /^PRE_825=yes$/m)
+ assert.match(r.stdout, /^HOST beta kind=STATELESS state=SUSPENDED template=LEGACY_INIT subPath=yes$/m)
+ assert.match(row(r.stdout, 'beta-workspace'), /result=WILL_MERGE$/)
+})
+
+test('--no-pvc-pods starts no Pod: inventory reports UNVERIFIED, verify fails', () => {
+ const inv = inventory(fleet(), '--no-pvc-pods')
+ assert.equal(inv.status, 0, inv.stdout + inv.stderr)
+ assert.match(row(inv.stdout, 'beta-workspace'), /result=UNVERIFIED$/)
+ assertReadOnlyCalls(inv.calls)
+
+ const ver = verify(fleet(), '--no-pvc-pods')
+ assert.equal(ver.status, 1, ver.stdout + ver.stderr)
+ assert.match(row(ver.stdout, 'alpha-workspace'), /result=POD_OK$/)
+ assert.match(row(ver.stdout, 'beta-workspace'), /result=UNVERIFIED$/)
+ assert.match(ver.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+ assertReadOnlyCalls(ver.calls)
+})
+
+test('the merge flag decides the verdict before any PVC is read', () => {
+ for (const [env, verdict] of [
+ [{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'ture' }, 'FLAG_INVALID'],
+ [{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', valueFrom: { configMapKeyRef: { name: 'c', key: 'k' } } }, 'FLAG_UNRESOLVED'],
+ ]) {
+ const f = fleet()
+ f.hcc = hcc([env])
+ const r = verify(f)
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(r.stdout, new RegExp(`^HOST_LAYOUT_VERIFY=${verdict}$`, 'm'))
+ assertReadOnlyCalls(r.calls)
+ }
+ const on = fleet()
+ on.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'true' }])
+ assert.equal(verify(on).status, 0)
+})
+
+test('flag false: ALREADY_OFF without split trees, FLAG_FALSE_WITH_SPLIT_TREES with one', () => {
+ const off = fleet()
+ off.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'false' }])
+ const clean = verify(off)
+ assert.equal(clean.status, 1, clean.stdout + clean.stderr)
+ assert.match(clean.stdout, /^HOST_LAYOUT_VERIFY=ALREADY_OFF$/m)
+ // With the flag off a running Pod has no merge init to read, so every PVC is read.
+ assertCheckPodsOnly(clean.calls, ['alpha-workspace', 'beta-workspace', 'gamma-workspace'])
+
+ off.checks['beta-workspace'].output = 'WILL_MERGE\n'
+ for (const r of [verify(off), inventory(off)]) {
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(r.stdout, /^FLAG_FALSE_WITH_SPLIT_TREES=1$/m)
+ }
+ assert.match(verify(off).stdout, /^HOST_LAYOUT_VERIFY=FLAG_FALSE_WITH_SPLIT_TREES$/m)
+})
+
+test('verify stops at PRE_825 for a legacy init or a workspace subPath', () => {
+ for (const dep of [
+ hostDeployment('alpha', { inits: ['workspace-layout-init'] }),
+ hostDeployment('alpha', { subPath: 'workspace' }),
+ ]) {
+ const f = fleet()
+ f.deployments[0] = dep
+ const r = verify(f)
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PRE_825$/m)
+ assertReadOnlyCalls(r.calls)
+ }
+})
+
+test('verify fails a Host Deployment that does not render the merge init', () => {
+ const f = fleet()
+ f.deployments[0] = hostDeployment('alpha', { inits: [] })
+ const r = verify(f)
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(r.stdout, /^HOST alpha kind=STATEFUL state=ACTIVE template=NO_INIT subPath=no$/m)
+ assert.match(r.stdout, / template_stale=1$/m)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+})
+
+test('verify reads the merge init status of running Pods', () => {
+ const cases = [
+ [{ state: { running: {} } }, 'MERGE_RUNNING', null],
+ [{ state: { terminated: { exitCode: 1 } } }, `MERGE_REFUSED ${F2_LINE}`, ''],
+ [{ state: { waiting: { reason: 'CrashLoopBackOff' } }, lastState: { terminated: { exitCode: 1 } } }, `MERGE_REFUSED ${F2_LINE}`, ' --previous'],
+ [{ state: { terminated: { exitCode: 137 } } }, 'UNKNOWN', null],
+ [undefined, 'UNKNOWN', null],
+ ]
+ for (const [status, result, logsFlag] of cases) {
+ const f = fleet()
+ f.pods[0] = hostPod('alpha', 'node-a', status)
+ if (logsFlag !== null) {
+ f.extra[`logs alpha-pod -n ${NS} -c workspace-layout-merge${logsFlag}`] = { stdout: `${F2_LINE}\n` }
+ }
+ const r = verify(f)
+ assert.equal(r.status, 1, `${result}: ${r.stdout}${r.stderr}`)
+ assert.ok(row(r.stdout, 'alpha-workspace').endsWith(`result=${result}`), row(r.stdout, 'alpha-workspace'))
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+ }
+ const recovered = fleet()
+ recovered.pods[0] = hostPod('alpha', 'node-a', { state: { terminated: { exitCode: 0 } }, lastState: { terminated: { exitCode: 1 } } })
+ const r = verify(recovered)
+ assert.equal(r.status, 0, r.stdout + r.stderr)
+ assert.match(row(r.stdout, 'alpha-workspace'), /result=POD_OK$/)
+})
+
+test('verify fails a suspended Host whose PVC is still split or unreadable', () => {
+ const cases = [
+ [{ output: 'WILL_MERGE\n' }, 'SPLIT', / split=1 /],
+ [{ output: `WILL_REFUSE_F2 ${F2_LINE}\n` }, `WILL_REFUSE_F2 ${F2_LINE}`, / refused=1 /],
+ [{ output: '' }, 'UNREADABLE', / unreadable=1 /],
+ [{ output: 'WILL_MERGE\nNO_SPLIT_SIGNATURE\n' }, 'UNREADABLE', / unreadable=1 /],
+ [{ phase: 'Failed', output: 'sh: boom\n' }, 'UNREADABLE', / unreadable=1 /],
+ [{ phase: 'Pending', containerStatuses: [{ state: { waiting: { reason: 'ImagePullBackOff' } } }] }, 'UNREADABLE', / unreadable=1 /],
+ ]
+ for (const [check, result, count] of cases) {
+ const f = fleet()
+ f.checks['beta-workspace'] = check
+ const r = verify(f, '--check-timeout', '0')
+ assert.equal(r.status, 1, `${result}: ${r.stdout}${r.stderr}`)
+ assert.ok(row(r.stdout, 'beta-workspace').includes(`result=${result}`), row(r.stdout, 'beta-workspace'))
+ assert.match(r.stdout, count)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+ // The check Pod is deleted whatever it printed.
+ assertCheckPodsOnly(r.calls, ['beta-workspace'])
+ }
+})
+
+test('an orphan split PVC is reported without blocking the verdict (T9)', () => {
+ const f = fleet()
+ f.pvcs.push(pvc('delta'))
+ f.checks['delta-workspace'] = { output: 'WILL_MERGE\n' }
+ const r = verify(f)
+ assert.equal(r.status, 0, r.stdout + r.stderr)
+ assert.match(row(r.stdout, 'delta-workspace'), /host=delta kind=ORPHAN state=NO_HOST result=SPLIT$/)
+ assert.match(r.stdout, /^ORPHANS=1 ORPHAN_SPLIT=1$/m)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)
+ const orphanPod = creates(r.calls).find(m => m.spec.volumes[0].persistentVolumeClaim.claimName === 'delta-workspace')
+ assert.equal(orphanPod.spec.containers[0].image, 'registry.test/mcp-host:head')
+ assert.equal(orphanPod.spec.containers[0].imagePullPolicy, 'Always')
+ assert.deepEqual(orphanPod.spec.imagePullSecrets, [{ name: 'clerum' }])
+})
+
+test('read failures, an empty scope and a failed cleanup exit 2', () => {
+ const empty = fleet()
+ empty.hosts = []
+ empty.deployments = []
+ empty.pods = []
+ empty.pvcs = []
+ const e = verify(empty)
+ assert.equal(e.status, 2, e.stdout + e.stderr)
+ assert.match(e.stderr, /no Host and no Host PVC/)
+
+ const failing = fleet()
+ failing.extra[`get pods -n ${NS} -l clerum.io/host -o json`] = { stdout: { items: [] }, status: 1, stderr: 'forbidden' }
+ const f = verify(failing)
+ assert.equal(f.status, 2, f.stdout + f.stderr)
+ assert.match(f.stderr, /forbidden/)
+ assert.doesNotMatch(f.stdout, /HOST_LAYOUT_VERIFY=/)
+
+ const notReady = fleet()
+ notReady.hcc.status.readyReplicas = 0
+ const n = verify(notReady)
+ assert.equal(n.status, 2, n.stdout + n.stderr)
+ assert.match(n.stderr, /HCC_NOT_READY/)
+
+ const stuck = fleet()
+ stuck.checks['beta-workspace'].deleteFails = true
+ const s = verify(stuck)
+ assert.equal(s.status, 2, s.stdout + s.stderr)
+ assert.match(s.stderr, /delete/)
+})
+
+test('usage errors exit 2 without calling kubectl', () => {
+ for (const args of [['verify'], ['inventory', '--context'], ['verify', '--context', CONTEXT, '--bogus'], ['frobnicate', '--context', CONTEXT], []]) {
+ const r = run(fleet(), args)
+ assert.equal(r.status, 2, `${args.join(' ')}: ${r.stdout}${r.stderr}`)
+ assert.equal(r.calls.length, 0)
+ }
+})
+
+// The check the Pods run, on real trees. Each tree's listing is asserted
+// before its token so a wrong fixture cannot pass as a wrong token.
+function tree(build) {
+ const root = mkdtempSync(path.join(tmpdir(), 'host-layout-check-tree-'))
+ build(root)
+ return root
+}
+
+function check(root) {
+ const printed = spawnSync('bash', [SCRIPT, 'check-script', '--root', root], { encoding: 'utf8' })
+ assert.equal(printed.status, 0, printed.stderr)
+ const res = spawnSync('/bin/sh', ['-c', printed.stdout], { encoding: 'utf8' })
+ assert.equal(res.stderr, '')
+ assert.equal(res.status, 0)
+ return res.stdout
+}
+
+const file = (p, body = 'x') => writeFileSync(p, body)
+
+test('check-script classifies split, foreign and refused trees', () => {
+ const trees = [
+ ['split', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); file(`${r}/workspace/SOUL.md`) }, ['state', 'workspace'], 'WILL_MERGE\n'],
+ ['backup only', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db.pre-1.bak`) }, ['state', 'workspace'], 'WILL_MERGE\n'],
+ ['foreign state.db name', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db.json`) }, ['state', 'workspace'], 'NO_SPLIT_SIGNATURE\n'],
+ ['symlinked state', r => { mkdirSync(`${r}/elsewhere`); file(`${r}/elsewhere/state.db`); symlinkSync('elsewhere', `${r}/state`); mkdirSync(`${r}/workspace`) }, ['elsewhere', 'state', 'workspace'], 'NO_SPLIT_SIGNATURE\n'],
+ ['workspace missing', r => { mkdirSync(`${r}/state`); file(`${r}/state/state.db`) }, ['state'], 'NO_SPLIT_SIGNATURE\n'],
+ ['stateful root', r => { file(`${r}/state.db`); file(`${r}/SOUL.md`) }, ['SOUL.md', 'state.db'], 'NO_SPLIT_SIGNATURE\n'],
+ ['T5', r => { file(`${r}/state.db`); mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`) }, ['state', 'state.db', 'workspace'], `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ROOT, ROOT/state and ROOT/workspace\n`],
+ ['read-only directory to move', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); mkdirSync(`${r}/workspace/users`); chmodSync(`${r}/workspace/users`, 0o555) }, ['state', 'workspace'], 'WILL_REFUSE_F4 workspace layout merge refused: not writable by uid UID: ROOT/workspace/users\n'],
+ ]
+ const uid = String(process.getuid())
+ for (const [name, build, listing, expected] of trees) {
+ const root = tree(build)
+ try {
+ assert.deepEqual(listDir(root), listing, name)
+ assert.equal(check(root), expected.replaceAll('ROOT', root).replaceAll('UID', uid), name)
+ assert.deepEqual(listDir(root), listing, `${name}: the check changed the tree`)
+ } finally {
+ chmodTree(root)
+ rmSync(root, { recursive: true, force: true })
+ }
+ }
+})
+
+test('check-script refuses an unlistable state/ like the merge init does', { skip: process.getuid() === 0 && 'root reads any mode' }, () => {
+ const root = tree(r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); chmodSync(`${r}/state`, 0o300) })
+ try {
+ assert.equal(check(root), `WILL_REFUSE_F1 workspace layout merge refused: state directory is not readable and searchable: ${root}/state\n`)
+ } finally {
+ chmodTree(root)
+ rmSync(root, { recursive: true, force: true })
+ }
+})
+
+test('check-script ends before the moves', () => {
+ const printed = spawnSync('bash', [SCRIPT, 'check-script'], { encoding: 'utf8' })
+ assert.equal(printed.status, 0, printed.stderr)
+ assert.match(printed.stdout, /^root='\/mnt\/workspace-root'$/m)
+ assert.match(printed.stdout, /\necho WILL_MERGE\n$/)
+ assert.doesNotMatch(printed.stdout, /\bmv |\brmdir |\[ -w /)
+})
+
+function listDir(root) {
+ return spawnSync('ls', ['-A', root], { encoding: 'utf8' }).stdout.split('\n').filter(Boolean).sort()
+}
+
+function chmodTree(root) {
+ spawnSync('chmod', ['-R', 'u+rwx', root])
+}
diff --git a/deploy/scripts/workspace-layout-merge.sh b/deploy/scripts/workspace-layout-merge.sh
new file mode 100644
index 000000000..60c33e2a0
--- /dev/null
+++ b/deploy/scripts/workspace-layout-merge.sh
@@ -0,0 +1,148 @@
+set -eu
+root='/mnt/workspace-root'
+fail_merge() {
+ echo "workspace layout merge refused: $1" >&2
+ exit 1
+}
+present() {
+ [ -e "$1" ] || [ -L "$1" ]
+}
+canonical_dir() {
+ (CDPATH= cd -P "$1" 2>/dev/null && pwd -P)
+}
+assert_managed_dir() {
+ path="$1"
+ label="$2"
+ [ ! -L "$path" ] || fail_merge "$label directory is a symlink: $path"
+ [ -d "$path" ] || fail_merge "$label path is not a directory: $path"
+ resolved="$(canonical_dir "$path")" || fail_merge "cannot resolve $label directory: $path"
+ case "$resolved" in
+ "$root_real"/*) ;;
+ *) fail_merge "$label directory escapes the workspace PVC root: $path" ;;
+ esac
+}
+is_empty_dir() {
+ [ -d "$1" ] && [ ! -L "$1" ] || return 1
+ listing="$(ls -A "$1" 2>/dev/null)" || fail_merge "cannot list directory: $1"
+ [ -z "$listing" ]
+}
+has_main() {
+ present "$1/state.db"
+}
+has_journal() {
+ present "$1/state.db-wal" || present "$1/state.db-shm"
+}
+
+[ ! -L "$root" ] || fail_merge "workspace PVC root is a symlink: $root"
+[ -d "$root" ] || fail_merge "workspace PVC root is not a directory: $root"
+root_real="$(canonical_dir "$root")" || fail_merge "cannot resolve workspace PVC root: $root"
+[ "$root_real" != "/" ] || fail_merge "workspace PVC root resolves to /"
+
+# Split-layout signature: real state/ and workspace/ directories, and a
+# database entry under state/.
+[ ! -L "$root/state" ] && [ -d "$root/state" ] || exit 0
+[ ! -L "$root/workspace" ] && [ -d "$root/workspace" ] || exit 0
+# The globs cannot see the entries of a state/ they cannot list or search.
+[ -r "$root/state" ] && [ -x "$root/state" ] ||
+ fail_merge "state directory is not readable and searchable: $root/state"
+# Only the names the earlier init writes count: the database, its journals
+# and the .pre-<ts>.bak copy it keeps of each of them from an older source.
+signature=0
+for entry in "$root/state"/state.db "$root/state"/state.db-wal "$root/state"/state.db-shm \
+ "$root/state"/state.db.pre-*.bak "$root/state"/state.db-wal.pre-*.bak "$root/state"/state.db-shm.pre-*.bak; do
+ if present "$entry"; then signature=1; fi
+done
+[ "$signature" -eq 1 ] || exit 0
+
+# F1
+assert_managed_dir "$root/state" state
+assert_managed_dir "$root/workspace" workspace
+# The move globs cannot see the entries of a workspace/ they cannot list.
+[ -r "$root/workspace" ] && [ -x "$root/workspace" ] ||
+ fail_merge "workspace directory is not readable and searchable: $root/workspace"
+for dir in "$root" "$root/state" "$root/workspace"; do
+ for entry in "$dir"/state.db "$dir"/state.db-wal "$dir"/state.db-shm \
+ "$dir"/state.db.pre-*.bak "$dir"/state.db-wal.pre-*.bak "$dir"/state.db-shm.pre-*.bak; do
+ present "$entry" || continue
+ [ ! -L "$entry" ] || fail_merge "SQLite file is a symlink: $entry"
+ [ -f "$entry" ] || fail_merge "SQLite file is not a regular file: $entry"
+ done
+done
+
+# F2
+live=0
+root_live=0
+workspace_live=0
+if has_main "$root" || has_journal "$root"; then root_live=1; live=$((live + 1)); fi
+if has_main "$root/state" || has_journal "$root/state"; then live=$((live + 1)); fi
+if has_main "$root/workspace" || has_journal "$root/workspace"; then
+ workspace_live=1
+ live=$((live + 1))
+fi
+resume=0
+if [ "$root_live" -eq 1 ] && ! has_main "$root" && has_main "$root/state" && [ "$workspace_live" -eq 0 ]; then
+ resume=1
+fi
+if [ "$live" -gt 1 ] && [ "$resume" -eq 0 ]; then
+ fail_merge "more than one live SQLite database across $root, $root/state and $root/workspace"
+fi
+
+# F3
+! present "$root/workspace/lost+found" || fail_merge "workspace/lost+found would remain: $root/workspace/lost+found"
+for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?*; do
+ present "$entry" || continue
+ base="${entry##*/}"
+ if present "$root/$base"; then
+ if is_empty_dir "$entry" && [ -d "$root/$base" ] && [ ! -L "$root/$base" ]; then continue; fi
+ fail_merge "destination exists: $root/$base"
+ fi
+done
+for entry in "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do
+ present "$entry" || continue
+ base="${entry##*/}"
+ if present "$root/$base"; then
+ if is_empty_dir "$entry" && [ -d "$root/$base" ] && [ ! -L "$root/$base" ]; then continue; fi
+ fail_merge "destination exists: $root/$base"
+ fi
+ if present "$root/workspace/$base"; then
+ fail_merge "$base exists under both state/ and workspace/"
+ fi
+done
+
+# F4
+for dir in "$root" "$root/state" "$root/workspace"; do
+ [ -w "$dir" ] || fail_merge "not writable by uid $(id -u): $dir"
+done
+# Renaming a directory to a new parent rewrites its .. entry, which needs
+# write permission on the directory itself.
+for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?* \
+ "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do
+ [ -d "$entry" ] && [ ! -L "$entry" ] || continue
+ ! present "$root/${entry##*/}" || continue
+ [ -w "$entry" ] || fail_merge "not writable by uid $(id -u): $entry"
+done
+
+move_to_root() {
+ entry="$1"
+ base="${entry##*/}"
+ assert_managed_dir "$root/state" state
+ assert_managed_dir "$root/workspace" workspace
+ if present "$root/$base"; then
+ rmdir "$entry"
+ else
+ mv "$entry" "$root/$base"
+ fi
+}
+for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?*; do
+ present "$entry" || continue
+ move_to_root "$entry"
+done
+for entry in "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do
+ present "$entry" || continue
+ [ "${entry##*/}" != state.db ] || continue
+ move_to_root "$entry"
+done
+if present "$root/state/state.db"; then move_to_root "$root/state/state.db"; fi
+rmdir "$root/workspace"
+rmdir "$root/state"
+sync
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index cd174bb0b..c75e50255 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.396",
+ "version": "0.1.397",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.396",
+ "version": "0.1.397",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 559c50b32..b2ec4d573 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.396",
+ "version": "0.1.397",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts
index cc390f9ef..ee0699eb0 100644
--- a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts
+++ b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts
@@ -6,7 +6,7 @@
* uncheckpointed WAL frames, split by the base `workspace-layout-init`
* (committed verbatim as a fixture), then merged by the new script.
*/
-import { afterEach, describe, expect, it } from 'vitest'
+import { afterAll, afterEach, describe, expect, it } from 'vitest'
import { spawnSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import {
@@ -65,10 +65,63 @@ function runScript(script: string, args: string[] = [], env?: NodeJS.ProcessEnv)
return { status: res.status ?? -1, stderr: res.stderr ?? '' }
}
+// deploy/scripts/host-layout-merge.sh predicts this merge on clusters from a
+// committed copy of the script, run read-only. Every merge below without a
+// crash env first asks it for a prediction and then checks it against what
+// the merge did, so each tree in this file is also a drift case.
+const DEPLOY_SCRIPTS = join(__dirname, '..', '..', 'deploy', 'scripts')
+const MERGE_SCRIPT_COPY = join(DEPLOY_SCRIPTS, 'workspace-layout-merge.sh')
+const VERIFY_SCRIPT = join(DEPLOY_SCRIPTS, 'host-layout-merge.sh')
+const PREDICTED_TOKENS = new Set<string>()
+
+function predictMerge(root: string): string {
+ const printed = spawnSync('bash', [VERIFY_SCRIPT, 'check-script', '--root', root], {
+ encoding: 'utf8',
+ })
+ if (printed.error) throw printed.error
+ expect(printed.status, printed.stderr).toBe(0)
+ const res = spawnSync('/bin/sh', ['-c', printed.stdout], { encoding: 'utf8' })
+ if (res.error) throw res.error
+ expect(res, `check on ${root}`).toMatchObject({ status: 0, stderr: '' })
+ return res.stdout
+}
+
+function rootListing(root: string): string {
+ return readdirSync(root).sort().join('\0')
+}
+
function merge(root: string, env?: NodeJS.ProcessEnv): RunResult {
- return runScript(buildWorkspaceLayoutMergeScript(root), [], env)
+ if (env) return runScript(buildWorkspaceLayoutMergeScript(root), [], env)
+ const predicted = predictMerge(root)
+ // A root that is not a directory is refused, so it has no listing to compare.
+ const before = statSync(root).isDirectory() ? rootListing(root) : null
+ const res = runScript(buildWorkspaceLayoutMergeScript(root))
+ const token = predicted.split(/[ \n]/, 1)[0]
+ PREDICTED_TOKENS.add(token)
+ if (res.status === 1) {
+ expect(token, predicted).toMatch(/^WILL_REFUSE_F[1-4]$/)
+ expect(predicted.slice(token.length + 1)).toBe(res.stderr)
+ } else {
+ expect(res.status, res.stderr).toBe(0)
+ expect(before, `${root} merged without being a directory`).not.toBeNull()
+ const expected = rootListing(root) === before ? 'NO_SPLIT_SIGNATURE' : 'WILL_MERGE'
+ expect(predicted).toBe(`${expected}\n`)
+ }
+ return res
}
+afterAll(() => {
+ // Liveness of the drift check: every outcome was predicted at least once.
+ expect([...PREDICTED_TOKENS].sort()).toEqual([
+ 'NO_SPLIT_SIGNATURE',
+ 'WILL_MERGE',
+ 'WILL_REFUSE_F1',
+ 'WILL_REFUSE_F2',
+ 'WILL_REFUSE_F3',
+ 'WILL_REFUSE_F4',
+ ])
+})
+
function baseInit(root: string): void {
const res = runScript(readFileSync(BASE_INIT, 'utf8'), [root])
expect(res, res.stderr).toMatchObject({ status: 0 })
@@ -176,6 +229,24 @@ function mvCalls(env: NodeJS.ProcessEnv): number {
}
describe('workspace layout merge', () => {
+ it('ships the rendered script as the copy the verify script reads', () => {
+ expect(
+ readFileSync(MERGE_SCRIPT_COPY, 'utf8'),
+ 'regenerate deploy/scripts/workspace-layout-merge.sh from buildWorkspaceLayoutMergeScript()'
+ ).toBe(buildWorkspaceLayoutMergeScript())
+ })
+
+ it('predicts a refusal for a Host switched to stateful before the upgrade (T5)', () => {
+ // A v0.10/v0.11 stateless Host switched to stateful: new chats in the
+ // root database, the older history still under state/.
+ const root = splitRoot()
+ cpSync(join(root, 'state', 'state.db'), join(root, 'state.db'))
+ expect(predictMerge(root)).toBe(
+ `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ${root}, ${root}/state and ${root}/workspace\n`
+ )
+ expect(merge(root).status).toBe(1)
+ })
+
it('merges a split tree with an uncheckpointed WAL back to the PVC root', () => {
const root = splitRoot()
// Witness that the split tree is the real one: the database and its