Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# Issue #825: read-only inventory and turn-off gate for the HCC workspace
# layout merge on any cluster (docs/deploy/host-pvc-layout-upgrade.md).
#
# Every Host PVC is classified with the merge init's own rules: the check
# Pods run deploy/scripts/workspace-layout-merge.sh (the script HCC renders,
# kept equal to it by a host-context-controller test) with the moves removed
# and the writability checks read from mode bits, because the PVC is mounted
# read-only. The script never opens a database and never execs into a Host.
set -euo pipefail
usage() {
cat <<'EOF'
usage:
host-layout-merge.sh inventory --context <kube-context> [options]
host-layout-merge.sh verify --context <kube-context> [options]
host-layout-merge.sh check-script [--root <path>]
inventory before an upgrade: what the merge init will do to every Host PVC.
Exit 1 when a tree will be refused (UPGRADE_BLOCKERS > 0), the
flag is "false" while a tree is split, the flag is invalid, or a
PVC could not be read.
verify whether CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE can be set to
"false": exit 0 only when every Host Deployment renders the merge
init and no Host PVC still carries the split layout.
check-script print the check the PVC check Pods run.
options:
--hcc-namespace <ns> HCC namespace (default control-plane)
--hcc-deployment <name> HCC Deployment (default host-context-controller)
--no-pvc-pods start no check Pod; PVC results are UNVERIFIED
--check-timeout <sec> wait per check Pod (default 120)
--root <path> check-script only: PVC root (default /mnt/workspace-root)
Reads only, except one short-lived Pod per inspected PVC: claim mounted
read-only, uid/gid 1001, no service-account token, deleted after its log is
read. Every kubectl call names --context; the current-context is never used.
Exit: 0 pass, 1 an observed blocker, 2 usage, read failure, empty scope or
check Pod cleanup failure.
EOF
}
KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
MERGE_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/workspace-layout-merge.sh"
PVC_ROOT=/mnt/workspace-root
die() {
echo "host-layout-merge: $*" >&2
exit 2
}
usage_error() {
echo "host-layout-merge: $*" >&2
usage >&2
exit 2
}
# ─── check script ─────────────────────────────────────────────────────
# Prints the merge script with:
# - the root replaced (check-script --root, used by the tests);
# - each refusal printed on stdout as WILL_REFUSE_<section> <line>, where
# the section is the "# F<n>" block it ran in (F1 before the first);
# - each "no signature" exit printed as NO_SPLIT_SIGNATURE;
# - `[ -w ]` replaced by a mode-bit test, since the check mount is read-only;
# - the T5 rename of state/ replaced by WILL_SUPERSEDE;
# - everything from the move function on replaced by WILL_MERGE.
# Each rewrite must match the expected number of lines, so a changed merge
# script stops here instead of producing a check that no longer matches it.
# The patterns and replacements below are shell text for the check Pod, so
# they are single-quoted on purpose.
# shellcheck disable=SC2016
print_check_script() {
local root=$1 line roots=0 sets=0 echoes=0 exits=0 sections=0 noops=0 writes=0 supersedes=0 cut=0
case "$root" in *"'"*) usage_error "--root must not contain a single quote" ;; esac
[ -f "$MERGE_SCRIPT" ] || die "missing $MERGE_SCRIPT"
while IFS= read -r line || [ -n "$line" ]; do
case "$line" in
"root='/mnt/workspace-root'")
printf "root='%s'\n" "$root"
roots=$((roots + 1))
;;
'set -eu')
printf '%s\n' "$line" 'phase=F1'
cat <<'EOF'
# Mode-bit write test: the check Pod mounts the PVC read-only, so the
# shell's -w test would refuse every tree. Same owner/group/other rule as
# access(2).
# A `+` in the mode string is a POSIX ACL, which the mode bits do not
# account for, so the check reports the tree UNREADABLE instead of guessing.
writable() {
mode_writable "$1" && return 0
[ "$?" -eq 2 ] || return 1
echo "UNREADABLE ACL on $1: mode bits do not decide access"
exit 0
}
mode_writable() (
set -f
[ "$(id -u)" -ne 0 ] || exit 0
listing="$(ls -ldn "$1")" || exit 1
set -- $listing
case "$1" in *+*) exit 2 ;; esac
if [ "$3" = "$(id -u)" ]; then
case "$1" in ??w*) exit 0 ;; *) exit 1 ;; esac
fi
for gid in $(id -G); do
if [ "$4" = "$gid" ]; then
case "$1" in ?????w*) exit 0 ;; *) exit 1 ;; esac
fi
done
case "$1" in ????????w*) exit 0 ;; *) exit 1 ;; esac
)
EOF
sets=$((sets + 1))
;;
' echo "workspace layout merge refused: $1" >&2')
printf '%s\n' ' echo "WILL_REFUSE_$phase workspace layout merge refused: $1"'
echoes=$((echoes + 1))
;;
' exit 1')
printf '%s\n' ' exit 0'
exits=$((exits + 1))
;;
'# F1' | '# F2' | '# F3' | '# F4')
printf 'phase=%s\n' "${line#'# '}"
sections=$((sections + 1))
;;
*'|| exit 0')
printf '%s\n' "${line%'|| exit 0'}|| { echo NO_SPLIT_SIGNATURE; exit 0; }"
noops=$((noops + 1))
;;
' [ -w "$dir" ] || fail_merge "not writable by uid $(id -u): $dir"')
printf '%s\n' ' writable "$dir" || fail_merge "not writable by uid $(id -u): $dir"'
writes=$((writes + 1))
;;
' [ -w "$entry" ] || fail_merge "not writable by uid $(id -u): $entry"')
printf '%s\n' ' writable "$entry" || fail_merge "not writable by uid $(id -u): $entry"'
writes=$((writes + 1))
;;
' [ -w "$root" ] || fail_merge "not writable by uid $(id -u): $root"')
printf '%s\n' ' phase=F4' ' writable "$root" || fail_merge "not writable by uid $(id -u): $root"' ' phase=F3'
writes=$((writes + 1))
;;
' mv "$root/state" "$superseded"')
printf '%s\n' ' echo WILL_SUPERSEDE'
supersedes=$((supersedes + 1))
;;
'move_to_root() {')
printf '%s\n' 'echo WILL_MERGE'
cut=1
break
;;
*)
printf '%s\n' "$line"
;;
esac
done <"$MERGE_SCRIPT"
local shape="root=$roots set=$sets echo=$echoes exit=$exits sections=$sections noops=$noops writes=$writes supersedes=$supersedes cut=$cut"
[ "$shape" = "root=1 set=1 echo=1 exit=1 sections=4 noops=3 writes=3 supersedes=1 cut=1" ] ||
die "$MERGE_SCRIPT has an unexpected shape ($shape); update print_check_script with it"
}
# ─── arguments ────────────────────────────────────────────────────────
COMMAND="${1:-}"
[ "$#" -eq 0 ] || shift
CONTEXT=""
HCC_NAMESPACE=control-plane
HCC_DEPLOYMENT='host-context-controller'
NO_PVC_PODS=0
CHECK_TIMEOUT=120
CHECK_ROOT="$PVC_ROOT"
need_value() { [ "$2" -ge 2 ] || usage_error "$1 needs a value"; }
while [ "$#" -gt 0 ]; do
case "$1" in
--context) need_value "$1" "$#"; CONTEXT="$2"; shift 2 ;;
--context=*) CONTEXT="${1#--context=}"; shift ;;
--hcc-namespace) need_value "$1" "$#"; HCC_NAMESPACE="$2"; shift 2 ;;
--hcc-deployment) need_value "$1" "$#"; HCC_DEPLOYMENT="$2"; shift 2 ;;
--no-pvc-pods) NO_PVC_PODS=1; shift ;;
--check-timeout) need_value "$1" "$#"; CHECK_TIMEOUT="$2"; shift 2 ;;
--root) need_value "$1" "$#"; CHECK_ROOT="$2"; shift 2 ;;
-h | --help) usage; exit 0 ;;
*) usage_error "unknown argument: $1" ;;
esac
done
case "$COMMAND" in
check-script)
print_check_script "$CHECK_ROOT"
exit 0
;;
inventory | verify) ;;
*) usage_error "expected inventory, verify or check-script" ;;
esac
[ -n "$CONTEXT" ] || usage_error "--context is required; the current-context is never used"
[ "$CHECK_ROOT" = "$PVC_ROOT" ] || usage_error "--root applies to check-script only"
case "$CHECK_TIMEOUT" in '' | *[!0-9]*) usage_error "--check-timeout must be a whole number of seconds" ;; esac
command -v jq >/dev/null 2>&1 || die "jq is required"
CHECK_SCRIPT="$(print_check_script "$PVC_ROOT")"
kc() { "$KUBECTL_BIN" --context="$CONTEXT" "$@"; }
WORK="$(mktemp -d "${TMPDIR:-/tmp}/host-layout-merge.XXXXXX")"
CHECK_PODS="$WORK/check-pods"
: >"$CHECK_PODS"
HOST_NS=""
# A check Pod left behind keeps an RWO claim attached to its node, which can
# stop the Host from starting elsewhere, so a failed delete is exit 2.
# shellcheck disable=SC2329 # invoked by the EXIT trap
cleanup() {
local status=$? pod
while IFS= read -r pod; do
[ -n "$pod" ] || continue
if ! kc delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>&1; then
echo "host-layout-merge: check Pod $HOST_NS/$pod was not deleted; delete it by hand" >&2
status=2
fi
done <"$CHECK_PODS"
rm -rf "$WORK"
exit "$status"
}
trap cleanup EXIT
trap 'exit 2' INT TERM
# Check Pod names carry this run's id, so a name held by another run (or any
# other Pod) is never reused, and cleanup deletes only Pods this run created.
RUN_ID="$(od -An -N4 -tx1 /dev/urandom | tr -d ' \n')"
case "$RUN_ID" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) die "cannot draw a run id from /dev/urandom" ;; esac
# read_json <file> <kubectl args...>: a JSON object or exit 2 with stderr.
read_json() {
local out=$1
shift
if ! kc "$@" >"$out" 2>"$out.err"; then
die "kubectl $* failed: $(cat "$out.err")"
fi
jq -e 'type == "object"' "$out" >/dev/null 2>&1 || die "kubectl $* did not return a JSON object"
}
# ─── HCC ──────────────────────────────────────────────────────────────
read_json "$WORK/hcc.json" get deployment "$HCC_DEPLOYMENT" -n "$HCC_NAMESPACE" -o json
jq -e '[.spec.template.spec.containers[]? | select(.name == "host-context-controller")] | length == 1' \
"$WORK/hcc.json" >/dev/null || die "$HCC_NAMESPACE/$HCC_DEPLOYMENT has no container named host-context-controller"
jq -e '(.spec.replicas // 1) as $r | $r > 0 and (.status.readyReplicas // 0) == $r and (.status.updatedReplicas // 0) == $r' \
"$WORK/hcc.json" >/dev/null || die "HCC_NOT_READY: $HCC_NAMESPACE/$HCC_DEPLOYMENT is not fully rolled out and Ready"
# hcc_env <name>: <unset>, <valueFrom> or the literal value. A variable not
# set in env but possibly set through envFrom is <valueFrom>: its value is
# not in the Deployment, so its default must not be assumed.
hcc_env() {
jq -r --arg n "$1" '.spec.template.spec.containers[] | select(.name == "host-context-controller")
| ((.envFrom // []) | length > 0) as $from
| [.env[]? | select(.name == $n)]
| if length == 0 then (if $from then "<valueFrom>" else "<unset>" end)
elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end' "$WORK/hcc.json"
}
FLAG_RAW="$(hcc_env CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE)"
case "$FLAG_RAW" in
'<unset>' | '' | true) FLAG=on ;;
false) FLAG=off ;;
'<valueFrom>') FLAG=unresolved ;;
*) FLAG=invalid ;;
esac
HOST_NS="$(hcc_env CONTEXT_MAPPER_HOST_NAMESPACE)"
case "$HOST_NS" in
'<unset>') HOST_NS=mcp-host ;;
'' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_NAMESPACE ($HOST_NS) from the HCC Deployment" ;;
esac
echo "HCC_IMAGE=$(jq -r '.spec.template.spec.containers[] | select(.name == "host-context-controller") | .image // "<none>"' "$WORK/hcc.json")"
echo "HCC_FLAG=$FLAG_RAW"
echo "HOST_NAMESPACE=$HOST_NS"
# ─── Hosts, Deployments, Pods, PVCs ───────────────────────────────────
read_json "$WORK/hosts.json" get hosts.clerum.io -n "$HOST_NS" -o json
read_json "$WORK/deployments.json" get deployments -n "$HOST_NS" -l clerum.io/host -o json
read_json "$WORK/pods.json" get pods -n "$HOST_NS" -l clerum.io/host -o json
read_json "$WORK/pvcs.json" get pvc -n "$HOST_NS" -l clerum.io/managed-by=host-context-controller -o json
# One tab-separated row per Host (H) and per Host PVC (P), "-" for empty.
# A Host Deployment and its Pods carry app == clerum.io/host == <host>; the
# per-Host channel-reader carries app=channel-reader and is excluded.
jq -r -n \
--slurpfile hosts "$WORK/hosts.json" \
--slurpfile deps "$WORK/deployments.json" \
--slurpfile pods "$WORK/pods.json" \
--slurpfile pvcs "$WORK/pvcs.json" '
def owned($h): select(.metadata.labels.app == $h and .metadata.labels["clerum.io/host"] == $h);
def kind: if (.spec.desktop.browser // false) or (.spec.desktop.x11 // false) then "DESKTOP"
elif (.spec.lifecycle.stateless // false) then "STATELESS" else "STATEFUL" end;
def template: [.spec.template.spec.initContainers[]?.name] as $i
| if any($i[]; . == "workspace-layout-init") then "LEGACY_INIT"
elif any($i[]; . == "workspace-layout-merge") then "MERGE" else "NO_INIT" end;
def subpath($claim): .spec.template.spec as $s
| [$s.volumes[]? | select(.persistentVolumeClaim.claimName == $claim) | .name] as $v
| if any(($s.containers + ($s.initContainers // []))[] | .volumeMounts[]?;
(.name as $n | $v | index($n)) != null and (.subPath // "") != "") then "yes" else "no" end;
def merge_status: [.status.initContainerStatuses[]? | select(.name == "workspace-layout-merge")][0] as $s
| if $s == null then "UNKNOWN"
elif $s.state.terminated.exitCode == 0 then "POD_OK"
elif $s.state.terminated.exitCode == 1 then "MERGE_REFUSED:current"
elif $s.lastState.terminated.exitCode == 1 then "MERGE_REFUSED:previous"
elif $s.state.terminated != null then "UNKNOWN"
elif $s.state.running != null or $s.state.waiting != null then "MERGE_RUNNING"
else "UNKNOWN" end;
def worst: (map({"MERGE_REFUSED": 0, "UNKNOWN": 1, "MERGE_RUNNING": 2, "POD_OK": 3}[split(":")[0]]) | min) as $m
| map(select({"MERGE_REFUSED": 0, "UNKNOWN": 1, "MERGE_RUNNING": 2, "POD_OK": 3}[split(":")[0]] == $m))[0];
def dash: if . == null or . == "" then "-" else . end;
def host_facts($h):
[$deps[0].items[] | owned($h)] as $d
| if ($d | length) > 1 then error("more than one Deployment for Host \($h)") else . end
| [$pods[0].items[] | owned($h) | select(.metadata.deletionTimestamp == null)] as $p
| ($d[0] // null) as $dep
| ($dep | if . == null then null else [.spec.template.spec.containers[]? | select(.name == "mcp-host")][0] end) as $main
| if $dep != null and $main == null then error("Deployment \($h) has no mcp-host container") else . end
| {
state: (if $dep == null then "NO_DEPLOYMENT" elif ($dep.spec.replicas // 1) == 0 then "SUSPENDED" else "ACTIVE" end),
template: (if $dep == null then "-" else ($dep | template) end),
subpath: (if $dep == null then "-" else ($dep | subpath("\($h)-workspace")) end),
pods: ($p | length),
nodes: ([$p[].spec.nodeName | select(. != null)] | unique),
merge: (if ($p | length) == 0 then "-" else ($p | map({s: merge_status, pod: .metadata.name}) | (map(.s) | worst) as $w | map(select(.s == $w))[0] | "\(.s):\(.pod)") end),
image: ($main.image // null),
policy: ($main.imagePullPolicy // null),
secrets: ($dep.spec.template.spec.imagePullSecrets // [] | map(.name) | join(","))
};
($hosts[0].items | map({key: .metadata.name, value: kind}) | from_entries) as $kinds
| [$pvcs[0].items[] | select(.metadata.name == "\(.metadata.labels["clerum.io/host"])-workspace") | .metadata.name] as $listed
| ($hosts[0].items[] | .metadata.name as $h | host_facts($h) as $f
| (["H", $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash)] | @tsv),
# Every Host with a Deployment must have its <host>-workspace claim
# mounted and in the PVC list; otherwise nothing reads that tree.
([$deps[0].items[] | owned($h)][0] // null
| if . == null then empty
elif ([.spec.template.spec.volumes[]?.persistentVolumeClaim.claimName] | index("\($h)-workspace")) == null then
["U", "\($h)-workspace", $h, $kinds[$h], $f.state, "Deployment mounts no \($h)-workspace claim"] | @tsv
elif ($listed | index("\($h)-workspace")) == null then
["U", "\($h)-workspace", $h, $kinds[$h], $f.state, "claim not in the PVC list (clerum.io/managed-by=host-context-controller, clerum.io/host=\($h))"] | @tsv
else empty end)),
($pvcs[0].items[] | .metadata.labels["clerum.io/host"] as $h
| if $h == null then
["I", .metadata.name] | @tsv
elif .metadata.name != "\($h)-workspace" then
["U", .metadata.name, $h, ($kinds[$h] // "ORPHAN"), (if $kinds[$h] == null then "NO_HOST" else host_facts($h).state end),
"clerum.io/host=\($h) on a claim not named \($h)-workspace"] | @tsv
elif $kinds[$h] == null then
["P", .metadata.name, $h, "ORPHAN", "NO_HOST", "-", "-", "0", "-", "-", "-", "-", "-"] | @tsv
else
host_facts($h) as $f
| ["P", .metadata.name, $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash),
($f.pods | tostring), ($f.nodes | join(",") | dash), ($f.merge | dash),
($f.image | dash), ($f.policy | dash), ($f.secrets | dash)] | @tsv
end)
' >"$WORK/rows.tsv" 2>"$WORK/rows.err" || die "cannot classify the Host objects: $(cat "$WORK/rows.err")"
HOSTS=0
PRE_825=no
DESKTOP_HOSTS=0
TEMPLATE_STALE=0
while IFS=$'\t' read -r type name kind state template subpath; do
[ "$type" = H ] || continue
HOSTS=$((HOSTS + 1))
echo "HOST $name kind=$kind state=$state template=$template subPath=$subpath"
[ "$kind" != DESKTOP ] || DESKTOP_HOSTS=$((DESKTOP_HOSTS + 1))
if [ "$template" = LEGACY_INIT ] || [ "$subpath" = yes ]; then PRE_825=yes; fi
if [ "$state" != NO_DEPLOYMENT ] && [ "$template" != MERGE ]; then TEMPLATE_STALE=$((TEMPLATE_STALE + 1)); fi
done <"$WORK/rows.tsv"
while IFS=$'\t' read -r type name _; do
[ "$type" = I ] || continue
echo "IGNORED_PVC $HOST_NS/$name (no clerum.io/host label)"
done <"$WORK/rows.tsv"
PVC_ROWS="$(awk -F'\t' '$1 == "P" || $1 == "U"' "$WORK/rows.tsv" | wc -l | tr -d ' ')"
[ "$HOSTS" -gt 0 ] || [ "$PVC_ROWS" -gt 0 ] || die "no Host and no Host PVC in namespace $HOST_NS; wrong --context or namespace?"
echo "PRE_825=$PRE_825"
# ─── verdicts that need no PVC read ───────────────────────────────────
if [ "$COMMAND" = verify ]; then
case "$FLAG" in
invalid) echo "HOST_LAYOUT_VERIFY=FLAG_INVALID"; exit 1 ;;
unresolved) echo "HOST_LAYOUT_VERIFY=FLAG_UNRESOLVED"; exit 1 ;;
esac
if [ "$PRE_825" = yes ]; then
echo "a Host Deployment still renders workspace-layout-init or a workspace subPath: run inventory and finish the upgrade first"
echo "HOST_LAYOUT_VERIFY=PRE_825"
exit 1
fi
fi
# ─── check Pods ───────────────────────────────────────────────────────
# run_check <pvc> <node|-> <image> <policy> <secrets|->: prints the result
# token (and refusal line) for that PVC.
run_check() {
local claim=$1 node=$2 image=$3 policy=$4 secrets=$5 pod phase elapsed=0 first lines
pod="$(printf 'host-layout-check-%s' "$claim" | cut -c1-54 | sed 's/[-.]*$//')-$RUN_ID"
jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$image" --arg policy "$policy" \
--arg secrets "$secrets" --arg claim "$claim" --arg script "$CHECK_SCRIPT" --arg root "$PVC_ROOT" '{
apiVersion: "v1", kind: "Pod",
metadata: {name: $pod, namespace: $ns, labels: {purpose: "host-layout-merge-check"}},
spec: ({
restartPolicy: "Never", automountServiceAccountToken: false, enableServiceLinks: false,
terminationGracePeriodSeconds: 0,
securityContext: {runAsUser: 1001, runAsGroup: 1001, runAsNonRoot: true, seccompProfile: {type: "RuntimeDefault"}},
imagePullSecrets: (if $secrets == "-" then [] else ($secrets | split(",") | map({name: .})) end),
containers: [{
name: "check", image: $image, imagePullPolicy: $policy,
command: ["/bin/sh", "-c", $script],
resources: {requests: {cpu: "10m", memory: "32Mi"}, limits: {cpu: "100m", memory: "64Mi"}},
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: ["ALL"]}},
volumeMounts: [{name: "pvc", mountPath: $root, readOnly: true}]
}],
volumes: [{name: "pvc", persistentVolumeClaim: {claimName: $claim, readOnly: true}}]
} + (if $node == "-" then {} else {nodeName: $node} end))
}' >"$WORK/pod.json"
# Registered before the create, so a create that times out after the API
# accepted it is still deleted.
printf '%s\n' "$pod" >>"$CHECK_PODS"
kc create -f - <"$WORK/pod.json" >/dev/null 2>"$WORK/create.err" ||
die "check Pod $HOST_NS/$pod create failed: $(cat "$WORK/create.err")"
while :; do
read_json "$WORK/status.json" get pod "$pod" -n "$HOST_NS" -o json
phase="$(jq -r '.status.phase // ""' "$WORK/status.json")"
case "$phase" in Succeeded | Failed) break ;; esac
[ "$elapsed" -lt "$CHECK_TIMEOUT" ] || break
sleep 2
elapsed=$((elapsed + 2))
done
: >"$WORK/check.log"
if [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; then
kc logs "$pod" -n "$HOST_NS" >"$WORK/check.log" 2>&1 || die "cannot read the log of check Pod $HOST_NS/$pod: $(cat "$WORK/check.log")"
fi
kc delete pod "$pod" -n "$HOST_NS" --wait=true --timeout=60s >/dev/null 2>"$WORK/delete.err" ||
die "check Pod $HOST_NS/$pod delete failed: $(cat "$WORK/delete.err")"
grep -vxF -- "$pod" "$CHECK_PODS" >"$CHECK_PODS.next" || true
mv "$CHECK_PODS.next" "$CHECK_PODS"
if [ "$phase" != Succeeded ]; then
echo "UNREADABLE phase=${phase:-<none>} $(jq -r '[.status.containerStatuses[]?.state.waiting.reason // empty] | join(",")' "$WORK/status.json") $(tail -n 2 "$WORK/check.log" | tr '\n' ' ')"
return
fi
first="$(head -n 1 "$WORK/check.log")"
lines="$(wc -l <"$WORK/check.log" | tr -d ' ')"
case "$first" in
WILL_MERGE | WILL_SUPERSEDE | NO_SPLIT_SIGNATURE)
if [ "$lines" = 1 ]; then echo "$first"; else echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")"; fi
;;
'WILL_REFUSE_F'[1-4]' workspace layout merge refused: '*) tr '\n' ' ' <"$WORK/check.log" | sed 's/ $//'; echo ;;
*) echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")" ;;
esac
}
# The image for a PVC without a Host Deployment comes from HCC itself.
hcc_host_image() {
local image policy secret
image="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE)"
policy="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY)"
secret="$(hcc_env CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET)"
case "$image" in '' | '<unset>' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE ($image) for a check Pod" ;; esac
case "$policy" in '<unset>') policy=Always ;; '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE_PULL_POLICY ($policy)" ;; esac
case "$secret" in '<unset>') secret=clerum ;; '' | '<valueFrom>') die "cannot resolve CONTEXT_MAPPER_HOST_IMAGE_PULL_SECRET ($secret)" ;; esac
printf '%s\t%s\t%s\n' "$image" "$policy" "$secret"
}
# merge_line <status:pod>: the refusal line from the merge init's log.
merge_line() {
local which=${1%%:*} pod=${1#*:} args=()
[ "$which" = previous ] && args=(--previous)
kc logs "$pod" -n "$HOST_NS" -c workspace-layout-merge ${args[@]+"${args[@]}"} >"$WORK/merge.log" 2>&1 ||
die "cannot read the workspace-layout-merge log of $HOST_NS/$pod: $(cat "$WORK/merge.log")"
grep -m 1 '^workspace layout merge refused: ' "$WORK/merge.log" || echo "<no refusal line in the log>"
}
TOTAL=0 POD_OK=0 NO_SPLIT=0 SPLIT=0 REFUSED=0 RUNNING=0 UNREADABLE=0 UNVERIFIED=0 UNKNOWN=0
ORPHANS=0 ORPHAN_SPLIT=0 ORPHAN_UNREADABLE=0 ORPHAN_UNVERIFIED=0 UPGRADE_BLOCKERS=0 SPLIT_TREES=0
while IFS=$'\t' read -r type pvc host kind state detail _subpath pods nodes merge image policy secrets; do
case "$type" in P | U) ;; *) continue ;; esac
result=""
merge_ok=0
if [ "$type" = U ]; then
result="UNKNOWN $detail"
elif [ "$COMMAND" = verify ] && [ "$FLAG" = on ] && [ "$pods" -gt 0 ]; then
# An init that exited 0 is not proof the tree has no split-layout
# signature (it can be rewritten after the start), so POD_OK still reads it.
case "${merge%%:*}" in
POD_OK) merge_ok=1 ;;
MERGE_RUNNING | UNKNOWN) result="${merge%%:*}" ;;
MERGE_REFUSED) result="MERGE_REFUSED $(merge_line "${merge#MERGE_REFUSED:}")" ;;
*) die "unexpected merge status $merge for $pvc" ;;
esac
fi
if [ -n "$result" ]; then
:
elif [ "$NO_PVC_PODS" -eq 1 ]; then
result=UNVERIFIED
else
node=-
if [ "$pods" -gt 0 ]; then
case "$nodes" in
- ) node=- ;;
*,*) result="UNREADABLE Host Pods on more than one node ($nodes)" ;;
*) node="$nodes" ;;
esac
fi
if [ -z "$result" ]; then
if [ "$image" = - ]; then
IFS=$'\t' read -r image policy secrets <<<"$(hcc_host_image)"
fi
result="$(run_check "$pvc" "$node" "$image" "$policy" "$secrets" </dev/null)"
# verify names a tree the merge will still change SPLIT.
case "$COMMAND:$result" in verify:WILL_MERGE | verify:WILL_SUPERSEDE) result=SPLIT ;; esac
[ "$merge_ok" -eq 0 ] || [ "$result" != NO_SPLIT_SIGNATURE ] || result=POD_OK
fi
fi
# A known split orphan does not block (T9); an orphan nobody could read does.
if [ "$type" = P ] && [ "$kind" = ORPHAN ]; then
echo "PVC $HOST_NS/$pvc host=$host kind=ORPHAN state=$state result=$result"
ORPHANS=$((ORPHANS + 1))
case "$result" in
WILL_MERGE | WILL_SUPERSEDE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;;
NO_SPLIT_SIGNATURE) ;;
UNREADABLE*) ORPHAN_UNREADABLE=$((ORPHAN_UNREADABLE + 1)) ;;
UNVERIFIED) ORPHAN_UNVERIFIED=$((ORPHAN_UNVERIFIED + 1)) ;;
*) die "unclassified orphan result for $pvc: $result" ;;
esac
continue
fi
echo "PVC $HOST_NS/$pvc host=$host kind=$kind state=$state result=$result"
TOTAL=$((TOTAL + 1))
case "$result" in
POD_OK) POD_OK=$((POD_OK + 1)) ;;
NO_SPLIT_SIGNATURE) NO_SPLIT=$((NO_SPLIT + 1)) ;;
WILL_MERGE | WILL_SUPERSEDE | SPLIT) SPLIT=$((SPLIT + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)) ;;
WILL_REFUSE_*) REFUSED=$((REFUSED + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)); UPGRADE_BLOCKERS=$((UPGRADE_BLOCKERS + 1)) ;;
MERGE_REFUSED*) REFUSED=$((REFUSED + 1)) ;;
MERGE_RUNNING) RUNNING=$((RUNNING + 1)) ;;
UNREADABLE*) UNREADABLE=$((UNREADABLE + 1)) ;;
UNVERIFIED) UNVERIFIED=$((UNVERIFIED + 1)) ;;
UNKNOWN | 'UNKNOWN '*) UNKNOWN=$((UNKNOWN + 1)) ;;
*) die "unclassified result for $pvc: $result" ;;
esac
done <"$WORK/rows.tsv"
SUM=$((POD_OK + NO_SPLIT + SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED + UNKNOWN))
[ "$SUM" -eq "$TOTAL" ] || die "counted $SUM results for $TOTAL Host PVCs"
echo "COUNTS total=$TOTAL pod_ok=$POD_OK no_split=$NO_SPLIT split=$SPLIT refused=$REFUSED running=$RUNNING unreadable=$UNREADABLE unverified=$UNVERIFIED unknown=$UNKNOWN template_stale=$TEMPLATE_STALE"
echo "ORPHANS=$ORPHANS ORPHAN_SPLIT=$ORPHAN_SPLIT"
echo "ORPHAN_UNDECIDED=$((ORPHAN_UNREADABLE + ORPHAN_UNVERIFIED))"
if [ "$ORPHAN_SPLIT" -gt 0 ]; then
echo "note: a Host recreated with an orphan's name after the flag is off starts on an empty database; delete the orphan PVC first or recreate the Host before turning the flag off"
fi
echo "PVC_CONTENTS=NOT_INSPECTED"
if [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; then
echo "FLAG_FALSE_WITH_SPLIT_TREES=$SPLIT_TREES"
fi
if [ "$COMMAND" = inventory ]; then
echo "UPGRADE_BLOCKERS=$UPGRADE_BLOCKERS"
if [ "$DESKTOP_HOSTS" -gt 0 ]; then
echo "DESKTOP_HOSTS=$DESKTOP_HOSTS export /workspace/state.db* from each desktop Host Pod before the first rollout of this HCC (docs/deploy/host-pvc-layout-upgrade.md)"
else
echo "DESKTOP_HOSTS=0"
fi
if [ "$UPGRADE_BLOCKERS" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] || [ "$UNKNOWN" -gt 0 ] ||
[ "$ORPHAN_UNREADABLE" -gt 0 ] || { [ "$HOSTS" -gt 0 ] && [ "$TOTAL" -eq 0 ]; } ||
{ [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; } ||
[ "$FLAG" = invalid ] || [ "$FLAG" = unresolved ]; then
echo "HOST_LAYOUT_INVENTORY=BLOCKED"
exit 1
fi
echo "HOST_LAYOUT_INVENTORY=OK"
exit 0
fi
# A PVC nobody read is undecided whatever the flag says: ALREADY_OFF is
# printed only when every PVC was read and none is split.
UNDECIDED=$((UNREADABLE + UNVERIFIED + UNKNOWN + ORPHAN_UNREADABLE + ORPHAN_UNVERIFIED))
if [ "$FLAG" = off ]; then
if [ "$SPLIT_TREES" -gt 0 ]; then
echo "HOST_LAYOUT_VERIFY=FLAG_FALSE_WITH_SPLIT_TREES"
elif [ "$UNDECIDED" -gt 0 ]; then
echo "HOST_LAYOUT_VERIFY=FAIL"
else
echo "HOST_LAYOUT_VERIFY=ALREADY_OFF"
fi
exit 1
fi
if [ "$TOTAL" -gt 0 ] && [ "$((SPLIT + REFUSED + RUNNING + UNDECIDED + TEMPLATE_STALE))" -eq 0 ]; then
echo "HOST_LAYOUT_VERIFY=PASS"
exit 0
fi
echo "HOST_LAYOUT_VERIFY=FAIL"
exit 1