Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(deploy): make host-layout-merge verify fail closed on every gap

Alfredo Lopez8cb757173f3f
diff --git a/deploy/scripts/host-layout-merge.sh b/deploy/scripts/host-layout-merge.sh
index 2354a5a23..9d1f37104 100755
--- a/deploy/scripts/host-layout-merge.sh
+++ b/deploy/scripts/host-layout-merge.sh
@@ -83,11 +83,20 @@ print_check_script() {
# Mode-bit write test: the check Pod mounts the PVC read-only, so the
# shell's -w test would refuse every tree. Same owner/group/other rule as
# access(2).
-writable() (
+# A `+` in the mode string is a POSIX ACL, which the mode bits do not
+# account for, so the check reports the tree UNREADABLE instead of guessing.
+writable() {
+ mode_writable "$1" && return 0
+ [ "$?" -eq 2 ] || return 1
+ echo "UNREADABLE ACL on $1: mode bits do not decide access"
+ exit 0
+}
+mode_writable() (
set -f
[ "$(id -u)" -ne 0 ] || exit 0
listing="$(ls -ldn "$1")" || exit 1
set -- $listing
+ case "$1" in *+*) exit 2 ;; esac
if [ "$3" = "$(id -u)" ]; then
case "$1" in ??w*) exit 0 ;; *) exit 1 ;; esac
fi
@@ -202,6 +211,10 @@ cleanup() {
}
trap cleanup EXIT
trap 'exit 2' INT TERM
+# Check Pod names carry this run's id, so a name held by another run (or any
+# other Pod) is never reused, and cleanup deletes only Pods this run created.
+RUN_ID="$(od -An -N4 -tx1 /dev/urandom | tr -d ' \n')"
+case "$RUN_ID" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) die "cannot draw a run id from /dev/urandom" ;; esac
# read_json <file> <kubectl args...>: a JSON object or exit 2 with stderr.
read_json() {
@@ -220,10 +233,15 @@ jq -e '[.spec.template.spec.containers[]? | select(.name == "host-context-contro
jq -e '(.spec.replicas // 1) as $r | $r > 0 and (.status.readyReplicas // 0) == $r and (.status.updatedReplicas // 0) == $r' \
"$WORK/hcc.json" >/dev/null || die "HCC_NOT_READY: $HCC_NAMESPACE/$HCC_DEPLOYMENT is not fully rolled out and Ready"
-# hcc_env <name>: <unset>, <valueFrom> or the literal value.
+# hcc_env <name>: <unset>, <valueFrom> or the literal value. A variable not
+# set in env but possibly set through envFrom is <valueFrom>: its value is
+# not in the Deployment, so its default must not be assumed.
hcc_env() {
- jq -r --arg n "$1" '[.spec.template.spec.containers[] | select(.name == "host-context-controller") | .env[]? | select(.name == $n)]
- | if length == 0 then "<unset>" elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end' "$WORK/hcc.json"
+ jq -r --arg n "$1" '.spec.template.spec.containers[] | select(.name == "host-context-controller")
+ | ((.envFrom // []) | length > 0) as $from
+ | [.env[]? | select(.name == $n)]
+ | if length == 0 then (if $from then "<valueFrom>" else "<unset>" end)
+ elif .[0].valueFrom then "<valueFrom>" else (.[0].value // "") end' "$WORK/hcc.json"
}
FLAG_RAW="$(hcc_env CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE)"
@@ -296,11 +314,24 @@ jq -r -n \
secrets: ($dep.spec.template.spec.imagePullSecrets // [] | map(.name) | join(","))
};
($hosts[0].items | map({key: .metadata.name, value: kind}) | from_entries) as $kinds
+ | [$pvcs[0].items[] | select(.metadata.name == "\(.metadata.labels["clerum.io/host"])-workspace") | .metadata.name] as $listed
| ($hosts[0].items[] | .metadata.name as $h | host_facts($h) as $f
- | ["H", $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash)] | @tsv),
+ | (["H", $h, $kinds[$h], $f.state, ($f.template | dash), ($f.subpath | dash)] | @tsv),
+ # Every Host with a Deployment must have its <host>-workspace claim
+ # mounted and in the PVC list; otherwise nothing reads that tree.
+ ([$deps[0].items[] | owned($h)][0] // null
+ | if . == null then empty
+ elif ([.spec.template.spec.volumes[]?.persistentVolumeClaim.claimName] | index("\($h)-workspace")) == null then
+ ["U", "\($h)-workspace", $h, $kinds[$h], $f.state, "Deployment mounts no \($h)-workspace claim"] | @tsv
+ elif ($listed | index("\($h)-workspace")) == null then
+ ["U", "\($h)-workspace", $h, $kinds[$h], $f.state, "claim not in the PVC list (clerum.io/managed-by=host-context-controller, clerum.io/host=\($h))"] | @tsv
+ else empty end)),
($pvcs[0].items[] | .metadata.labels["clerum.io/host"] as $h
- | if $h == null or .metadata.name != "\($h)-workspace" then
+ | if $h == null then
["I", .metadata.name] | @tsv
+ elif .metadata.name != "\($h)-workspace" then
+ ["U", .metadata.name, $h, ($kinds[$h] // "ORPHAN"), (if $kinds[$h] == null then "NO_HOST" else host_facts($h).state end),
+ "clerum.io/host=\($h) on a claim not named \($h)-workspace"] | @tsv
elif $kinds[$h] == null then
["P", .metadata.name, $h, "ORPHAN", "NO_HOST", "-", "-", "0", "-", "-", "-", "-", "-"] | @tsv
else
@@ -325,9 +356,9 @@ while IFS=$'\t' read -r type name kind state template subpath; do
done <"$WORK/rows.tsv"
while IFS=$'\t' read -r type name _; do
[ "$type" = I ] || continue
- echo "IGNORED_PVC $HOST_NS/$name (not <host>-workspace of a clerum.io/host label)"
+ echo "IGNORED_PVC $HOST_NS/$name (no clerum.io/host label)"
done <"$WORK/rows.tsv"
-PVC_ROWS="$(awk -F'\t' '$1 == "P"' "$WORK/rows.tsv" | wc -l | tr -d ' ')"
+PVC_ROWS="$(awk -F'\t' '$1 == "P" || $1 == "U"' "$WORK/rows.tsv" | wc -l | tr -d ' ')"
[ "$HOSTS" -gt 0 ] || [ "$PVC_ROWS" -gt 0 ] || die "no Host and no Host PVC in namespace $HOST_NS; wrong --context or namespace?"
echo "PRE_825=$PRE_825"
@@ -349,7 +380,7 @@ fi
# token (and refusal line) for that PVC.
run_check() {
local claim=$1 node=$2 image=$3 policy=$4 secrets=$5 pod phase elapsed=0 first lines
- pod="$(printf 'host-layout-check-%s' "$claim" | cut -c1-63 | sed 's/[-.]*$//')"
+ pod="$(printf 'host-layout-check-%s' "$claim" | cut -c1-54 | sed 's/[-.]*$//')-$RUN_ID"
jq -cn --arg pod "$pod" --arg ns "$HOST_NS" --arg node "$node" --arg image "$image" --arg policy "$policy" \
--arg secrets "$secrets" --arg claim "$claim" --arg script "$CHECK_SCRIPT" --arg root "$PVC_ROOT" '{
apiVersion: "v1", kind: "Pod",
@@ -428,16 +459,25 @@ merge_line() {
}
TOTAL=0 POD_OK=0 NO_SPLIT=0 SPLIT=0 REFUSED=0 RUNNING=0 UNREADABLE=0 UNVERIFIED=0 UNKNOWN=0
-ORPHANS=0 ORPHAN_SPLIT=0 UPGRADE_BLOCKERS=0 SPLIT_TREES=0
-while IFS=$'\t' read -r type pvc host kind state _template _subpath pods nodes merge image policy secrets; do
- [ "$type" = P ] || continue
+ORPHANS=0 ORPHAN_SPLIT=0 ORPHAN_UNREADABLE=0 ORPHAN_UNVERIFIED=0 UPGRADE_BLOCKERS=0 SPLIT_TREES=0
+while IFS=$'\t' read -r type pvc host kind state detail _subpath pods nodes merge image policy secrets; do
+ case "$type" in P | U) ;; *) continue ;; esac
result=""
- if [ "$COMMAND" = verify ] && [ "$FLAG" = on ] && [ "$pods" -gt 0 ]; then
+ merge_ok=0
+ if [ "$type" = U ]; then
+ result="UNKNOWN $detail"
+ elif [ "$COMMAND" = verify ] && [ "$FLAG" = on ] && [ "$pods" -gt 0 ]; then
+ # An init that exited 0 is not proof the tree has no split-layout
+ # signature (it can be rewritten after the start), so POD_OK still reads it.
case "${merge%%:*}" in
- POD_OK | MERGE_RUNNING | UNKNOWN) result="${merge%%:*}" ;;
+ POD_OK) merge_ok=1 ;;
+ MERGE_RUNNING | UNKNOWN) result="${merge%%:*}" ;;
MERGE_REFUSED) result="MERGE_REFUSED $(merge_line "${merge#MERGE_REFUSED:}")" ;;
*) die "unexpected merge status $merge for $pvc" ;;
esac
+ fi
+ if [ -n "$result" ]; then
+ :
elif [ "$NO_PVC_PODS" -eq 1 ]; then
result=UNVERIFIED
else
@@ -456,12 +496,20 @@ while IFS=$'\t' read -r type pvc host kind state _template _subpath pods nodes m
result="$(run_check "$pvc" "$node" "$image" "$policy" "$secrets" </dev/null)"
# verify names a tree the merge will still change SPLIT.
[ "$COMMAND" = inventory ] || [ "$result" != WILL_MERGE ] || result=SPLIT
+ [ "$merge_ok" -eq 0 ] || [ "$result" != NO_SPLIT_SIGNATURE ] || result=POD_OK
fi
fi
- if [ "$kind" = ORPHAN ]; then
+ # A known split orphan does not block (T9); an orphan nobody could read does.
+ if [ "$type" = P ] && [ "$kind" = ORPHAN ]; then
echo "PVC $HOST_NS/$pvc host=$host kind=ORPHAN state=$state result=$result"
ORPHANS=$((ORPHANS + 1))
- case "$result" in WILL_MERGE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;; esac
+ case "$result" in
+ WILL_MERGE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;;
+ NO_SPLIT_SIGNATURE) ;;
+ UNREADABLE*) ORPHAN_UNREADABLE=$((ORPHAN_UNREADABLE + 1)) ;;
+ UNVERIFIED) ORPHAN_UNVERIFIED=$((ORPHAN_UNVERIFIED + 1)) ;;
+ *) die "unclassified orphan result for $pvc: $result" ;;
+ esac
continue
fi
echo "PVC $HOST_NS/$pvc host=$host kind=$kind state=$state result=$result"
@@ -475,7 +523,7 @@ while IFS=$'\t' read -r type pvc host kind state _template _subpath pods nodes m
MERGE_RUNNING) RUNNING=$((RUNNING + 1)) ;;
UNREADABLE*) UNREADABLE=$((UNREADABLE + 1)) ;;
UNVERIFIED) UNVERIFIED=$((UNVERIFIED + 1)) ;;
- UNKNOWN) UNKNOWN=$((UNKNOWN + 1)) ;;
+ UNKNOWN | 'UNKNOWN '*) UNKNOWN=$((UNKNOWN + 1)) ;;
*) die "unclassified result for $pvc: $result" ;;
esac
done <"$WORK/rows.tsv"
@@ -484,6 +532,7 @@ SUM=$((POD_OK + NO_SPLIT + SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED +
[ "$SUM" -eq "$TOTAL" ] || die "counted $SUM results for $TOTAL Host PVCs"
echo "COUNTS total=$TOTAL pod_ok=$POD_OK no_split=$NO_SPLIT split=$SPLIT refused=$REFUSED running=$RUNNING unreadable=$UNREADABLE unverified=$UNVERIFIED unknown=$UNKNOWN template_stale=$TEMPLATE_STALE"
echo "ORPHANS=$ORPHANS ORPHAN_SPLIT=$ORPHAN_SPLIT"
+echo "ORPHAN_UNDECIDED=$((ORPHAN_UNREADABLE + ORPHAN_UNVERIFIED))"
if [ "$ORPHAN_SPLIT" -gt 0 ]; then
echo "note: a Host recreated with an orphan's name after the flag is off starts on an empty database; delete the orphan PVC first or recreate the Host before turning the flag off"
fi
@@ -499,7 +548,8 @@ if [ "$COMMAND" = inventory ]; then
else
echo "DESKTOP_HOSTS=0"
fi
- if [ "$UPGRADE_BLOCKERS" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] ||
+ if [ "$UPGRADE_BLOCKERS" -gt 0 ] || [ "$UNREADABLE" -gt 0 ] || [ "$UNKNOWN" -gt 0 ] ||
+ [ "$ORPHAN_UNREADABLE" -gt 0 ] || { [ "$HOSTS" -gt 0 ] && [ "$TOTAL" -eq 0 ]; } ||
{ [ "$FLAG" = off ] && [ "$SPLIT_TREES" -gt 0 ]; } ||
[ "$FLAG" = invalid ] || [ "$FLAG" = unresolved ]; then
echo "HOST_LAYOUT_INVENTORY=BLOCKED"
@@ -509,15 +559,20 @@ if [ "$COMMAND" = inventory ]; then
exit 0
fi
+# A PVC nobody read is undecided whatever the flag says: ALREADY_OFF is
+# printed only when every PVC was read and none is split.
+UNDECIDED=$((UNREADABLE + UNVERIFIED + UNKNOWN + ORPHAN_UNREADABLE + ORPHAN_UNVERIFIED))
if [ "$FLAG" = off ]; then
if [ "$SPLIT_TREES" -gt 0 ]; then
echo "HOST_LAYOUT_VERIFY=FLAG_FALSE_WITH_SPLIT_TREES"
+ elif [ "$UNDECIDED" -gt 0 ]; then
+ echo "HOST_LAYOUT_VERIFY=FAIL"
else
echo "HOST_LAYOUT_VERIFY=ALREADY_OFF"
fi
exit 1
fi
-if [ "$TOTAL" -gt 0 ] && [ "$((SPLIT + REFUSED + RUNNING + UNREADABLE + UNVERIFIED + UNKNOWN + TEMPLATE_STALE))" -eq 0 ]; then
+if [ "$TOTAL" -gt 0 ] && [ "$((SPLIT + REFUSED + RUNNING + UNDECIDED + TEMPLATE_STALE))" -eq 0 ]; then
echo "HOST_LAYOUT_VERIFY=PASS"
exit 0
fi
diff --git a/deploy/scripts/host-layout-merge.test.mjs b/deploy/scripts/host-layout-merge.test.mjs
index 68688f376..515b99e51 100644
--- a/deploy/scripts/host-layout-merge.test.mjs
+++ b/deploy/scripts/host-layout-merge.test.mjs
@@ -32,10 +32,11 @@ const out = (s, code = 0) => { process.stdout.write(s); process.exit(code) }
if (!args.includes('--context=' + fx.context)) { process.stderr.write('missing --context'); process.exit(97) }
const rest = args.filter(a => !a.startsWith('--context=')).join(' ')
const statePath = process.env.FAKE_KUBECTL_LOG + '.pods.json'
-const pods = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, 'utf8')) : {}
+const pods = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, 'utf8')) : { ...fx.preexisting }
const save = () => fs.writeFileSync(statePath, JSON.stringify(pods))
if (rest === 'create -f -') {
const m = JSON.parse(stdin)
+ if (pods[m.metadata.name]) { process.stderr.write('Error from server (AlreadyExists): pods "' + m.metadata.name + '" already exists'); process.exit(1) }
const claim = m.spec.volumes[0].persistentVolumeClaim.claimName
pods[m.metadata.name] = claim
save()
@@ -185,7 +186,7 @@ function run(f, args) {
const kubectl = path.join(dir, 'kubectl')
writeFileSync(kubectl, FAKE_KUBECTL, { mode: 0o755 })
const fixture = path.join(dir, 'fixture.json')
- writeFileSync(fixture, JSON.stringify({ context: CONTEXT, responses: responses(f), checks: f.checks }))
+ writeFileSync(fixture, JSON.stringify({ context: CONTEXT, responses: responses(f), checks: f.checks, preexisting: f.preexisting ?? {} }))
const log = path.join(dir, 'calls.jsonl')
writeFileSync(log, '')
const res = spawnSync('bash', [SCRIPT, ...args], {
@@ -211,6 +212,7 @@ function row(out, pvcName) {
const verbs = calls => calls.map(c => c.args.filter(a => !a.startsWith('--context='))[0])
const creates = calls => calls.filter(c => verbs([c])[0] === 'create').map(c => JSON.parse(c.stdin))
+const claimOf = m => m.spec.volumes[0].persistentVolumeClaim.claimName
function assertReadOnlyCalls(calls) {
assert.ok(calls.length > 0, 'kubectl was called')
@@ -225,7 +227,7 @@ function assertCheckPodsOnly(calls, claims) {
assert.deepEqual(creates(calls).map(m => m.spec.volumes[0].persistentVolumeClaim.claimName).sort(), [...claims].sort())
}
-test('verify passes a merged fleet and checks only the PVC without a Pod', () => {
+test('verify passes a merged fleet and reads every PVC, pinning running Hosts to their node', () => {
const r = verify(fleet())
assert.equal(r.status, 0, r.stdout + r.stderr)
assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)
@@ -234,9 +236,11 @@ test('verify passes a merged fleet and checks only the PVC without a Pod', () =>
assert.match(row(r.stdout, 'gamma-workspace'), /host=gamma kind=DESKTOP state=ACTIVE result=POD_OK$/)
assert.match(r.stdout, /^COUNTS total=3 pod_ok=2 no_split=1 split=0 refused=0 running=0 unreadable=0 unverified=0 unknown=0 template_stale=0$/m)
assert.match(r.stdout, /^PVC_CONTENTS=NOT_INSPECTED$/m)
- assertCheckPodsOnly(r.calls, ['beta-workspace'])
+ assertCheckPodsOnly(r.calls, ['alpha-workspace', 'beta-workspace', 'gamma-workspace'])
+ const nodes = Object.fromEntries(creates(r.calls).map(m => [claimOf(m), m.spec.nodeName]))
+ assert.deepEqual(nodes, { 'alpha-workspace': 'node-a', 'beta-workspace': undefined, 'gamma-workspace': 'node-b' })
- const [pod] = creates(r.calls)
+ const pod = creates(r.calls).find(m => claimOf(m) === 'beta-workspace')
assert.equal(pod.spec.nodeName, undefined)
assert.equal(pod.spec.automountServiceAccountToken, false)
assert.equal(pod.spec.securityContext.fsGroup, undefined)
@@ -295,7 +299,8 @@ test('--no-pvc-pods starts no Pod: inventory reports UNVERIFIED, verify fails',
const ver = verify(fleet(), '--no-pvc-pods')
assert.equal(ver.status, 1, ver.stdout + ver.stderr)
- assert.match(row(ver.stdout, 'alpha-workspace'), /result=POD_OK$/)
+ // An init that exited 0 does not stand in for reading the tree.
+ assert.match(row(ver.stdout, 'alpha-workspace'), /result=UNVERIFIED$/)
assert.match(row(ver.stdout, 'beta-workspace'), /result=UNVERIFIED$/)
assert.match(ver.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
assertReadOnlyCalls(ver.calls)
@@ -402,8 +407,8 @@ test('verify fails a suspended Host whose PVC is still split or unreadable', ()
assert.ok(row(r.stdout, 'beta-workspace').includes(`result=${result}`), row(r.stdout, 'beta-workspace'))
assert.match(r.stdout, count)
assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
- // The check Pod is deleted whatever it printed.
- assertCheckPodsOnly(r.calls, ['beta-workspace'])
+ // Every check Pod is deleted whatever it printed.
+ assertCheckPodsOnly(r.calls, ['alpha-workspace', 'beta-workspace', 'gamma-workspace'])
}
})
@@ -422,6 +427,130 @@ test('an orphan split PVC is reported without blocking the verdict (T9)', () =>
assert.deepEqual(orphanPod.spec.imagePullSecrets, [{ name: 'clerum' }])
})
+test('r5 C1: a Host whose workspace PVC was not inspected fails both verdicts', () => {
+ // Witness: the unmodified fleet passes with all three claims read.
+ assert.equal(verify(fleet()).status, 0)
+
+ const missing = fleet()
+ missing.pvcs = missing.pvcs.filter(p => p.metadata.name !== 'beta-workspace')
+ const mv = verify(missing)
+ const mi = inventory(missing)
+ for (const r of [mv, mi]) {
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(row(r.stdout, 'beta-workspace'), /host=beta kind=STATELESS state=SUSPENDED result=UNKNOWN /)
+ assert.match(r.stdout, / unknown=1 /)
+ }
+ assert.match(mv.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+ assert.match(mi.stdout, /^HOST_LAYOUT_INVENTORY=BLOCKED$/m)
+
+ const renamed = fleet()
+ renamed.deployments[1].spec.template.spec.volumes[0].persistentVolumeClaim.claimName = 'beta-legacy'
+ renamed.pvcs[1] = { metadata: { name: 'beta-legacy', labels: { 'clerum.io/managed-by': 'host-context-controller', 'clerum.io/host': 'beta' } } }
+ const rv = verify(renamed)
+ assert.equal(rv.status, 1, rv.stdout + rv.stderr)
+ assert.match(row(rv.stdout, 'beta-legacy'), /host=beta kind=STATELESS state=SUSPENDED result=UNKNOWN /)
+ assert.match(row(rv.stdout, 'beta-workspace'), /host=beta kind=STATELESS state=SUSPENDED result=UNKNOWN /)
+ assert.match(rv.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+
+ const none = fleet()
+ none.pvcs = []
+ const ni = inventory(none)
+ assert.equal(ni.status, 1, ni.stdout + ni.stderr)
+ assert.match(ni.stdout, / unknown=3 /)
+ assert.match(ni.stdout, /^HOST_LAYOUT_INVENTORY=BLOCKED$/m)
+
+ // A PVC with no Host label (a SharedFileSystem claim) stays out of the verdict.
+ const sfs = fleet()
+ sfs.pvcs.push({ metadata: { name: 'shared-files', labels: { 'clerum.io/managed-by': 'host-context-controller' } } })
+ const sv = verify(sfs)
+ assert.equal(sv.status, 0, sv.stdout + sv.stderr)
+ assert.match(sv.stdout, /^IGNORED_PVC mcp-host\/shared-files /m)
+ assert.match(sv.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)
+})
+
+test('r5 C2: a check Pod name held by another run is neither reused nor deleted', () => {
+ const foreign = 'host-layout-check-beta-workspace'
+ const f = fleet()
+ f.preexisting = { [foreign]: 'other-run' }
+ const r = verify(f)
+ assert.equal(r.status, 0, r.stdout + r.stderr)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=PASS$/m)
+ const created = creates(r.calls).map(m => m.metadata.name)
+ const beta = creates(r.calls).find(m => claimOf(m) === 'beta-workspace')
+ assert.match(beta.metadata.name, /^host-layout-check-beta-workspace-[0-9a-f]{8}$/)
+ const deleted = r.calls.filter(c => verbs([c])[0] === 'delete').map(c => c.args[c.args.indexOf('pod') + 1])
+ // Witness: every Pod this run created was deleted, and nothing else.
+ assert.deepEqual(deleted.sort(), created.sort())
+ assert.ok(!deleted.includes(foreign))
+})
+
+test('r5 C3: an orphan PVC that could not be read fails both verdicts', () => {
+ const f = fleet()
+ f.pvcs.push(pvc('delta'))
+ f.checks['delta-workspace'] = { phase: 'Failed', output: 'read failed\n' }
+ const v = verify(f)
+ assert.equal(v.status, 1, v.stdout + v.stderr)
+ assert.match(row(v.stdout, 'delta-workspace'), /host=delta kind=ORPHAN state=NO_HOST result=UNREADABLE /)
+ assert.match(v.stdout, /^ORPHAN_UNDECIDED=1$/m)
+ assert.match(v.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+ const i = inventory(f)
+ assert.equal(i.status, 1, i.stdout + i.stderr)
+ assert.match(i.stdout, /^HOST_LAYOUT_INVENTORY=BLOCKED$/m)
+})
+
+test('r5 C4: verify reads the tree of a running Host and fails one still split', () => {
+ const f = fleet()
+ f.checks['alpha-workspace'].output = 'WILL_MERGE\n'
+ const r = verify(f)
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(row(r.stdout, 'alpha-workspace'), /host=alpha kind=STATEFUL state=ACTIVE result=SPLIT$/)
+ assert.match(r.stdout, / split=1 /)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+ const alpha = creates(r.calls).find(m => claimOf(m) === 'alpha-workspace')
+ assert.equal(alpha?.spec.nodeName, 'node-a')
+})
+
+test('r5 C5: flag off with a PVC left unread is FAIL, not ALREADY_OFF', () => {
+ const f = fleet()
+ f.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'false' }])
+ f.checks['beta-workspace'] = { phase: 'Failed', output: 'boom\n' }
+ const r = verify(f)
+ assert.equal(r.status, 1, r.stdout + r.stderr)
+ assert.match(row(r.stdout, 'beta-workspace'), /result=UNREADABLE /)
+ assert.match(r.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+
+ const unread = fleet()
+ unread.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'false' }])
+ const u = verify(unread, '--no-pvc-pods')
+ assert.equal(u.status, 1, u.stdout + u.stderr)
+ assert.match(row(u.stdout, 'beta-workspace'), /result=UNVERIFIED$/)
+ assert.match(u.stdout, /^HOST_LAYOUT_VERIFY=FAIL$/m)
+})
+
+test('r5 C10: HCC settings that may come from envFrom are unresolved, never defaults', () => {
+ const ns = fleet()
+ ns.hcc.spec.template.spec.containers[0].envFrom = [{ configMapRef: { name: 'hcc-env' } }]
+ const n = verify(ns)
+ assert.equal(n.status, 2, n.stdout + n.stderr)
+ assert.match(n.stderr, /cannot resolve CONTEXT_MAPPER_HOST_NAMESPACE/)
+ assertReadOnlyCalls(n.calls)
+
+ const literalNs = [{ name: 'CONTEXT_MAPPER_HOST_NAMESPACE', value: NS }]
+ const witness = fleet()
+ witness.hcc = hcc(literalNs)
+ assert.equal(verify(witness).status, 0)
+
+ const flag = fleet()
+ flag.hcc = hcc(literalNs)
+ flag.hcc.spec.template.spec.containers[0].envFrom = [{ secretRef: { name: 'hcc-env' } }]
+ const v = verify(flag)
+ assert.equal(v.status, 1, v.stdout + v.stderr)
+ assert.match(v.stdout, /^HOST_LAYOUT_VERIFY=FLAG_UNRESOLVED$/m)
+ const i = inventory(flag)
+ assert.equal(i.status, 1, i.stdout + i.stderr)
+ assert.match(i.stdout, /^HOST_LAYOUT_INVENTORY=BLOCKED$/m)
+})
+
test('read failures, an empty scope and a failed cleanup exit 2', () => {
const empty = fleet()
empty.hosts = []
@@ -468,10 +597,10 @@ function tree(build) {
return root
}
-function check(root) {
+function check(root, env = {}) {
const printed = spawnSync('bash', [SCRIPT, 'check-script', '--root', root], { encoding: 'utf8' })
assert.equal(printed.status, 0, printed.stderr)
- const res = spawnSync('/bin/sh', ['-c', printed.stdout], { encoding: 'utf8' })
+ const res = spawnSync('/bin/sh', ['-c', printed.stdout], { encoding: 'utf8', env: { ...process.env, ...env } })
assert.equal(res.stderr, '')
assert.equal(res.status, 0)
return res.stdout
@@ -514,6 +643,24 @@ test('check-script refuses an unlistable state/ like the merge init does', { ski
}
})
+// An `ls -l` mode string ending in `+` means a POSIX ACL, which the mode bits
+// do not account for. A PATH wrapper adds the marker to `ls -ldn` output only.
+test('r5 C9: an ACL on a directory the merge writes makes the check UNREADABLE', { skip: process.getuid() === 0 && 'root reads any mode' }, () => {
+ const bin = mkdtempSync(path.join(tmpdir(), 'host-layout-check-bin-'))
+ writeFileSync(path.join(bin, 'ls'), `#!/bin/sh
+if [ "$1" = -ldn ]; then /bin/ls "$@" | sed '1s/^\\([^ ]*\\)/\\1+/'; else exec /bin/ls "$@"; fi
+`, { mode: 0o755 })
+ const root = tree(r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`) })
+ try {
+ // Witness: the same tree without the marker is predicted to merge.
+ assert.equal(check(root), 'WILL_MERGE\n')
+ assert.equal(check(root, { PATH: `${bin}:${process.env.PATH}` }), `UNREADABLE ACL on ${root}: mode bits do not decide access\n`)
+ } finally {
+ rmSync(root, { recursive: true, force: true })
+ rmSync(bin, { recursive: true, force: true })
+ }
+})
+
test('check-script ends before the moves', () => {
const printed = spawnSync('bash', [SCRIPT, 'check-script'], { encoding: 'utf8' })
assert.equal(printed.status, 0, printed.stderr)