Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): supersede the stale split database of a Host back on stateful
Alfredo Lopeze31e4693ecb1
diff --git a/deploy/scripts/host-layout-merge.sh b/deploy/scripts/host-layout-merge.shindex 9d1f37104..3b6c7aa22 100755--- a/deploy/scripts/host-layout-merge.sh+++ b/deploy/scripts/host-layout-merge.sh@@ -61,6 +61,7 @@ usage_error() {# the section is the "# F<n>" block it ran in (F1 before the first);# - each "no signature" exit printed as NO_SPLIT_SIGNATURE;# - `[ -w ]` replaced by a mode-bit test, since the check mount is read-only;+# - the T5 rename of state/ replaced by WILL_SUPERSEDE;# - everything from the move function on replaced by WILL_MERGE.# Each rewrite must match the expected number of lines, so a changed merge# script stops here instead of producing a check that no longer matches it.@@ -68,7 +69,7 @@ usage_error() {# they are single-quoted on purpose.# shellcheck disable=SC2016print_check_script() {- local root=$1 line roots=0 sets=0 echoes=0 exits=0 sections=0 noops=0 writes=0 cut=0+ local root=$1 line roots=0 sets=0 echoes=0 exits=0 sections=0 noops=0 writes=0 supersedes=0 cut=0case "$root" in *"'"*) usage_error "--root must not contain a single quote" ;; esac[ -f "$MERGE_SCRIPT" ] || die "missing $MERGE_SCRIPT"while IFS= read -r line || [ -n "$line" ]; do@@ -134,6 +135,14 @@ EOFprintf '%s\n' ' writable "$entry" || fail_merge "not writable by uid $(id -u): $entry"'writes=$((writes + 1));;+ ' [ -w "$root" ] || fail_merge "not writable by uid $(id -u): $root"')+ printf '%s\n' ' phase=F4' ' writable "$root" || fail_merge "not writable by uid $(id -u): $root"' ' phase=F3'+ writes=$((writes + 1))+ ;;+ ' mv "$root/state" "$superseded"')+ printf '%s\n' ' echo WILL_SUPERSEDE'+ supersedes=$((supersedes + 1))+ ;;'move_to_root() {')printf '%s\n' 'echo WILL_MERGE'cut=1@@ -144,8 +153,8 @@ EOF;;esacdone <"$MERGE_SCRIPT"- local shape="root=$roots set=$sets echo=$echoes exit=$exits sections=$sections noops=$noops writes=$writes cut=$cut"- [ "$shape" = "root=1 set=1 echo=1 exit=1 sections=4 noops=3 writes=2 cut=1" ] ||+ local shape="root=$roots set=$sets echo=$echoes exit=$exits sections=$sections noops=$noops writes=$writes supersedes=$supersedes cut=$cut"+ [ "$shape" = "root=1 set=1 echo=1 exit=1 sections=4 noops=3 writes=3 supersedes=1 cut=1" ] ||die "$MERGE_SCRIPT has an unexpected shape ($shape); update print_check_script with it"}@@ -429,7 +438,7 @@ run_check() {first="$(head -n 1 "$WORK/check.log")"lines="$(wc -l <"$WORK/check.log" | tr -d ' ')"case "$first" in- WILL_MERGE | NO_SPLIT_SIGNATURE)+ WILL_MERGE | WILL_SUPERSEDE | NO_SPLIT_SIGNATURE)if [ "$lines" = 1 ]; then echo "$first"; else echo "UNREADABLE unexpected output: $(tr '\n' ' ' <"$WORK/check.log")"; fi;;'WILL_REFUSE_F'[1-4]' workspace layout merge refused: '*) tr '\n' ' ' <"$WORK/check.log" | sed 's/ $//'; echo ;;@@ -495,7 +504,7 @@ while IFS=$'\t' read -r type pvc host kind state detail _subpath pods nodes mergfiresult="$(run_check "$pvc" "$node" "$image" "$policy" "$secrets" </dev/null)"# verify names a tree the merge will still change SPLIT.- [ "$COMMAND" = inventory ] || [ "$result" != WILL_MERGE ] || result=SPLIT+ case "$COMMAND:$result" in verify:WILL_MERGE | verify:WILL_SUPERSEDE) result=SPLIT ;; esac[ "$merge_ok" -eq 0 ] || [ "$result" != NO_SPLIT_SIGNATURE ] || result=POD_OKfifi@@ -504,7 +513,7 @@ while IFS=$'\t' read -r type pvc host kind state detail _subpath pods nodes mergecho "PVC $HOST_NS/$pvc host=$host kind=ORPHAN state=$state result=$result"ORPHANS=$((ORPHANS + 1))case "$result" in- WILL_MERGE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;;+ WILL_MERGE | WILL_SUPERSEDE | SPLIT | WILL_REFUSE_*) ORPHAN_SPLIT=$((ORPHAN_SPLIT + 1)) ;;NO_SPLIT_SIGNATURE) ;;UNREADABLE*) ORPHAN_UNREADABLE=$((ORPHAN_UNREADABLE + 1)) ;;UNVERIFIED) ORPHAN_UNVERIFIED=$((ORPHAN_UNVERIFIED + 1)) ;;@@ -517,7 +526,7 @@ while IFS=$'\t' read -r type pvc host kind state detail _subpath pods nodes mergcase "$result" inPOD_OK) POD_OK=$((POD_OK + 1)) ;;NO_SPLIT_SIGNATURE) NO_SPLIT=$((NO_SPLIT + 1)) ;;- WILL_MERGE | SPLIT) SPLIT=$((SPLIT + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)) ;;+ WILL_MERGE | WILL_SUPERSEDE | SPLIT) SPLIT=$((SPLIT + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)) ;;WILL_REFUSE_*) REFUSED=$((REFUSED + 1)); SPLIT_TREES=$((SPLIT_TREES + 1)); UPGRADE_BLOCKERS=$((UPGRADE_BLOCKERS + 1)) ;;MERGE_REFUSED*) REFUSED=$((REFUSED + 1)) ;;MERGE_RUNNING) RUNNING=$((RUNNING + 1)) ;;diff --git a/deploy/scripts/host-layout-merge.test.mjs b/deploy/scripts/host-layout-merge.test.mjsindex 515b99e51..4ed99e0a1 100644--- a/deploy/scripts/host-layout-merge.test.mjs+++ b/deploy/scripts/host-layout-merge.test.mjs@@ -270,7 +270,23 @@ test('inventory reads every PVC, pinning running Hosts to their node', () => {assert.deepEqual(nodes, { 'alpha-workspace': 'node-a', 'beta-workspace': undefined, 'gamma-workspace': 'node-b' })})-test('inventory reports a Host switched to stateful before the upgrade as a blocker (T5)', () => {+test('inventory reports a Host switched to stateful before the upgrade as split, not a blocker (T5)', () => {+ const f = fleet()+ f.checks['alpha-workspace'].output = 'WILL_SUPERSEDE\n'+ const r = inventory(f)+ assert.equal(r.status, 0, r.stdout + r.stderr)+ assert.match(r.stdout, /^UPGRADE_BLOCKERS=0$/m)+ assert.match(r.stdout, / split=1 /)+ assert.match(r.stdout, /^HOST_LAYOUT_INVENTORY=OK$/m)+ assert.match(row(r.stdout, 'alpha-workspace'), /result=WILL_SUPERSEDE$/)+ // The same fleet with the flag off is a split tree the init would not reach.+ const off = fleet()+ off.checks['alpha-workspace'].output = 'WILL_SUPERSEDE\n'+ off.hcc = hcc([{ name: 'CONTEXT_MAPPER_WORKSPACE_LAYOUT_MERGE', value: 'false' }])+ assert.match(inventory(off).stdout, /^FLAG_FALSE_WITH_SPLIT_TREES=1$/m)+})++test('inventory still reports a refused tree as a blocker', () => {const f = fleet()f.checks['alpha-workspace'].output = `WILL_REFUSE_F2 ${F2_LINE}\n`const r = inventory(f)@@ -393,6 +409,7 @@ test('verify reads the merge init status of running Pods', () => {test('verify fails a suspended Host whose PVC is still split or unreadable', () => {const cases = [[{ output: 'WILL_MERGE\n' }, 'SPLIT', / split=1 /],+ [{ output: 'WILL_SUPERSEDE\n' }, 'SPLIT', / split=1 /],[{ output: `WILL_REFUSE_F2 ${F2_LINE}\n` }, `WILL_REFUSE_F2 ${F2_LINE}`, / refused=1 /],[{ output: '' }, 'UNREADABLE', / unreadable=1 /],[{ output: 'WILL_MERGE\nNO_SPLIT_SIGNATURE\n' }, 'UNREADABLE', / unreadable=1 /],@@ -616,7 +633,8 @@ test('check-script classifies split, foreign and refused trees', () => {['symlinked state', r => { mkdirSync(`${r}/elsewhere`); file(`${r}/elsewhere/state.db`); symlinkSync('elsewhere', `${r}/state`); mkdirSync(`${r}/workspace`) }, ['elsewhere', 'state', 'workspace'], 'NO_SPLIT_SIGNATURE\n'],['workspace missing', r => { mkdirSync(`${r}/state`); file(`${r}/state/state.db`) }, ['state'], 'NO_SPLIT_SIGNATURE\n'],['stateful root', r => { file(`${r}/state.db`); file(`${r}/SOUL.md`) }, ['SOUL.md', 'state.db'], 'NO_SPLIT_SIGNATURE\n'],- ['T5', r => { file(`${r}/state.db`); mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`) }, ['state', 'state.db', 'workspace'], `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ROOT, ROOT/state and ROOT/workspace\n`],+ ['T5', r => { file(`${r}/state.db`); mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); file(`${r}/workspace/SOUL.md`); file(`${r}/SOUL.md`) }, ['SOUL.md', 'state', 'state.db', 'workspace'], 'WILL_SUPERSEDE\n'],+ ['root and workspace databases', r => { file(`${r}/state.db`); mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); file(`${r}/workspace/state.db`) }, ['state', 'state.db', 'workspace'], `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ROOT, ROOT/state and ROOT/workspace\n`],['read-only directory to move', r => { mkdirSync(`${r}/state`); mkdirSync(`${r}/workspace`); file(`${r}/state/state.db`); mkdirSync(`${r}/workspace/users`); chmodSync(`${r}/workspace/users`, 0o555) }, ['state', 'workspace'], 'WILL_REFUSE_F4 workspace layout merge refused: not writable by uid UID: ROOT/workspace/users\n'],]const uid = String(process.getuid())diff --git a/deploy/scripts/workspace-layout-merge.sh b/deploy/scripts/workspace-layout-merge.shindex 2796e9c4d..a065fa8ce 100755--- a/deploy/scripts/workspace-layout-merge.sh+++ b/deploy/scripts/workspace-layout-merge.sh@@ -70,6 +70,20 @@ for dir in "$root" "$root/state" "$root/workspace"; dodonedone+# Superseded split (T5): a stateless start moves the root database into+# state/, so a root state.db comes from a stateful run after the last+# stateless run and state/ holds the older history. Renaming state/ removes+# the signature, so later starts exit 0.+if has_main "$root" && { has_main "$root/state" || has_journal "$root/state"; } &&+ ! has_main "$root/workspace" && ! has_journal "$root/workspace"; then+ [ -w "$root" ] || fail_merge "not writable by uid $(id -u): $root"+ superseded="$root/state.db.superseded-$(date -u +%Y%m%dT%H%M%SZ)"+ ! present "$superseded" || fail_merge "destination exists: $superseded"+ mv "$root/state" "$superseded"+ sync+ exit 0+fi+# F2live=0root_live=0diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex a744de541..c49f0f750 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.402",+ "version": "0.1.403","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.402",+ "version": "0.1.403","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 9ca41dd95..4bcd62ee5 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.402",+ "version": "0.1.403","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/statelessDeployment.ts b/host-context-controller/src/statelessDeployment.tsindex 30d57b64b..c530ddb5f 100644--- a/host-context-controller/src/statelessDeployment.ts+++ b/host-context-controller/src/statelessDeployment.ts@@ -38,6 +38,14 @@ function shellSingleQuote(value: string): string {* symlink or not a directory (an agent can create those on a stateful Host,* so they must not stop it).*+ * A tree with a root `state.db` main file, a database under `state/` and+ * none under `workspace/` belongs to a Host that went back to stateful after+ * its last stateless run (T5). The root database is the current one; the+ * whole `state/` directory is renamed to `state.db.superseded-<UTC ts>` in+ * one step, `workspace/` stays where it is, and the script exits 0. The+ * renamed directory starts with `state.db`, so the workspace file guards+ * deny it to agent tools.+ ** Every refusal prints one line and exits 1 before the first move:* F1 `state/` or `workspace/` cannot be listed and searched, or a database* entry is a symlink or not a regular file;@@ -127,6 +135,20 @@ for dir in "$root" "$root/state" "$root/workspace"; dodonedone+# Superseded split (T5): a stateless start moves the root database into+# state/, so a root state.db comes from a stateful run after the last+# stateless run and state/ holds the older history. Renaming state/ removes+# the signature, so later starts exit 0.+if has_main "$root" && { has_main "$root/state" || has_journal "$root/state"; } &&+ ! has_main "$root/workspace" && ! has_journal "$root/workspace"; then+ [ -w "$root" ] || fail_merge "not writable by uid $(id -u): $root"+ superseded="$root/state.db.superseded-$(date -u +%Y%m%dT%H%M%SZ)"+ ! present "$superseded" || fail_merge "destination exists: $superseded"+ mv "$root/state" "$superseded"+ sync+ exit 0+fi+# F2live=0root_live=0diff --git a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.tsindex a67837e82..6e07cf6cb 100644--- a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts+++ b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts@@ -104,7 +104,14 @@ function merge(root: string, env?: NodeJS.ProcessEnv): RunResult {} else {expect(res.status, res.stderr).toBe(0)expect(before, `${root} merged without being a directory`).not.toBeNull()- const expected = rootListing(root) === before ? 'NO_SPLIT_SIGNATURE' : 'WILL_MERGE'+ const after = rootListing(root)+ // A merge removes workspace/; superseding state/ leaves workspace/ in place.+ const superseded =+ before!.split('\0').includes('state') &&+ !after.split('\0').includes('state') &&+ after.split('\0').includes('workspace')+ const expected =+ after === before ? 'NO_SPLIT_SIGNATURE' : superseded ? 'WILL_SUPERSEDE' : 'WILL_MERGE'expect(predicted).toBe(`${expected}\n`)}return res@@ -119,6 +126,7 @@ afterAll(() => {'WILL_REFUSE_F2','WILL_REFUSE_F3','WILL_REFUSE_F4',+ 'WILL_SUPERSEDE',])})@@ -162,6 +170,47 @@ function splitRoot(): string {return root}+/** A committed SQLite database with `rows` messages, closed cleanly. */+function writeDb(dbPath: string, rows: number): void {+ const writer = `+ const { DatabaseSync } = require('node:sqlite')+ const db = new DatabaseSync(${JSON.stringify(dbPath)})+ db.exec('CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL)')+ const insert = db.prepare('INSERT INTO messages (body) VALUES (?)')+ for (let i = 0; i < ${rows}; i++) insert.run('stateful ' + i)+ db.close()+ `+ const res = spawnSync(process.execPath, ['-e', writer], { encoding: 'utf8' })+ expect(res.status, res.stderr).toBe(0)+}++const STATEFUL_ROWS = 7++/**+ * A stateless Host switched back to stateful before the upgrade (T5): the+ * stateful Pod serves a new root state.db while the older history stays under+ * state/ next to workspace/. The two state/ shapes are the ones read on the+ * clerum PVCs on 2026-10-17.+ */+function supersededSplitRoot(stateShape: 'main only' | 'empty WAL and SHM'): string {+ const root = splitRoot()+ if (stateShape === 'main only') {+ rmSync(join(root, 'state', 'state.db-wal'))+ rmSync(join(root, 'state', 'state.db-shm'))+ } else {+ writeFileSync(join(root, 'state', 'state.db-wal'), '')+ }+ writeDb(join(root, 'state.db'), STATEFUL_ROWS)+ return root+}++function supersededName(root: string): string {+ const names = readdirSync(root).filter(n => n.startsWith('state.db.superseded-'))+ expect(names).toHaveLength(1)+ expect(names[0]).toMatch(/^state\.db\.superseded-\d{8}T\d{6}Z$/)+ return names[0]+}+function copyTree(root: string): string {const copy = tempDir('clerum-merge-copy-')rmSync(copy, { recursive: true })@@ -238,16 +287,95 @@ describe('workspace layout merge', () => {).toBe(`# shellcheck shell=sh\n${buildWorkspaceLayoutMergeScript()}`)})- it('predicts a refusal for a Host switched to stateful before the upgrade (T5)', () => {- // A v0.10/v0.11 stateless Host switched to stateful: new chats in the- // root database, the older history still under state/.- const root = splitRoot()- cpSync(join(root, 'state', 'state.db'), join(root, 'state.db'))- expect(predictMerge(root)).toBe(- `WILL_REFUSE_F2 workspace layout merge refused: more than one live SQLite database across ${root}, ${root}/state and ${root}/workspace\n`- )- expect(merge(root).status).toBe(1)- })+ // A root state.db exists only after a stateful run that followed the last+ // stateless run (the base init moves it into state/ on every stateless+ // start), so state/ holds the older history. The base init kept the root+ // database and backed the state/ one up; the merge keeps the root one and+ // renames state/ whole, so no signature is left for a later start.+ it.each([+ { name: 'state/state.db only', shape: 'main only' as const, backup: false },+ {+ name: 'state/state.db with an empty WAL and SHM',+ shape: 'empty WAL and SHM' as const,+ backup: false,+ },+ { name: 'state/ with a .pre-*.bak copy', shape: 'main only' as const, backup: true },+ ])(+ 'supersedes the stale state/ of a Host that went back to stateful (T5, $name)',+ ({ shape, backup }) => {+ const root = supersededSplitRoot(shape)+ if (backup) writeFileSync(join(root, 'state', 'state.db.pre-1.bak'), 'older backup')+ // The stateful run rewrote personalization files at the root too, so+ // the same names exist under workspace/.+ writeFileSync(join(root, 'SOUL.md'), 'stateful copy')+ const before = snapshot(root)+ expect(before['workspace/SOUL.md']).toBeDefined()+ expect(before['SOUL.md']).not.toBe(before['workspace/SOUL.md'])+ expect(before['state/state.db']).toBeDefined()++ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ const after = snapshot(root)+ const superseded = supersededName(root)+ const expected: Record<string, string> = {}+ for (const [key, value] of Object.entries(before)) {+ const moved = key === 'state' || key.startsWith('state/')+ expected[moved ? `${superseded}${key.slice('state'.length)}` : key] = value+ }+ expect(after).toEqual(expected)+ if (backup) expect(after[`${superseded}/state.db.pre-1.bak`]).toBeDefined()++ // A second start finds no signature and changes nothing.+ expect(merge(root)).toEqual({ status: 0, stderr: '' })+ expect(snapshot(root)).toEqual(after)+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: STATEFUL_ROWS, check: 'ok' })+ },+ 15_000+ )++ it('refuses to supersede state/ when the PVC root is not writable (F4)', () => {+ const root = supersededSplitRoot('main only')+ chmodSync(root, 0o555)+ const before = snapshot(root)++ const refused = merge(root)++ chmodSync(root, 0o755)+ expect(refused).toEqual({+ status: 1,+ stderr: `workspace layout merge refused: not writable by uid ${process.getuid!()}: ${root}\n`,+ })+ expect(snapshot(root)).toEqual(before)+ // Witness: once the root is writable the same tree is superseded.+ expect(merge(root)).toEqual({ status: 0, stderr: '' })+ expect(readdirSync(join(root, supersededName(root)))).toContain('state.db')+ }, 15_000)++ it('merges a superseded tree back after a rollback re-splits it with the base init', () => {+ // No root personalization file here: the base init refuses a name present+ // at the root and under workspace/, before and after this PR.+ const root = supersededSplitRoot('main only')+ expect(merge(root).status).toBe(0)+ const superseded = supersededName(root)+ const supersededTree = snapshot(root)++ baseInit(root)+ // Witness: the base init really split the tree again.+ expect(snapshot(root)['state/state.db']).toBe(supersededTree['state.db'])+ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ // Roll-forward: the stale history and the served database are back at the+ // root, and the files the stateless run left under workspace/ are merged.+ const after = snapshot(root)+ const expected: Record<string, string> = {}+ for (const [key, value] of Object.entries(supersededTree)) {+ if (key === 'workspace') continue+ expected[key.startsWith('workspace/') ? key.slice('workspace/'.length) : key] = value+ }+ expect(after).toEqual(expected)+ expect(after[`${superseded}/state.db`]).toBe(supersededTree[`${superseded}/state.db`])+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: STATEFUL_ROWS, check: 'ok' })+ }, 15_000)it('merges a split tree with an uncheckpointed WAL back to the PVC root', () => {const root = splitRoot()@@ -317,9 +445,16 @@ describe('workspace layout merge', () => {message: 'SQLite file is not a regular file',},{- name: 'a second live database at the root (F2)',- prepare: (root: string) => writeFileSync(join(root, 'state.db'), 'second database'),- undo: (root: string) => rmSync(join(root, 'state.db')),+ // A root database is superseded over state/ only while workspace/ has none.+ name: 'live databases at the root and under workspace (F2)',+ prepare: (root: string) => {+ writeFileSync(join(root, 'state.db'), 'second database')+ writeFileSync(join(root, 'workspace', 'state.db'), 'third database')+ },+ undo: (root: string) => {+ rmSync(join(root, 'state.db'))+ rmSync(join(root, 'workspace', 'state.db'))+ },message: 'more than one live SQLite database',},{