Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): one storage layout for every lifecycle
Alfredo Lopez4086bf40b553
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 35ab630fe..dabaeca28 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.381",+ "version": "0.1.382","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.381",+ "version": "0.1.382","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 6a9df9017..4f0e9abfe 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.381",+ "version": "0.1.382","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts b/host-context-controller/src/hostReconciler.grokHostRuntime.test.tsindex 9811f0da9..0433d7cb9 100644--- a/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts+++ b/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts@@ -338,6 +338,8 @@ describe('HostReconciler Grok Host runtime gating', () => {'MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE','MCP_HOST_RUNTIME_AUTH_STATE_DIR','MCP_HOST_GATEWAY_URL',+ 'CLERUM_SESSION_STORE',+ 'CLERUM_SESSION_DB_DIR',])})diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 742d84f31..f11f4740b 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -61,7 +61,7 @@ import {mcpHostRuntimeTokenSecretName,} from './secretFactory'import {- buildWorkspaceLayoutInitContainer,+ buildWorkspaceLayoutMergeContainer,decodeJwtExpMs,effectiveRotateBeforeMs,resolveStatelessImagePullPolicy,@@ -426,9 +426,27 @@ const LEGACY_EMPTY_RUNTIME_TOKEN_REVISION = createHash('sha256')const WORKFLOW_TOKEN_MOUNT_PATH = '/var/run/clerum/workflow-tokens'const MCP_HOST_WORKFLOW_CONTROL_TOKEN_FILE_PATH = `${WORKFLOW_TOKEN_MOUNT_PATH}/${MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY}`const MCP_HOST_RUNTIME_AUTH_STATE_PATH = '/var/run/clerum/workflow-auth'-// ─── Stateless lifecycle (Stage 2) ─────────────────────────────────────────-/** Durable session-state dir, mounted from the workspace PVC's state/ subPath. */-const STATE_MOUNT_PATH = '/var/lib/clerum/state'+/**+ * Restricted-PSA-compliant container securityContext shared by the mcp-host+ * container and the workspace-layout-merge initContainer. MCC shared-tenant+ * namespaces (mcp-host-<slug>) enforce the `restricted` PodSecurity standard;+ * without these fields the ReplicaSet gets FailedCreate and never schedules a+ * pod. UID/GID 1001 = the image's baked-in `nodejs` user; it must match the+ * pod-level securityContext and the existing workspace PVC ownership so+ * re-rolls don't EACCES. readOnlyRootFilesystem is intentionally NOT set —+ * mcp-host writes outside its mounted /tmp + workspace volumes and restricted+ * PSA does not require it.+ */+function hostContainerSecurityContext(): k8s.V1SecurityContext {+ return {+ allowPrivilegeEscalation: false,+ runAsNonRoot: true,+ runAsUser: 1001,+ runAsGroup: 1001,+ capabilities: { drop: ['ALL'] },+ seccompProfile: { type: 'RuntimeDefault' },+ }+}const log = hccLogger.child({ module: 'host-reconciler' })function throwCleanupFailures(failures: unknown[], message: string): void {@@ -3576,7 +3594,6 @@ export class HostReconciler {'/tmp',WORKFLOW_TOKEN_MOUNT_PATH,MCP_HOST_RUNTIME_AUTH_STATE_PATH,- ...(isStateless ? [STATE_MOUNT_PATH] : []),]const contextMounts = mounts.filter(m => {const reason = HostReconciler.contextMountPathRejectionReason(m.mountPath, reservedMountPaths)@@ -3649,11 +3666,16 @@ export class HostReconciler {if (grokExecutionEnabled) {env.push({ name: 'MCP_HOST_GROK_SUBSCRIPTION_ENABLED', value: 'true' })}+ // One storage layout for every lifecycle (issue #825): the conversation+ // database lives at the workspace PVC root, so a lifecycle flip changes+ // compute only and never where the history is read from.+ env.push(+ { name: 'CLERUM_SESSION_STORE', value: 'sqlite' },+ { name: 'CLERUM_SESSION_DB_DIR', value: workspacePath }+ )if (isStateless) {env.push({ name: 'CLERUM_STATELESS_LIFECYCLE', value: 'true' },- { name: 'CLERUM_SESSION_STORE', value: 'sqlite' },- { name: 'CLERUM_SESSION_DB_DIR', value: STATE_MOUNT_PATH },// Stage 3: the heartbeat emitter requires its own pod UID so HCC// can discard heartbeats from stale pods after a wake.{@@ -3765,13 +3787,10 @@ export class HostReconciler {// (event + resync): the state lives in the CRD status, so neither a// routine reconcile nor an HCC restart resurrects it.replicas: isStateless && effectiveLifecycle.state === 'suspended' ? 0 : 1,- strategy: {- type: 'RollingUpdate',- rollingUpdate: {- maxSurge: 0,- maxUnavailable: 1,- },- },+ // Recreate: the old Pod is gone before the new one starts, so two+ // Pods never share the RWO workspace PVC (a RollingUpdate with+ // maxSurge 0 still starts the new Pod while the old one terminates).+ strategy: { type: 'Recreate' },selector: {matchLabels: { app: host.name },},@@ -3795,12 +3814,16 @@ export class HostReconciler {imagePullSecrets: config.hostImagePullSecretName? [{ name: config.hostImagePullSecretName }]: undefined,- // Stage 1.2: stateless Hosts migrate the workspace PVC to the- // dual-subPath layout before mcp-host starts. Non-stateless pods- // keep the pre-stateless shape untouched.- ...(isStateless- ? { initContainers: [buildWorkspaceLayoutInitContainer(image, imagePullPolicy)] }- : {}),+ // Every Host merges a PVC split by the earlier stateless layout+ // back to the root layout before mcp-host starts; on any other+ // tree the init exits 0 without touching it.+ initContainers: [+ buildWorkspaceLayoutMergeContainer(+ image,+ imagePullPolicy,+ hostContainerSecurityContext()+ ),+ ],containers: [{name: 'mcp-host',@@ -3810,14 +3833,7 @@ export class HostReconciler {envFrom: [{ configMapRef: { name: config.hostConfigMapName } }],env,volumeMounts: [- ...(isStateless- ? [- // Same PVC mounted twice: workspace/ at the workspace- // path, state/ at the durable session-state dir.- { name: 'workspace', mountPath: workspacePath, subPath: 'workspace' },- { name: 'workspace', mountPath: STATE_MOUNT_PATH, subPath: 'state' },- ]- : [{ name: 'workspace', mountPath: workspacePath }]),+ { name: 'workspace', mountPath: workspacePath },{name: 'mcp-host-runtime-tokens',mountPath: WORKFLOW_TOKEN_MOUNT_PATH,@@ -3859,24 +3875,7 @@ export class HostReconciler {cpu: resources.limits.cpu,},},- // Restricted-PSA-compliant container securityContext. MCC- // shared-tenant namespaces (mcp-host-<slug>) enforce the- // `restricted` PodSecurity standard; without these fields the- // ReplicaSet gets FailedCreate and never schedules a pod.- // readOnlyRootFilesystem is intentionally NOT set — mcp-host- // writes outside its mounted /tmp + workspace volumes and- // restricted PSA does not require it.- securityContext: {- allowPrivilegeEscalation: false,- runAsNonRoot: true,- // UID/GID 1001 = the image's baked-in `nodejs` user. Must- // match the pod-level securityContext above and the existing- // workspace PVC ownership so re-rolls don't EACCES.- runAsUser: 1001,- runAsGroup: 1001,- capabilities: { drop: ['ALL'] },- seccompProfile: { type: 'RuntimeDefault' },- },+ securityContext: hostContainerSecurityContext(),},],securityContext,diff --git a/host-context-controller/src/statelessDeployment.ts b/host-context-controller/src/statelessDeployment.tsindex a21f3c573..3948f7909 100644--- a/host-context-controller/src/statelessDeployment.ts+++ b/host-context-controller/src/statelessDeployment.ts@@ -1,7 +1,7 @@/*** Pure, reconciler-state-free helpers for the stateless Host lifecycle:* image pull-policy resolution (Stage 6, W5), runtime-token JWT metadata- * inspection, and the Stage 1.2 workspace-layout initContainer.+ * inspection, and the workspace-layout merge initContainer every Host runs.*/import * as k8s from '@kubernetes/client-node'import { config } from './config'@@ -22,172 +22,167 @@ function shellSingleQuote(value: string): string {return `'${value.replace(/'/g, `'"'"'`)}'`}/**- * Idempotent workspace-layout migration run before mcp-host starts on- * STATELESS Hosts only (Stage 1.2). The initContainer mounts the workspace- * PVC at its root and reorganizes it into the dual-subPath layout:- * workspace/ → agent workspace (mounted at the workspace path)- * state/ → durable session state (mounted at /var/lib/clerum/state)- * Legacy content at the PVC root moves into workspace/; legacy SQLite files- * (state.db + WAL/SHM) move into state/. Every move is a same-filesystem- * rename and `sync` flushes the directory updates.+ * Init script run before mcp-host on every Host (issue #825). Every Host now+ * mounts the workspace PVC root at its workspace path and keeps the+ * conversation database at `<root>/state.db`. A PVC that a stateless Host+ * split with the earlier `workspace-layout-init` (agent files in+ * `workspace/`, the database in `state/`) is merged back to that layout.*- * Round-trip resolution (Addendum 6 item 6): a stateful<->stateless round-trip- * on the shared PVC can leave BOTH a root state.db (the freshest data, from the- * stateful life the operator just left) AND an earlier stateless life's- * state/state.db. Newest-source-wins: the older destination is preserved (never- * deleted) as state/<db>.pre-<timestamp>.bak, the fresher root source then- * migrates in via the normal path, and the resolution is logged loudly. This is- * genuinely idempotent across round-trips — a post-migration run (no source) is- * a no-op, and each later collision backs up under a NEW unique suffix.+ * The merge runs only when `state/` is a real directory holding a+ * `state.db*` entry, the signature the earlier init leaves. Any other tree+ * (stateful, desktop, empty, or a user folder named `state/` or+ * `workspace/`) exits 0 untouched.*- * Genuinely ambiguous states still fail before the first move, preserving every- * source for operator recovery: SQLite sources SPLIT across root and workspace,- * a workspace SQLite source colliding with the durable destination, duplicate- * non-SQLite entries, symlinks, and non-directory managed paths.- * lost+found stays put: it is root-owned on ext4 volumes and never user data.- * No chown/chmod — the container runs as the same uid/fsGroup (1001) that owns- * the PVC content.+ * Every refusal prints one line and exits 1 before the first move:+ * F1 `state` or `workspace` is a symlink or not a directory, or a+ * `state.db*` entry is a symlink;+ * F2 more than one live database (a `state.db` main file or its journals)+ * across the root, `state/` and `workspace/`, except the root journals+ * a stopped merge leaves next to the remaining `state/state.db`;+ * F3 a name to move already exists at the root, or exists under both+ * `state/` and `workspace/` (an empty source directory is removed+ * instead), or `workspace/lost+found` would keep `workspace/` alive;+ * F4 the root, `state/` or `workspace/` is not writable.+ * Moves: `workspace/*`, then `state/*` except `state.db`, then+ * `state/state.db` last, so a stopped merge keeps the signature and the+ * next start finishes it. Symlinks are renamed, never followed. The script+ * never opens the database and deletes nothing but the emptied directories.*/-export function buildWorkspaceLayoutInitScript(root = WORKSPACE_PVC_ROOT_MOUNT_PATH): string {+export function buildWorkspaceLayoutMergeScript(root = WORKSPACE_PVC_ROOT_MOUNT_PATH): string {return `set -euroot=${shellSingleQuote(root)}-fail_layout() {- echo "workspace layout migration unsafe: $1" >&2+fail_merge() {+ echo "workspace layout merge refused: $1" >&2exit 1}+present() {+ [ -e "$1" ] || [ -L "$1" ]+}canonical_dir() {(CDPATH= cd -P "$1" 2>/dev/null && pwd -P)}assert_managed_dir() {path="$1"label="$2"- [ ! -L "$path" ] || fail_layout "$label directory is a symlink: $path"- [ -d "$path" ] || fail_layout "$label path is not a directory: $path"- resolved="$(canonical_dir "$path")" || fail_layout "cannot resolve $label directory: $path"+ [ ! -L "$path" ] || fail_merge "$label directory is a symlink: $path"+ [ -d "$path" ] || fail_merge "$label path is not a directory: $path"+ resolved="$(canonical_dir "$path")" || fail_merge "cannot resolve $label directory: $path"case "$resolved" in"$root_real"/*) ;;- *) fail_layout "$label directory escapes the workspace PVC root: $path" ;;+ *) fail_merge "$label directory escapes the workspace PVC root: $path" ;;esac}+is_empty_dir() {+ [ -d "$1" ] && [ ! -L "$1" ] && [ -z "$(ls -A "$1")" ]+}+has_main() {+ present "$1/state.db"+}+has_journal() {+ present "$1/state.db-wal" || present "$1/state.db-shm"+}-[ ! -L "$root" ] || fail_layout "workspace PVC root is a symlink: $root"-[ -d "$root" ] || fail_layout "workspace PVC root is not a directory: $root"-root_real="$(canonical_dir "$root")" || fail_layout "cannot resolve workspace PVC root: $root"-[ "$root_real" != "/" ] || fail_layout "workspace PVC root resolves to /"+[ ! -L "$root" ] || fail_merge "workspace PVC root is a symlink: $root"+[ -d "$root" ] || fail_merge "workspace PVC root is not a directory: $root"+root_real="$(canonical_dir "$root")" || fail_merge "cannot resolve workspace PVC root: $root"+[ "$root_real" != "/" ] || fail_merge "workspace PVC root resolves to /"-# Validate both managed paths before creating either one. A symlink or regular-# file must never redirect a migration outside the mounted PVC.-for dir in workspace state; do- path="$root/$dir"- [ ! -L "$path" ] || fail_layout "$dir directory is a symlink: $path"- if [ -e "$path" ] && [ ! -d "$path" ]; then- fail_layout "$dir path is not a directory: $path"- fi-done-for dir in workspace state; do- path="$root/$dir"- [ -d "$path" ] || mkdir "$path"- assert_managed_dir "$path" "$dir"-done-# Validate every destination before the first rename: a later SQLite collision-# must not leave earlier root entries moved into a partial layout.-for entry in "$root"/* "$root"/.[!.]* "$root"/..?*; do- [ -e "$entry" ] || [ -L "$entry" ] || continue- base="$(basename "$entry")"- case "$base" in- workspace|state|lost+found) continue ;;- esac- [ ! -L "$entry" ] || fail_layout "legacy source is a symlink: $entry"- case "$base" in- state.db|state.db-wal|state.db-shm) continue ;;- esac- if [ -e "$root/workspace/$base" ] || [ -L "$root/workspace/$base" ]; then- echo "workspace layout migration collision: destination exists: $root/workspace/$base" >&2- exit 1- fi-done-root_sqlite_source=0-workspace_sqlite_source=0-for db in state.db state.db-wal state.db-shm; do- [ ! -L "$root/$db" ] || fail_layout "legacy SQLite source is a symlink: $root/$db"- [ ! -L "$root/workspace/$db" ] || fail_layout "workspace SQLite source is a symlink: $root/workspace/$db"- [ ! -L "$root/state/$db" ] || fail_layout "durable SQLite destination is a symlink: $root/state/$db"- if [ -e "$root/$db" ] || [ -L "$root/$db" ]; then root_sqlite_source=1; fi- if [ -e "$root/workspace/$db" ] || [ -L "$root/workspace/$db" ]; then workspace_sqlite_source=1; fi+# Split-layout signature: a real state/ directory holding a state.db* entry.+[ ! -L "$root/state" ] && [ -d "$root/state" ] || exit 0+signature=0+for entry in "$root/state"/state.db*; do+ if present "$entry"; then signature=1; fidone-if [ "$root_sqlite_source" -eq 1 ] && [ "$workspace_sqlite_source" -eq 1 ]; then- echo "workspace layout migration collision: SQLite artifacts are split across root and workspace" >&2- exit 1+[ "$signature" -eq 1 ] || exit 0++# F1+assert_managed_dir "$root/state" state+workspace_present=0+if present "$root/workspace"; then+ assert_managed_dir "$root/workspace" workspace+ workspace_present=1fi-# Newest-source-wins round-trip resolution (Addendum 6 item 6). A confirmed ROOT-# SQLite source colliding with an existing durable state/ destination is a-# stateful<->stateless round-trip: the operator just left the stateful life-# (root state.db is the freshest data) over an earlier stateless life's-# state/state.db. Preserve the older destination as state/<db>.pre-<ts>.bak-# (NOTHING is ever deleted), then let the fresher root source migrate in via the-# normal path below. The destination is proven non-symlink above.-# WHY positional implies temporal: every stateless boot MOVES the SQLite set out-# of root into state/ (the mv loops below), so root is left with no state.db. A-# root state.db can therefore only have been (re)created by a STATEFUL life that-# ran strictly AFTER the last stateless life — which is what makes "root source"-# equivalent to "newer source" here, with no timestamp read required.-root_dest_collision=0-if [ "$root_sqlite_source" -eq 1 ]; then- for db in state.db state.db-wal state.db-shm; do- if [ -e "$root/state/$db" ] || [ -L "$root/state/$db" ]; then root_dest_collision=1; fi+for dir in "$root" "$root/state" "$root/workspace"; do+ for entry in "$dir"/state.db*; do+ [ ! -L "$entry" ] || fail_merge "SQLite file is a symlink: $entry"done+done++# F2+live=0+root_live=0+workspace_live=0+if has_main "$root" || has_journal "$root"; then root_live=1; live=$((live + 1)); fi+if has_main "$root/state" || has_journal "$root/state"; then live=$((live + 1)); fi+if [ "$workspace_present" -eq 1 ] && { has_main "$root/workspace" || has_journal "$root/workspace"; }; then+ workspace_live=1+ live=$((live + 1))fi-if [ "$root_dest_collision" -eq 1 ]; then- # One timestamped backup namespace per collision; guard uniqueness so two- # round-trips within the same second (or a re-collision) never overwrite an- # existing backup.- bak_ts="$(date -u +%Y%m%d%H%M%S)"- bak_suffix="$bak_ts"- bak_seq=0- while [ -e "$root/state/state.db.pre-$bak_suffix.bak" ] \- || [ -e "$root/state/state.db-wal.pre-$bak_suffix.bak" ] \- || [ -e "$root/state/state.db-shm.pre-$bak_suffix.bak" ]; do- bak_seq=$((bak_seq + 1))- bak_suffix="$bak_ts-$bak_seq"- done- backed_up=""- for db in state.db state.db-wal state.db-shm; do- if [ -e "$root/state/$db" ]; then- mv "$root/state/$db" "$root/state/$db.pre-$bak_suffix.bak"- backed_up="$backed_up $db"+resume=0+if [ "$root_live" -eq 1 ] && ! has_main "$root" && has_main "$root/state" && [ "$workspace_live" -eq 0 ]; then+ resume=1+fi+if [ "$live" -gt 1 ] && [ "$resume" -eq 0 ]; then+ fail_merge "more than one live SQLite database across $root, $root/state and $root/workspace"+fi++# F3+if [ "$workspace_present" -eq 1 ]; then+ ! present "$root/workspace/lost+found" || fail_merge "workspace/lost+found would remain: $root/workspace/lost+found"+ for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?*; do+ present "$entry" || continue+ base="$(basename "$entry")"+ if present "$root/$base"; then+ if is_empty_dir "$entry" && [ -d "$root/$base" ] && [ ! -L "$root/$base" ]; then continue; fi+ fail_merge "destination exists: $root/$base"fidone- echo "workspace layout migration: newest-source-wins round-trip resolution -- backed up prior destination(s)$backed_up under state/*.pre-$bak_suffix.bak; migrating the fresher root SQLite source into state/" >&2fi-# A WORKSPACE SQLite source colliding with the durable destination is a genuinely-# ambiguous partial-migration state (not a clean round-trip): fail loud,-# preserving every source for operator recovery.-for db in state.db state.db-wal state.db-shm; do- if [ -e "$root/workspace/$db" ] || [ -L "$root/workspace/$db" ]; then- if [ -e "$root/state/$db" ] || [ -L "$root/state/$db" ]; then- echo "workspace layout migration collision: destination exists: $root/state/$db" >&2- exit 1- fi+for entry in "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do+ present "$entry" || continue+ base="$(basename "$entry")"+ if present "$root/$base"; then+ if is_empty_dir "$entry" && [ -d "$root/$base" ] && [ ! -L "$root/$base" ]; then continue; fi+ fail_merge "destination exists: $root/$base"+ fi+ if [ "$workspace_present" -eq 1 ] && present "$root/workspace/$base"; then+ fail_merge "$base exists under both state/ and workspace/"fidone-for entry in "$root"/* "$root"/.[!.]* "$root"/..?*; do- [ -e "$entry" ] || [ -L "$entry" ] || continue- base="$(basename "$entry")"- case "$base" in- workspace|state|lost+found) continue ;;- esac- assert_managed_dir "$root/workspace" workspace- [ ! -L "$entry" ] || fail_layout "legacy source became a symlink: $entry"- mv "$entry" "$root/workspace/$base"++# F4+for dir in "$root" "$root/state"; do+ [ -w "$dir" ] || fail_merge "not writable by uid $(id -u): $dir"done-for db in state.db state.db-wal state.db-shm; do- if [ -e "$root/workspace/$db" ]; then- assert_managed_dir "$root/workspace" workspace- assert_managed_dir "$root/state" state- [ ! -L "$root/workspace/$db" ] || fail_layout "workspace SQLite source became a symlink: $root/workspace/$db"- mv "$root/workspace/$db" "$root/state/$db"+if [ "$workspace_present" -eq 1 ]; then+ [ -w "$root/workspace" ] || fail_merge "not writable by uid $(id -u): $root/workspace"+fi++move_to_root() {+ entry="$1"+ base="$(basename "$entry")"+ assert_managed_dir "$root/state" state+ if [ "$workspace_present" -eq 1 ]; then assert_managed_dir "$root/workspace" workspace; fi+ if present "$root/$base"; then+ rmdir "$entry"+ else+ mv "$entry" "$root/$base"fi+}+if [ "$workspace_present" -eq 1 ]; then+ for entry in "$root/workspace"/* "$root/workspace"/.[!.]* "$root/workspace"/..?*; do+ present "$entry" || continue+ move_to_root "$entry"+ done+fi+for entry in "$root/state"/* "$root/state"/.[!.]* "$root/state"/..?*; do+ present "$entry" || continue+ [ "$(basename "$entry")" != state.db ] || continue+ move_to_root "$entry"done+if present "$root/state/state.db"; then move_to_root "$root/state/state.db"; fi+if [ "$workspace_present" -eq 1 ]; then rmdir "$root/workspace"; fi+rmdir "$root/state"sync`}@@ -313,33 +308,26 @@ export function effectiveRotateBeforeMs(expMs: number, issuedAtMs: number | null}/**- * Stage 1.2 initContainer for STATELESS Hosts: mounts the workspace PVC at- * its root and runs the idempotent layout migration (see- * WORKSPACE_LAYOUT_INIT_SCRIPT). Runs as the same non-root identity as the- * mcp-host container (uid/gid 1001, fsGroup 1001) so renames need no- * chown/chmod.+ * InitContainer that runs buildWorkspaceLayoutMergeScript() on the workspace+ * PVC root, on every Host and at every start. It uses the Host image and the+ * mcp-host container's securityContext, so it owns the files it renames+ * exactly as the main container does.*/-export function buildWorkspaceLayoutInitContainer(+export function buildWorkspaceLayoutMergeContainer(image: string,- imagePullPolicy: 'Always' | 'IfNotPresent' | 'Never'+ imagePullPolicy: 'Always' | 'IfNotPresent' | 'Never',+ securityContext: k8s.V1SecurityContext): k8s.V1Container {return {- name: 'workspace-layout-init',+ name: 'workspace-layout-merge',image,imagePullPolicy,- command: ['/bin/sh', '-c', buildWorkspaceLayoutInitScript()],+ command: ['/bin/sh', '-c', buildWorkspaceLayoutMergeScript()],volumeMounts: [{ name: 'workspace', mountPath: WORKSPACE_PVC_ROOT_MOUNT_PATH }],resources: {requests: { memory: '32Mi', cpu: '10m' },limits: { memory: '64Mi', cpu: '200m' },},- securityContext: {- allowPrivilegeEscalation: false,- runAsNonRoot: true,- runAsUser: 1001,- runAsGroup: 1001,- capabilities: { drop: ['ALL'] },- seccompProfile: { type: 'RuntimeDefault' },- },+ securityContext,}}diff --git a/host-context-controller/test/__fixtures__/base-workspace-layout-init.sh b/host-context-controller/test/__fixtures__/base-workspace-layout-init.shnew file mode 100644index 000000000..91591bb6f--- /dev/null+++ b/host-context-controller/test/__fixtures__/base-workspace-layout-init.sh@@ -0,0 +1,142 @@+#!/bin/sh+# Verbatim render of buildWorkspaceLayoutInitScript() at 87f962fb9 (base of+# issue #825), with the PVC root taken from "$1". Used only by tests to+# reproduce the split layout and the rollback round trip.+set -eu+root="$1"+fail_layout() {+ echo "workspace layout migration unsafe: $1" >&2+ exit 1+}+canonical_dir() {+ (CDPATH= cd -P "$1" 2>/dev/null && pwd -P)+}+assert_managed_dir() {+ path="$1"+ label="$2"+ [ ! -L "$path" ] || fail_layout "$label directory is a symlink: $path"+ [ -d "$path" ] || fail_layout "$label path is not a directory: $path"+ resolved="$(canonical_dir "$path")" || fail_layout "cannot resolve $label directory: $path"+ case "$resolved" in+ "$root_real"/*) ;;+ *) fail_layout "$label directory escapes the workspace PVC root: $path" ;;+ esac+}++[ ! -L "$root" ] || fail_layout "workspace PVC root is a symlink: $root"+[ -d "$root" ] || fail_layout "workspace PVC root is not a directory: $root"+root_real="$(canonical_dir "$root")" || fail_layout "cannot resolve workspace PVC root: $root"+[ "$root_real" != "/" ] || fail_layout "workspace PVC root resolves to /"++# Validate both managed paths before creating either one. A symlink or regular+# file must never redirect a migration outside the mounted PVC.+for dir in workspace state; do+ path="$root/$dir"+ [ ! -L "$path" ] || fail_layout "$dir directory is a symlink: $path"+ if [ -e "$path" ] && [ ! -d "$path" ]; then+ fail_layout "$dir path is not a directory: $path"+ fi+done+for dir in workspace state; do+ path="$root/$dir"+ [ -d "$path" ] || mkdir "$path"+ assert_managed_dir "$path" "$dir"+done+# Validate every destination before the first rename: a later SQLite collision+# must not leave earlier root entries moved into a partial layout.+for entry in "$root"/* "$root"/.[!.]* "$root"/..?*; do+ [ -e "$entry" ] || [ -L "$entry" ] || continue+ base="$(basename "$entry")"+ case "$base" in+ workspace|state|lost+found) continue ;;+ esac+ [ ! -L "$entry" ] || fail_layout "legacy source is a symlink: $entry"+ case "$base" in+ state.db|state.db-wal|state.db-shm) continue ;;+ esac+ if [ -e "$root/workspace/$base" ] || [ -L "$root/workspace/$base" ]; then+ echo "workspace layout migration collision: destination exists: $root/workspace/$base" >&2+ exit 1+ fi+done+root_sqlite_source=0+workspace_sqlite_source=0+for db in state.db state.db-wal state.db-shm; do+ [ ! -L "$root/$db" ] || fail_layout "legacy SQLite source is a symlink: $root/$db"+ [ ! -L "$root/workspace/$db" ] || fail_layout "workspace SQLite source is a symlink: $root/workspace/$db"+ [ ! -L "$root/state/$db" ] || fail_layout "durable SQLite destination is a symlink: $root/state/$db"+ if [ -e "$root/$db" ] || [ -L "$root/$db" ]; then root_sqlite_source=1; fi+ if [ -e "$root/workspace/$db" ] || [ -L "$root/workspace/$db" ]; then workspace_sqlite_source=1; fi+done+if [ "$root_sqlite_source" -eq 1 ] && [ "$workspace_sqlite_source" -eq 1 ]; then+ echo "workspace layout migration collision: SQLite artifacts are split across root and workspace" >&2+ exit 1+fi+# Newest-source-wins round-trip resolution (Addendum 6 item 6). A confirmed ROOT+# SQLite source colliding with an existing durable state/ destination is a+# stateful<->stateless round-trip: the operator just left the stateful life+# (root state.db is the freshest data) over an earlier stateless life's+# state/state.db. Preserve the older destination as state/<db>.pre-<ts>.bak+# (NOTHING is ever deleted), then let the fresher root source migrate in via the+# normal path below. The destination is proven non-symlink above.+# WHY positional implies temporal: every stateless boot MOVES the SQLite set out+# of root into state/ (the mv loops below), so root is left with no state.db. A+# root state.db can therefore only have been (re)created by a STATEFUL life that+# ran strictly AFTER the last stateless life — which is what makes "root source"+# equivalent to "newer source" here, with no timestamp read required.+root_dest_collision=0+if [ "$root_sqlite_source" -eq 1 ]; then+ for db in state.db state.db-wal state.db-shm; do+ if [ -e "$root/state/$db" ] || [ -L "$root/state/$db" ]; then root_dest_collision=1; fi+ done+fi+if [ "$root_dest_collision" -eq 1 ]; then+ # One timestamped backup namespace per collision; guard uniqueness so two+ # round-trips within the same second (or a re-collision) never overwrite an+ # existing backup.+ bak_ts="$(date -u +%Y%m%d%H%M%S)"+ bak_suffix="$bak_ts"+ bak_seq=0+ while [ -e "$root/state/state.db.pre-$bak_suffix.bak" ] || [ -e "$root/state/state.db-wal.pre-$bak_suffix.bak" ] || [ -e "$root/state/state.db-shm.pre-$bak_suffix.bak" ]; do+ bak_seq=$((bak_seq + 1))+ bak_suffix="$bak_ts-$bak_seq"+ done+ backed_up=""+ for db in state.db state.db-wal state.db-shm; do+ if [ -e "$root/state/$db" ]; then+ mv "$root/state/$db" "$root/state/$db.pre-$bak_suffix.bak"+ backed_up="$backed_up $db"+ fi+ done+ echo "workspace layout migration: newest-source-wins round-trip resolution -- backed up prior destination(s)$backed_up under state/*.pre-$bak_suffix.bak; migrating the fresher root SQLite source into state/" >&2+fi+# A WORKSPACE SQLite source colliding with the durable destination is a genuinely+# ambiguous partial-migration state (not a clean round-trip): fail loud,+# preserving every source for operator recovery.+for db in state.db state.db-wal state.db-shm; do+ if [ -e "$root/workspace/$db" ] || [ -L "$root/workspace/$db" ]; then+ if [ -e "$root/state/$db" ] || [ -L "$root/state/$db" ]; then+ echo "workspace layout migration collision: destination exists: $root/state/$db" >&2+ exit 1+ fi+ fi+done+for entry in "$root"/* "$root"/.[!.]* "$root"/..?*; do+ [ -e "$entry" ] || [ -L "$entry" ] || continue+ base="$(basename "$entry")"+ case "$base" in+ workspace|state|lost+found) continue ;;+ esac+ assert_managed_dir "$root/workspace" workspace+ [ ! -L "$entry" ] || fail_layout "legacy source became a symlink: $entry"+ mv "$entry" "$root/workspace/$base"+done+for db in state.db state.db-wal state.db-shm; do+ if [ -e "$root/workspace/$db" ]; then+ assert_managed_dir "$root/workspace" workspace+ assert_managed_dir "$root/state" state+ [ ! -L "$root/workspace/$db" ] || fail_layout "workspace SQLite source became a symlink: $root/workspace/$db"+ mv "$root/workspace/$db" "$root/state/$db"+ fi+done+syncdiff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex ef84ce9ea..807702749 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -1481,12 +1481,11 @@ describe('HostReconciler resource rollout bootstrap', () => {})describe('HostReconciler stateless lifecycle — buildDeployment replicas', () => {- it('pins replicas=1 and maxSurge=0 for a non-stateless host', () => {+ it('pins replicas=1 and the Recreate strategy for a non-stateless host', () => {const { reconciler } = createReconciler()const dep = reconciler.buildDeployment(makeHost())expect(dep.spec?.replicas).toBe(1)- expect(dep.spec?.strategy?.rollingUpdate?.maxSurge).toBe(0)- expect(dep.spec?.strategy?.rollingUpdate?.maxUnavailable).toBe(1)+ expect(dep.spec?.strategy).toEqual({ type: 'Recreate' })expect(dep.spec?.template.spec?.priorityClassName).toBeUndefined()})@@ -1866,22 +1865,22 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {)expect(hostReplacements).toHaveLength(1)const replacement = hostReplacements[0][0].body as k8s.V1Deployment+ // Witness: the flip really changed the lifecycle-specific template.expect(replacement.spec?.replicas).toBe(1)expect(replacement.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')- expect(- replacement.spec?.template.spec?.initContainers?.map(container => container.name)- ).toContain('workspace-layout-init')expect(envValue(replacement, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')- expect(replacement.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({- name: 'workspace',- mountPath: '/workspace',- subPath: 'workspace',- })- expect(replacement.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({- name: 'workspace',- mountPath: '/var/lib/clerum/state',- subPath: 'state',- })+ // Storage is identical across the flip: mounts, storage env and the init.+ const storage = (dep: k8s.V1Deployment) => ({+ mounts: dep.spec?.template.spec?.containers?.[0]?.volumeMounts,+ store: envValue(dep, 'CLERUM_SESSION_STORE'),+ dbDir: envValue(dep, 'CLERUM_SESSION_DB_DIR'),+ inits: dep.spec?.template.spec?.initContainers,+ })+ expect(JSON.stringify(storage(replacement))).toBe(JSON.stringify(storage(runningStateful)))+ expect(storage(replacement).dbDir).toBe('/workspace')+ expect(storage(replacement).inits?.map(container => container.name)).toEqual([+ 'workspace-layout-merge',+ ])})})@@ -1919,36 +1918,31 @@ describe('HostReconciler stateless lifecycle — env injection', () => {const created = hostDeploymentBody(appsApi, 'new-stateless-host')expect(created.spec?.replicas).toBe(1)expect(created.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')- expect(created.spec?.template.spec?.initContainers?.map(container => container.name)).toContain(- 'workspace-layout-init'- )+ expect(created.spec?.template.spec?.initContainers?.map(container => container.name)).toEqual([+ 'workspace-layout-merge',+ ])expect(envValue(created, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')expect(created.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({name: 'workspace',mountPath: '/workspace',- subPath: 'workspace',- })- expect(created.spec?.template.spec?.containers?.[0]?.volumeMounts).toContainEqual({- name: 'workspace',- mountPath: '/var/lib/clerum/state',- subPath: 'state',})})- it('injects the three stateless env vars when stateless is enabled', () => {+ it('injects the stateless env and the root storage env when stateless is enabled', () => {const { reconciler } = createReconciler()const dep = reconciler.buildDeployment(makeStatelessHost())expect(envValue(dep, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')expect(envValue(dep, 'CLERUM_SESSION_STORE')).toBe('sqlite')- expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')+ expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')})- it('does not inject the stateless env vars when stateless is off', () => {+ it('injects the root storage env but not the stateless env when stateless is off', () => {const { reconciler } = createReconciler()- const names = containerEnv(reconciler.buildDeployment(makeHost())).map(e => e.name)- expect(names).not.toContain('CLERUM_STATELESS_LIFECYCLE')- expect(names).not.toContain('CLERUM_SESSION_STORE')- expect(names).not.toContain('CLERUM_SESSION_DB_DIR')+ const dep = reconciler.buildDeployment(makeHost())+ expect(containerEnv(dep).map(e => e.name)).not.toContain('CLERUM_STATELESS_LIFECYCLE')+ // Witness: the same container carries the storage env every Host gets.+ expect(envValue(dep, 'CLERUM_SESSION_STORE')).toBe('sqlite')+ expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')})})@@ -4025,7 +4019,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {await reconciler.reconcile(host)const live = persistHostDeployment(appsApi, host, hostDeploymentBody(appsApi, host.name))const baseline = structuredClone(live().spec?.template)- expect(envValue(live(), 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')+ expect(envValue(live(), 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')for (let cycle = 1; cycle <= 2; cycle++) {cacheSynced = false@@ -4039,7 +4033,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {const deployment = live()expect(deployment.spec?.replicas).toBe(1)expect(deployment.spec?.template).toEqual(baseline)- expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')+ expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')expect(reconciler.getEffectiveLifecycle(host)).toMatchObject({stateless: true,suspensionBlocked: true,@@ -4107,7 +4101,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(deployment.spec?.replicas).toBe(0)expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')expect(envValue(deployment, 'CLERUM_SESSION_STORE')).toBe('sqlite')- expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')+ expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')const writes = lifecycleStatusWrites(customApi)expect(writes).toHaveLength(2)expect(writes[0].lifecycle?.state).toBe('suspended')@@ -4412,7 +4406,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(deployment.spec?.replicas).toBe(1)expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')expect(envValue(deployment, 'CLERUM_SESSION_STORE')).toBe('sqlite')- expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/var/lib/clerum/state')+ expect(envValue(deployment, 'CLERUM_SESSION_DB_DIR')).toBe('/workspace')})it('rejects stateless by default when CommunicationChannels reference the host', async () => {@@ -5124,62 +5118,96 @@ describe('HostReconciler stateless lifecycle — AP-1 fresh-read status writer',})})-describe('HostReconciler stateless lifecycle — initContainer and mounts', () => {- it('adds the workspace-layout initContainer with dual subPath mounts', () => {- const { reconciler } = createReconciler()- const dep = reconciler.buildDeployment(makeStatelessHost())+describe('HostReconciler storage layout — one shape for every lifecycle', () => {+ const kinds = [+ { kind: 'stateful', host: () => makeHost(), workspacePath: '/workspace' },+ { kind: 'stateless', host: () => makeStatelessHost(), workspacePath: '/workspace' },+ {+ kind: 'desktop',+ host: () => makeHost({ spec: { ...makeHost().spec, desktop: { browser: true } } }),+ workspacePath: '/config/workspace',+ },+ ]- const initContainers = dep.spec?.template?.spec?.initContainers- if (!initContainers || initContainers.length !== 1) {- throw new Error('expected exactly one initContainer on the stateless pod')+ it.each(kinds)(+ 'renders the root mount, storage env and merge init for a $kind Host',+ ({ host, workspacePath }) => {+ const { reconciler } = createReconciler()+ const dep = reconciler.buildDeployment(host())+ const podSpec = dep.spec?.template?.spec+ const main = podSpec?.containers?.[0]++ expect(main?.volumeMounts?.filter(mount => mount.name === 'workspace')).toEqual([+ { name: 'workspace', mountPath: workspacePath },+ ])+ expect(JSON.stringify(dep)).not.toContain('subPath')+ expect(envValue(dep, 'CLERUM_SESSION_STORE')).toBe('sqlite')+ expect(envValue(dep, 'CLERUM_SESSION_DB_DIR')).toBe(workspacePath)+ expect(dep.spec?.strategy).toEqual({ type: 'Recreate' })++ const inits = podSpec?.initContainers ?? []+ expect(inits.map(init => init.name)).toEqual(['workspace-layout-merge'])+ const [merge] = inits+ expect(merge.image).toBe(main?.image)+ expect(merge.imagePullPolicy).toBe(main?.imagePullPolicy)+ expect(merge.securityContext).toEqual(main?.securityContext)+ expect(merge.securityContext).toMatchObject({ runAsUser: 1001, runAsNonRoot: true })+ expect(merge.volumeMounts).toEqual([{ name: 'workspace', mountPath: '/mnt/workspace-root' }])+ expect(merge.command?.[2]).toContain('workspace layout merge refused')+ // A Pod's scheduling request counts its largest init container; dev+ // requests 25m CPU for a Host, so the merge stays at or below 25m/128Mi.+ expect(merge.resources?.requests).toEqual({ cpu: '10m', memory: '32Mi' })++ const names = [...(main?.env ?? []), ...(merge.env ?? [])].map(env => env.name)+ // Witness: the env list is populated with the storage env asserted above.+ expect(names).toContain('CLERUM_SESSION_DB_DIR')+ for (const withdrawn of [+ 'CLERUM_STORAGE_LAYOUT',+ 'CLERUM_STORAGE_STORE_ID',+ 'CLERUM_STORAGE_GENESIS',+ 'CLERUM_DB_BARRIER_MODE',+ ]) {+ expect(names).not.toContain(withdrawn)+ }}- const init = initContainers[0]- expect(init.name).toBe('workspace-layout-init')- expect(init.image).toBe('clerum/mcp-host:0.6.0')- expect(init.command?.[0]).toBe('/bin/sh')- expect(init.command?.[2]).toContain('assert_managed_dir "$root/workspace" workspace')- expect(init.command?.[2]).toContain('directory is a symlink')- expect(init.command?.[2]).toContain('state.db-wal')- expect(init.volumeMounts).toEqual([{ name: 'workspace', mountPath: '/mnt/workspace-root' }])- expect(init.securityContext).toEqual({- allowPrivilegeEscalation: false,- runAsNonRoot: true,- runAsUser: 1001,- runAsGroup: 1001,- capabilities: { drop: ['ALL'] },- seccompProfile: { type: 'RuntimeDefault' },- })-- const mounts = dep.spec?.template?.spec?.containers?.[0]?.volumeMounts ?? []- expect(mounts).toContainEqual({- name: 'workspace',- mountPath: '/workspace',- subPath: 'workspace',- })- expect(mounts).toContainEqual({- name: 'workspace',- mountPath: '/var/lib/clerum/state',- subPath: 'state',- })+ )++ it('changes only lifecycle fields when a Host flips lifecycle', () => {+ const { reconciler } = createReconciler()+ const stateful = reconciler.buildDeployment(makeHost())+ const stateless = reconciler.buildDeployment(+ makeHost({ spec: { ...makeHost().spec, lifecycle: { stateless: true } } })+ )++ // Witness: the lifecycle-specific template really differs.+ expect(envValue(stateless, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')+ expect(containerEnv(stateful).map(env => env.name)).not.toContain('CLERUM_STATELESS_LIFECYCLE')+ expect(stateless.spec?.template?.spec?.priorityClassName).toBe('clerum-interactive-host')+ expect(stateful.spec?.template?.spec?.priorityClassName).toBeUndefined()++ const storage = (dep: k8s.V1Deployment) =>+ JSON.stringify({+ mounts: dep.spec?.template?.spec?.containers?.[0]?.volumeMounts,+ volumes: dep.spec?.template?.spec?.volumes,+ store: envValue(dep, 'CLERUM_SESSION_STORE'),+ dbDir: envValue(dep, 'CLERUM_SESSION_DB_DIR'),+ inits: dep.spec?.template?.spec?.initContainers,+ strategy: dep.spec?.strategy,+ })+ expect(storage(stateless)).toBe(storage(stateful))})- it('keeps the non-stateless pod spec identical to the legacy shape', () => {+ it('renders the same manifest for an explicit non-stateless lifecycle argument', () => {const { reconciler } = createReconciler()const dep = reconciler.buildDeployment(makeHost())-- expect(dep.spec?.template?.spec?.initContainers).toBeUndefined()- const mounts = dep.spec?.template?.spec?.containers?.[0]?.volumeMounts ?? []- expect(mounts).toContainEqual({ name: 'workspace', mountPath: '/workspace' })- const serialized = JSON.stringify(dep)- expect(serialized).not.toContain('subPath')- expect(serialized).not.toContain('CLERUM_STATELESS_LIFECYCLE')- expect(serialized).not.toContain('workspace-layout-init')- // An explicit non-stateless lifecycle argument yields the exact same manifest.const explicit = reconciler.buildDeployment(makeHost(), [], '', {stateless: false,state: 'active',})expect(explicit).toEqual(dep)+ expect(dep.spec?.template?.spec?.initContainers?.map(init => init.name)).toEqual([+ 'workspace-layout-merge',+ ])})})@@ -5401,6 +5429,8 @@ describe('HostReconciler stateless lifecycle — guarded image pull policy (Stagconst { reconciler } = createReconciler()const { container, init } = podContainers(reconciler.buildDeployment(makeHost()))expect(container.imagePullPolicy).toBe('Always')- expect(init).toBeUndefined()+ // The merge init follows the main container's policy on every Host.+ expect(init?.name).toBe('workspace-layout-merge')+ expect(init?.imagePullPolicy).toBe('Always')})})diff --git a/host-context-controller/test/statelessDeployment.workspaceLayout.test.ts b/host-context-controller/test/statelessDeployment.workspaceLayout.test.tsdeleted file mode 100644index 6a74125c4..000000000--- a/host-context-controller/test/statelessDeployment.workspaceLayout.test.ts+++ /dev/null@@ -1,331 +0,0 @@-import { afterEach, describe, expect, it } from 'vitest'-import { execFileSync, spawnSync } from 'node:child_process'-import {- existsSync,- lstatSync,- mkdirSync,- mkdtempSync,- readFileSync,- readdirSync,- rmSync,- symlinkSync,- writeFileSync,-} from 'node:fs'-import { tmpdir } from 'node:os'-import { join } from 'node:path'-import { buildWorkspaceLayoutInitScript } from '../src/statelessDeployment'--const roots: string[] = []--function makeRoot(): string {- const root = mkdtempSync(join(tmpdir(), 'clerum-workspace-layout-'))- roots.push(root)- return root-}--function runMigration(root: string): void {- execFileSync('/bin/sh', ['-c', buildWorkspaceLayoutInitScript(root)], { stdio: 'pipe' })-}--function migrationFailure(root: string): { stderr?: Buffer } {- try {- runMigration(root)- } catch (error) {- return error as { stderr?: Buffer }- }- throw new Error('expected workspace layout migration to fail')-}--// Captures stderr even on SUCCESS (execFileSync only surfaces stderr on throw),-// so the loud round-trip resolution log can be asserted on a passing migration.-function runMigrationCapture(root: string): { status: number; stderr: string } {- const res = spawnSync('/bin/sh', ['-c', buildWorkspaceLayoutInitScript(root)], {- encoding: 'utf8',- })- if (res.error) throw res.error- return { status: res.status ?? -1, stderr: res.stderr ?? '' }-}--function stateBackups(root: string, prefix = 'state.db'): string[] {- return readdirSync(join(root, 'state'))- .filter(name => name.startsWith(`${prefix}.pre-`) && name.endsWith('.bak'))- .sort()-}--afterEach(() => {- for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true })-})--describe('stateless workspace layout migration', () => {- // This is the only case that launches the migration shell twice. Keep its- // runner budget above the generic 5s limit so CI process contention cannot- // turn a completed idempotency check into a harness timeout.- it('moves legacy workspace and SQLite files once and is idempotent', () => {- const root = makeRoot()- writeFileSync(join(root, 'notes.md'), 'workspace-data')- writeFileSync(join(root, 'state.db'), 'durable-db')-- runMigration(root)- runMigration(root)-- expect(readFileSync(join(root, 'workspace', 'notes.md'), 'utf8')).toBe('workspace-data')- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('durable-db')- }, 15_000)-- it('finishes a partial migration whose SQLite source is already under workspace', () => {- const root = makeRoot()- mkdirSync(join(root, 'workspace'))- writeFileSync(join(root, 'workspace', 'state.db'), 'durable-db')-- runMigration(root)-- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('durable-db')- })-- it.each([- {- rootArtifact: 'state.db',- workspaceArtifact: 'state.db-wal',- },- {- rootArtifact: 'state.db-shm',- workspaceArtifact: 'state.db',- },- ])(- 'fails closed when $rootArtifact and $workspaceArtifact split SQLite source authority',- ({ rootArtifact, workspaceArtifact }) => {- const root = makeRoot()- mkdirSync(join(root, 'workspace'))- writeFileSync(join(root, 'notes.md'), 'must-stay-root')- writeFileSync(join(root, rootArtifact), 'root-artifact')- writeFileSync(join(root, 'workspace', workspaceArtifact), 'workspace-artifact')-- expect(String(migrationFailure(root).stderr)).toContain(- 'SQLite artifacts are split across root and workspace'- )- expect(readFileSync(join(root, 'notes.md'), 'utf8')).toBe('must-stay-root')- expect(readFileSync(join(root, rootArtifact), 'utf8')).toBe('root-artifact')- expect(readFileSync(join(root, 'workspace', workspaceArtifact), 'utf8')).toBe(- 'workspace-artifact'- )- }- )-- it('fails closed without overwriting a duplicate workspace entry', () => {- const root = makeRoot()- mkdirSync(join(root, 'workspace'))- writeFileSync(join(root, 'notes.md'), 'legacy-data')- writeFileSync(join(root, 'workspace', 'notes.md'), 'current-data')-- expect(String(migrationFailure(root).stderr)).toContain('workspace layout migration collision')- expect(readFileSync(join(root, 'notes.md'), 'utf8')).toBe('legacy-data')- expect(readFileSync(join(root, 'workspace', 'notes.md'), 'utf8')).toBe('current-data')- })-- it('fails closed without overwriting an existing durable SQLite file', () => {- const root = makeRoot()- mkdirSync(join(root, 'workspace'))- mkdirSync(join(root, 'state'))- writeFileSync(join(root, 'workspace', 'state.db'), 'legacy-db')- writeFileSync(join(root, 'state', 'state.db'), 'current-db')-- expect(String(migrationFailure(root).stderr)).toContain('workspace layout migration collision')- expect(readFileSync(join(root, 'workspace', 'state.db'), 'utf8')).toBe('legacy-db')- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('current-db')- })-- it('resolves a root SQLite round-trip collision: backs up the older destination and migrates the fresher root source', () => {- // Addendum 6 item 6 (newest-source-wins): a stateful<->stateless round-trip- // leaves BOTH root state.db (freshest — the stateful life just left) and an- // earlier stateless life's state/state.db. The migration must self-heal, not- // deadlock: back up the older destination (never delete) and migrate the- // fresher root source in, while non-SQLite root content still moves to- // workspace/.- const root = makeRoot()- mkdirSync(join(root, 'workspace'))- mkdirSync(join(root, 'state'))- writeFileSync(join(root, 'notes.md'), 'must-migrate-to-workspace')- writeFileSync(join(root, 'state.db'), 'fresh-stateful-db')- writeFileSync(join(root, 'state', 'state.db'), 'older-stateless-db')-- const { status, stderr } = runMigrationCapture(root)-- expect(status).toBe(0)- // Fresher root data migrated into the durable destination.- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('fresh-stateful-db')- expect(existsSync(join(root, 'state.db'))).toBe(false)- // Non-SQLite root content still migrated to workspace/.- expect(readFileSync(join(root, 'workspace', 'notes.md'), 'utf8')).toBe(- 'must-migrate-to-workspace'- )- // Older destination preserved (never deleted) as a timestamped backup.- const backups = stateBackups(root)- expect(backups).toHaveLength(1)- expect(readFileSync(join(root, 'state', backups[0]), 'utf8')).toBe('older-stateless-db')- // Loud one-line resolution log naming the backup + migration.- expect(stderr).toContain('newest-source-wins round-trip resolution')- expect(stderr).toContain('.pre-')- })-- it('renames wal/shm destination companions in lockstep with the state.db backup', () => {- const root = makeRoot()- mkdirSync(join(root, 'state'))- for (const db of ['state.db', 'state.db-wal', 'state.db-shm']) {- writeFileSync(join(root, db), `fresh-${db}`)- writeFileSync(join(root, 'state', db), `older-${db}`)- }-- const { status } = runMigrationCapture(root)- expect(status).toBe(0)-- // All three fresher root companions migrated into state/.- for (const db of ['state.db', 'state.db-wal', 'state.db-shm']) {- expect(readFileSync(join(root, 'state', db), 'utf8')).toBe(`fresh-${db}`)- expect(existsSync(join(root, db))).toBe(false)- }- // All three older destinations backed up under ONE shared timestamp suffix.- const backups = readdirSync(join(root, 'state'))- .filter(name => name.endsWith('.bak'))- .sort()- expect(backups).toHaveLength(3)- const suffixes = new Set(- backups.map(name => name.replace(/^state\.db(-wal|-shm)?\.pre-/, '').replace(/\.bak$/, ''))- )- expect(suffixes.size).toBe(1)- expect(- readFileSync(join(root, 'state', backups.find(f => f.startsWith('state.db.pre-'))!), 'utf8')- ).toBe('older-state.db')- expect(- readFileSync(- join(root, 'state', backups.find(f => f.startsWith('state.db-wal.pre-'))!),- 'utf8'- )- ).toBe('older-state.db-wal')- expect(- readFileSync(- join(root, 'state', backups.find(f => f.startsWith('state.db-shm.pre-'))!),- 'utf8'- )- ).toBe('older-state.db-shm')- })-- it('is idempotent after a backup: a second run with no root source is a no-op', () => {- const root = makeRoot()- mkdirSync(join(root, 'state'))- writeFileSync(join(root, 'state.db'), 'fresh-db')- writeFileSync(join(root, 'state', 'state.db'), 'older-db')-- const first = runMigrationCapture(root)- expect(first.status).toBe(0)- const afterFirst = stateBackups(root)- expect(afterFirst).toHaveLength(1)-- const second = runMigrationCapture(root)- expect(second.status).toBe(0)- // No new backup, destination unchanged, no resolution log on the no-op run.- expect(stateBackups(root)).toEqual(afterFirst)- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('fresh-db')- expect(second.stderr).not.toContain('newest-source-wins')- })-- it('backs up under a NEW unique suffix on each later round-trip collision', () => {- const root = makeRoot()- mkdirSync(join(root, 'state'))- writeFileSync(join(root, 'state.db'), 'stateful-life-1')- writeFileSync(join(root, 'state', 'state.db'), 'stateless-life-0')-- expect(runMigrationCapture(root).status).toBe(0)- // A new stateful life re-creates the root source for a second round-trip.- writeFileSync(join(root, 'state.db'), 'stateful-life-2')- expect(runMigrationCapture(root).status).toBe(0)-- // Two distinct backups: nothing was overwritten across the two collisions.- const backups = stateBackups(root)- expect(backups).toHaveLength(2)- const contents = backups.map(name => readFileSync(join(root, 'state', name), 'utf8')).sort()- expect(contents).toEqual(['stateful-life-1', 'stateless-life-0'].sort())- // Final durable state is the newest life.- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('stateful-life-2')- })-- it('keeps a WORKSPACE SQLite source colliding with the destination fail-loud (genuinely ambiguous)', () => {- // Only the ROOT round-trip collision self-heals. A workspace SQLite source- // (an interrupted partial migration) colliding with an existing durable- // destination stays fail-loud, preserving every source for recovery.- const root = makeRoot()- mkdirSync(join(root, 'workspace'))- mkdirSync(join(root, 'state'))- writeFileSync(join(root, 'workspace', 'state.db'), 'legacy-db')- writeFileSync(join(root, 'state', 'state.db'), 'current-db')-- expect(String(migrationFailure(root).stderr)).toContain('workspace layout migration collision')- expect(readFileSync(join(root, 'workspace', 'state.db'), 'utf8')).toBe('legacy-db')- expect(readFileSync(join(root, 'state', 'state.db'), 'utf8')).toBe('current-db')- expect(stateBackups(root)).toHaveLength(0)- })-- it.each(['workspace', 'state'])(- 'rejects a symlinked %s directory without writing through it',- managedDir => {- const root = makeRoot()- const outside = makeRoot()- symlinkSync(outside, join(root, managedDir), 'dir')- writeFileSync(join(root, managedDir === 'workspace' ? 'notes.md' : 'state.db'), 'must-stay')-- expect(String(migrationFailure(root).stderr)).toContain(- `${managedDir} directory is a symlink`- )- expect(- readFileSync(join(root, managedDir === 'workspace' ? 'notes.md' : 'state.db'), 'utf8')- ).toBe('must-stay')- expect(existsSync(join(outside, 'notes.md'))).toBe(false)- expect(existsSync(join(outside, 'state.db'))).toBe(false)- }- )-- it('rejects a symlinked PVC root without touching its target', () => {- const target = makeRoot()- const parent = makeRoot()- const linkedRoot = join(parent, 'workspace-root')- writeFileSync(join(target, 'notes.md'), 'target-data')- symlinkSync(target, linkedRoot, 'dir')-- expect(String(migrationFailure(linkedRoot).stderr)).toContain('PVC root is a symlink')- expect(readFileSync(join(target, 'notes.md'), 'utf8')).toBe('target-data')- expect(existsSync(join(target, 'workspace'))).toBe(false)- expect(existsSync(join(target, 'state'))).toBe(false)- })-- it('rejects a symlinked legacy source without moving it or touching its target', () => {- const root = makeRoot()- const outside = makeRoot()- const outsideTarget = join(outside, 'target.txt')- const source = join(root, 'notes.md')- writeFileSync(outsideTarget, 'outside-data')- symlinkSync(outsideTarget, source)-- expect(String(migrationFailure(root).stderr)).toContain('legacy source is a symlink')- expect(lstatSync(source).isSymbolicLink()).toBe(true)- expect(readFileSync(outsideTarget, 'utf8')).toBe('outside-data')- expect(existsSync(join(root, 'workspace', 'notes.md'))).toBe(false)- })-- it.each(['workspace', 'state'])(- 'rejects a symlinked SQLite artifact under %s without touching its target',- sqliteDir => {- const root = makeRoot()- const outside = makeRoot()- const outsideTarget = join(outside, 'outside-state.db')- mkdirSync(join(root, sqliteDir))- writeFileSync(outsideTarget, 'outside-db')- symlinkSync(outsideTarget, join(root, sqliteDir, 'state.db'))- if (sqliteDir === 'state') writeFileSync(join(root, 'state.db'), 'legacy-db')-- expect(String(migrationFailure(root).stderr)).toContain('SQLite')- expect(readFileSync(outsideTarget, 'utf8')).toBe('outside-db')- expect(lstatSync(join(root, sqliteDir, 'state.db')).isSymbolicLink()).toBe(true)- if (sqliteDir === 'state')- expect(readFileSync(join(root, 'state.db'), 'utf8')).toBe('legacy-db')- }- )-})diff --git a/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.tsnew file mode 100644index 000000000..3a04d992e--- /dev/null+++ b/host-context-controller/test/statelessDeployment.workspaceLayoutMerge.test.ts@@ -0,0 +1,431 @@+/**+ * The workspace-layout merge init returns a split stateless PVC+ * (`state/` + `workspace/`) to the single layout every Host now uses: the PVC+ * root is the workspace and holds `state.db`. Trees are built the way they+ * arise on a cluster: a real SQLite database whose writer was killed with+ * uncheckpointed WAL frames, split by the base `workspace-layout-init`+ * (committed verbatim as a fixture), then merged by the new script.+ */+import { afterEach, describe, expect, it } from 'vitest'+import { spawnSync } from 'node:child_process'+import { createHash } from 'node:crypto'+import {+ chmodSync,+ cpSync,+ lstatSync,+ mkdirSync,+ mkdtempSync,+ readFileSync,+ readdirSync,+ readlinkSync,+ renameSync,+ rmSync,+ statSync,+ symlinkSync,+ writeFileSync,+} from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { buildWorkspaceLayoutMergeScript } from '../src/statelessDeployment'++const BASE_INIT = join(__dirname, '__fixtures__', 'base-workspace-layout-init.sh')+const ROWS = 40+const dirs: string[] = []++function tempDir(prefix: string): string {+ const dir = mkdtempSync(join(tmpdir(), prefix))+ dirs.push(dir)+ return dir+}++afterEach(() => {+ for (const dir of dirs.splice(0)) {+ chmodRecursive(dir)+ rmSync(dir, { recursive: true, force: true })+ }+})++function chmodRecursive(dir: string): void {+ chmodSync(dir, 0o755)+ for (const name of readdirSync(dir)) {+ const path = join(dir, name)+ if (lstatSync(path).isDirectory()) chmodRecursive(path)+ }+}++interface RunResult {+ status: number+ stderr: string+}++function runScript(script: string, args: string[] = [], env?: NodeJS.ProcessEnv): RunResult {+ const res = spawnSync('/bin/sh', ['-c', script, 'sh', ...args], {+ encoding: 'utf8',+ env: env ?? process.env,+ })+ if (res.error) throw res.error+ return { status: res.status ?? -1, stderr: res.stderr ?? '' }+}++function merge(root: string, env?: NodeJS.ProcessEnv): RunResult {+ return runScript(buildWorkspaceLayoutMergeScript(root), [], env)+}++function baseInit(root: string): void {+ const res = runScript(readFileSync(BASE_INIT, 'utf8'), [root])+ expect(res, res.stderr).toMatchObject({ status: 0 })+}++/**+ * A stateful Host's PVC root: `ROWS` rows committed only to the WAL (the+ * writer is SIGKILLed with auto-checkpoint off), personalization files, a+ * user directory and the ext4 `lost+found`.+ */+function statefulRoot(): string {+ const root = tempDir('clerum-merge-')+ const writer = `+ const { DatabaseSync } = require('node:sqlite')+ const db = new DatabaseSync(${JSON.stringify(join(root, 'state.db'))})+ db.exec('PRAGMA journal_mode=WAL')+ db.exec('PRAGMA wal_autocheckpoint=0')+ db.exec('CREATE TABLE messages (id INTEGER PRIMARY KEY, body TEXT NOT NULL)')+ const insert = db.prepare('INSERT INTO messages (body) VALUES (?)')+ for (let i = 0; i < ${ROWS}; i++) insert.run('message ' + i)+ process.kill(process.pid, 'SIGKILL')+ `+ const res = spawnSync(process.execPath, ['-e', writer])+ expect(res.signal).toBe('SIGKILL')+ mkdirSync(join(root, 'lost+found'))+ for (const name of ['AGENTS.md', 'IDENTITY.md', 'SOUL.md', 'USER.md']) {+ writeFileSync(join(root, name), `${name} content`)+ }+ mkdirSync(join(root, 'users', 'u1'), { recursive: true })+ writeFileSync(join(root, 'users', 'u1', 'notes.md'), 'user notes')+ return root+}++/** A stateless Host's PVC as the base init leaves it. */+function splitRoot(): string {+ const root = statefulRoot()+ baseInit(root)+ return root+}++function copyTree(root: string): string {+ const copy = tempDir('clerum-merge-copy-')+ rmSync(copy, { recursive: true })+ cpSync(root, copy, { recursive: true, verbatimSymlinks: true })+ return copy+}++/** Path → type and content hash for every entry, symlinks unresolved. */+function snapshot(root: string): Record<string, string> {+ const out: Record<string, string> = {}+ const walk = (dir: string, rel: string) => {+ for (const name of readdirSync(dir).sort()) {+ const path = join(dir, name)+ const key = rel ? `${rel}/${name}` : name+ const st = lstatSync(path)+ if (st.isSymbolicLink()) out[key] = `link:${readlinkSync(path)}`+ else if (st.isDirectory()) {+ out[key] = 'dir'+ walk(path, key)+ } else out[key] = createHash('sha256').update(readFileSync(path)).digest('hex')+ }+ }+ walk(root, '')+ return out+}++/** Row count and quick_check, read in a child process (node:sqlite). */+function messageCount(dbPath: string): { rows: number; check: string } {+ const reader = `+ const { DatabaseSync } = require('node:sqlite')+ const db = new DatabaseSync(${JSON.stringify(dbPath)})+ const rows = db.prepare('SELECT count(*) AS n FROM messages').get().n+ const check = db.prepare('PRAGMA quick_check').get().quick_check+ db.close()+ process.stdout.write(JSON.stringify({ rows, check }))+ `+ const res = spawnSync(process.execPath, ['-e', reader], { encoding: 'utf8' })+ expect(res.status, res.stderr).toBe(0)+ return JSON.parse(res.stdout) as { rows: number; check: string }+}++/** A PATH whose `mv` fails on call number `failAt` without moving anything. */+function crashingMvEnv(failAt: number): NodeJS.ProcessEnv {+ const bin = tempDir('clerum-merge-bin-')+ const counter = join(bin, 'count')+ writeFileSync(counter, '0')+ const realMv = spawnSync('/bin/sh', ['-c', 'command -v mv'], { encoding: 'utf8' }).stdout.trim()+ writeFileSync(+ join(bin, 'mv'),+ `#!/bin/sh+n=$(( $(cat '${counter}') + 1 ))+echo "$n" > '${counter}'+[ "$n" -ne ${failAt} ] || exit 1+exec '${realMv}' "$@"+`,+ { mode: 0o755 }+ )+ return { ...process.env, PATH: `${bin}:${process.env.PATH}`, MV_COUNTER: counter }+}++function mvCalls(env: NodeJS.ProcessEnv): number {+ return Number(readFileSync(env.MV_COUNTER as string, 'utf8').trim())+}++describe('workspace layout merge', () => {+ it('merges a split tree with an uncheckpointed WAL back to the PVC root', () => {+ const root = splitRoot()+ // Witness that the split tree is the real one: the database and its+ // non-empty WAL sit under state/, agent files under workspace/.+ expect(statSync(join(root, 'state', 'state.db-wal')).size).toBeGreaterThan(0)+ expect(readFileSync(join(root, 'workspace', 'SOUL.md'), 'utf8')).toBe('SOUL.md content')++ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ expect(readdirSync(root).sort()).toEqual([+ 'AGENTS.md',+ 'IDENTITY.md',+ 'SOUL.md',+ 'USER.md',+ 'lost+found',+ 'state.db',+ 'state.db-shm',+ 'state.db-wal',+ 'users',+ ])+ expect(statSync(join(root, 'state.db-wal')).size).toBeGreaterThan(0)+ expect(readFileSync(join(root, 'users', 'u1', 'notes.md'), 'utf8')).toBe('user notes')+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ }, 15_000)++ it('finishes a merge stopped after the journals moved and before the database', () => {+ const split = splitRoot()+ const interrupted = copyTree(split)+ // The merge moves every state/ entry except state.db first.+ renameSync(join(interrupted, 'state', 'state.db-wal'), join(interrupted, 'state.db-wal'))+ renameSync(join(interrupted, 'state', 'state.db-shm'), join(interrupted, 'state.db-shm'))+ for (const name of readdirSync(join(interrupted, 'workspace'))) {+ renameSync(join(interrupted, 'workspace', name), join(interrupted, name))+ }++ expect(merge(split).status).toBe(0)+ expect(merge(interrupted)).toEqual({ status: 0, stderr: '' })++ expect(snapshot(interrupted)).toEqual(snapshot(split))+ expect(messageCount(join(interrupted, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ }, 15_000)++ it.each([+ {+ name: 'workspace is a symlink',+ prepare: (root: string) => {+ renameSync(join(root, 'workspace'), join(root, 'workspace-real'))+ symlinkSync(join(root, 'workspace-real'), join(root, 'workspace'))+ },+ undo: (root: string) => {+ rmSync(join(root, 'workspace'))+ renameSync(join(root, 'workspace-real'), join(root, 'workspace'))+ },+ message: 'workspace directory is a symlink',+ },+ {+ name: 'workspace is a regular file',+ prepare: (root: string) => {+ renameSync(join(root, 'workspace'), join(root, 'workspace-real'))+ writeFileSync(join(root, 'workspace'), 'not a directory')+ },+ undo: (root: string) => {+ rmSync(join(root, 'workspace'))+ renameSync(join(root, 'workspace-real'), join(root, 'workspace'))+ },+ message: 'workspace path is not a directory',+ },+ {+ name: 'a state.db file is a symlink',+ prepare: (root: string) => {+ renameSync(join(root, 'state', 'state.db-shm'), join(root, 'shm-real'))+ symlinkSync(join(root, 'shm-real'), join(root, 'state', 'state.db-shm'))+ },+ undo: (root: string) => {+ rmSync(join(root, 'state', 'state.db-shm'))+ renameSync(join(root, 'shm-real'), join(root, 'state', 'state.db-shm'))+ },+ message: 'SQLite file is a symlink',+ },+ {+ name: 'a second live database at the root (F2)',+ prepare: (root: string) => writeFileSync(join(root, 'state.db'), 'second database'),+ undo: (root: string) => rmSync(join(root, 'state.db')),+ message: 'more than one live SQLite database',+ },+ {+ name: 'a second live database under workspace (F2)',+ prepare: (root: string) =>+ writeFileSync(join(root, 'workspace', 'state.db-wal'), 'orphan journal'),+ undo: (root: string) => rmSync(join(root, 'workspace', 'state.db-wal')),+ message: 'more than one live SQLite database',+ },+ {+ name: 'a workspace name that already exists at the root (F3)',+ prepare: (root: string) => writeFileSync(join(root, 'SOUL.md'), 'root copy'),+ undo: (root: string) => rmSync(join(root, 'SOUL.md')),+ message: 'destination exists',+ },+ {+ name: 'a name present under both state/ and workspace/ (F3)',+ prepare: (root: string) => writeFileSync(join(root, 'state', 'SOUL.md'), 'state copy'),+ undo: (root: string) => rmSync(join(root, 'state', 'SOUL.md')),+ message: 'exists under both state/ and workspace/',+ },+ {+ name: 'a lost+found under workspace/ that cannot be removed (F3)',+ prepare: (root: string) => mkdirSync(join(root, 'workspace', 'lost+found')),+ undo: (root: string) => rmSync(join(root, 'workspace', 'lost+found'), { recursive: true }),+ message: 'workspace/lost+found',+ },+ {+ name: 'state/ is not writable (F4)',+ prepare: (root: string) => chmodSync(join(root, 'state'), 0o555),+ undo: (root: string) => chmodSync(join(root, 'state'), 0o755),+ message: 'not writable',+ },+ ])(+ 'refuses before the first move when $name',+ ({ prepare, undo, message }) => {+ const root = splitRoot()+ prepare(root)+ const before = snapshot(root)++ const refused = merge(root)++ expect(refused.status).toBe(1)+ expect(refused.stderr).toContain('workspace layout merge refused:')+ expect(refused.stderr).toContain(message)+ expect(refused.stderr.trim().split('\n')).toHaveLength(1)+ expect(snapshot(root)).toEqual(before)+ // Witness: once the cause is removed, the same tree merges.+ undo(root)+ expect(merge(root).status).toBe(0)+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ },+ 15_000+ )++ it('moves symlinks under workspace/ as links without following them', () => {+ const root = splitRoot()+ symlinkSync('/tmp/evenfire', join(root, 'workspace', 'evenfire'))+ symlinkSync('/workspace/users/u1/src', join(root, 'workspace', 'src-link'))++ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ expect(readlinkSync(join(root, 'evenfire'))).toBe('/tmp/evenfire')+ expect(readlinkSync(join(root, 'src-link'))).toBe('/workspace/users/u1/src')+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ }, 15_000)++ it('removes an empty source directory whose name exists at the root', () => {+ const root = splitRoot()+ mkdirSync(join(root, 'workspace', 'state'))++ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ expect(readdirSync(root)).not.toContain('state')+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ }, 15_000)++ it('reaches the uninterrupted result after a crash before any move', () => {+ const seed = splitRoot()+ const uninterrupted = copyTree(seed)+ const counting = crashingMvEnv(Number.MAX_SAFE_INTEGER)+ expect(merge(uninterrupted, counting).status).toBe(0)+ const moves = mvCalls(counting)+ // Witness: every workspace entry and every state/ entry was one rename.+ expect(moves).toBe(+ readdirSync(join(seed, 'workspace')).length + readdirSync(join(seed, 'state')).length+ )+ const expected = snapshot(uninterrupted)++ for (let failAt = 1; failAt <= moves; failAt++) {+ const root = copyTree(seed)+ const crashed = merge(root, crashingMvEnv(failAt))+ expect(crashed.status, `crash at move ${failAt}`).not.toBe(0)+ expect(merge(root), `re-run after move ${failAt}`).toEqual({ status: 0, stderr: '' })+ expect(snapshot(root), `tree after crash at move ${failAt}`).toEqual(expected)+ }+ }, 60_000)++ it('leaves empty state/ and workspace/ when it stops after the database moved', () => {+ const root = splitRoot()+ for (const name of readdirSync(join(root, 'workspace'))) {+ renameSync(join(root, 'workspace', name), join(root, name))+ }+ for (const name of readdirSync(join(root, 'state'))) {+ renameSync(join(root, 'state', name), join(root, name))+ }+ const before = snapshot(root)++ // Without a state.db* entry under state/ the tree is not split: no-op.+ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ expect(snapshot(root)).toEqual(before)+ expect(before.state).toBe('dir')+ expect(before.workspace).toBe('dir')+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ }, 15_000)++ it('leaves every tree without the split signature unchanged', () => {+ const cases: Record<string, (root: string) => void> = {+ 'stateful or desktop root': () => undefined,+ 'empty PVC': root => {+ for (const name of readdirSync(root)) rmSync(join(root, name), { recursive: true })+ },+ 'files without a database': root => {+ for (const name of readdirSync(root).filter(n => n.startsWith('state.db'))) {+ rmSync(join(root, name))+ }+ },+ 'user workspace/ folder': root => {+ mkdirSync(join(root, 'workspace'))+ writeFileSync(join(root, 'workspace', 'state.db'), 'user file')+ },+ 'user state/ folder without state.db*': root => {+ mkdirSync(join(root, 'state'))+ writeFileSync(join(root, 'state', 'notes.md'), 'user file')+ },+ 'state/ is a symlink': root => {+ mkdirSync(join(root, 'elsewhere'))+ writeFileSync(join(root, 'elsewhere', 'state.db'), 'not ours')+ symlinkSync(join(root, 'elsewhere'), join(root, 'state'))+ },+ }+ for (const [name, prepare] of Object.entries(cases)) {+ const root = statefulRoot()+ prepare(root)+ const before = snapshot(root)+ expect(merge(root), name).toEqual({ status: 0, stderr: '' })+ expect(snapshot(root), name).toEqual(before)+ }+ // Witness: the split tree is changed by the same script.+ const split = splitRoot()+ const splitBefore = snapshot(split)+ expect(merge(split).status).toBe(0)+ expect(snapshot(split)).not.toEqual(splitBefore)+ }, 30_000)++ it('restores the merged tree after a rollback re-splits it with the base init', () => {+ const root = splitRoot()+ expect(merge(root).status).toBe(0)+ const merged = snapshot(root)++ baseInit(root)+ // Witness: the base init really split the tree again.+ expect(snapshot(root)['state/state.db']).toBe(merged['state.db'])+ expect(merge(root)).toEqual({ status: 0, stderr: '' })++ expect(snapshot(root)).toEqual(merged)+ expect(messageCount(join(root, 'state.db'))).toEqual({ rows: ROWS, check: 'ok' })+ }, 15_000)+})diff --git a/scripts/e2e/e2e-hcc-stateless-cache-containment.sh b/scripts/e2e/e2e-hcc-stateless-cache-containment.shindex d850b3236..f3eacd9ce 100755--- a/scripts/e2e/e2e-hcc-stateless-cache-containment.sh+++ b/scripts/e2e/e2e-hcc-stateless-cache-containment.sh@@ -213,8 +213,8 @@ baseline="$(kctl get deployment "$HOST_REF" -n "$HOST_NS" -o json)"baseline_template="$(jq -Sc '.spec.template' <<<"$baseline")"baseline_pod="$(kctl get pods -n "$HOST_NS" -l "app=${HOST_REF}" -o json |jq -er '[.items[]|select(.metadata.deletionTimestamp==null)|select(any(.status.conditions[]?;.type=="Ready" and .status=="True"))]|if length==1 then .[0].metadata.uid else error("pod identity ambiguous") end')"-jq -e '[.spec.template.spec.containers[].env[]? | select(.name=="CLERUM_SESSION_DB_DIR" and .value=="/var/lib/clerum/state")]|length==1' <<<"$baseline" >/dev/null ||- die 'baseline SQLite directory is not the stateless path'+jq -e '[.spec.template.spec.containers[].env[]? | select(.name=="CLERUM_SESSION_DB_DIR" and .value=="/workspace")]|length==1' <<<"$baseline" >/dev/null ||+ die 'baseline SQLite directory is not the workspace PVC root'cache_held_active() {local status deployment pod