Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: serialize maintenance recovery and preserve replacement transactions

[._.]/ Adam Eivy790c95209ae3
diff --git a/docs/STORAGE.md b/docs/STORAGE.md
index efe98f9a1b..d76b0641b2 100644
--- a/docs/STORAGE.md
+++ b/docs/STORAGE.md
@@ -99,8 +99,14 @@ state hash and retained hold under `data/workflow-maintenance/recovered-<timesta
It publishes a **Transaction recovery** hold before moving the lock when no hold
exists; an existing hold and every operation/exclusive owner remain intact.
A failed repair retains admission fencing and evidence; after addressing its
-cause, repeat the confirmed command. There is no automatic expiry or PID-based
-lock removal. Starting both apps clears their in-memory uncertainty, but does
+cause, repeat the confirmed command. The runtime `transaction/` lock never expires
+or gets removed by PID. Recovery commands serialize through a separate `recovery/`
+fence with an atomically published owner. A proven dead recovery PID permits a
+retry to preserve that fence under `recovery-interrupted-<id>/` and continue.
+Unknown ownership or an interrupted `.transition` remains closed: keep both apps
+stopped, stop every recovery command, inspect and preserve the whole `recovery/`
+directory under a unique sibling name, then repeat the confirmed command. Never
+remove the runtime `transaction/` evidence. Starting both apps clears their in-memory uncertainty, but does
not clear the hold or certify that unresolved work finished.
After restarting, refresh maintenance status, inspect/reconcile any remaining
diff --git a/scripts/recover-maintenance-transaction.mjs b/scripts/recover-maintenance-transaction.mjs
index a2c99b15b9..e6a2b09a4d 100644
--- a/scripts/recover-maintenance-transaction.mjs
+++ b/scripts/recover-maintenance-transaction.mjs
@@ -10,6 +10,84 @@ import { assertNotRealDataWrite } from '../server/lib/testDataIsolation.js';
const listProcesses = () => import('../server/services/pm2.js').then(pm2 => pm2.listMaintenanceProcesses());
const refuse = message => { throw new Error(message); };
+// Publish a complete owner with the fence, never an ownerless mkdir/write gap.
+// Only a proven dead PID permits takeover; unknown/unfinished transitions stay
+// closed for explicit operator inspection, as with the migration runner lock.
+function claimRecoveryFence(directory, io, stat) {
+ const lock = join(directory, 'recovery');
+ const physical = value => value && [value.dev, value.ino, value.birthtimeMs].join(':');
+ const readOwner = path => {
+ const file = join(path, 'owner.json');
+ const info = stat(file);
+ if (!info?.isFile() || info.size > 4096) return null;
+ try {
+ const bytes = io.readFileSync(file);
+ const owner = JSON.parse(bytes);
+ return owner.version === 1 && Number.isSafeInteger(owner.pid) && owner.pid > 0 && typeof owner.token === 'string'
+ ? { ...owner, bytes } : null;
+ } catch { return null; }
+ };
+ const dead = pid => {
+ try { process.kill(pid, 0); return false; }
+ catch (error) { return error.code === 'ESRCH'; }
+ };
+ const owner = { version: 1, pid: process.pid, token: randomUUID() };
+ const bytes = Buffer.from(`${JSON.stringify(owner)}\n`);
+ for (let attempt = 0; attempt < 2; attempt++) {
+ if (!stat(lock)) {
+ const staged = join(directory, `recovery-acquire-${randomUUID()}`);
+ io.mkdirSync(staged, { mode: 0o700 });
+ let acquired = false;
+ try {
+ const fd = io.openSync(join(staged, 'owner.json'), 'wx', 0o600);
+ try { io.writeFileSync(fd, bytes); io.fsyncSync(fd); } finally { io.closeSync(fd); }
+ syncMaintenanceDirectory(staged, io);
+ try { io.renameSync(staged, lock); acquired = true; }
+ catch (error) { if (!stat(lock)) throw error; }
+ } finally {
+ if (!acquired) {
+ io.unlinkSync(join(staged, 'owner.json'));
+ io.rmdirSync(staged);
+ }
+ }
+ if (acquired) {
+ syncMaintenanceDirectory(directory, io);
+ const observed = physical(stat(lock));
+ const owns = () => physical(stat(lock)) === observed && readOwner(lock)?.bytes.equals(bytes) === true;
+ return { owns, release() {
+ if (!owns()) refuse('The maintenance recovery fence changed; preserve it for inspection.');
+ // Rename the nonempty fence first; never expose an empty shared name
+ // that another publisher could replace before our final rmdir.
+ const released = join(directory, `recovery-release-${owner.token}`);
+ io.renameSync(lock, released);
+ io.unlinkSync(join(released, 'owner.json'));
+ io.rmdirSync(released);
+ syncMaintenanceDirectory(directory, io);
+ } };
+ }
+ }
+ const observed = stat(lock);
+ const holder = observed?.isDirectory() ? readOwner(lock) : null;
+ if (!holder) refuse('Maintenance recovery ownership is unverified; preserve its fence for inspection.');
+ if (!dead(holder.pid)) refuse('Maintenance recovery is already owned by a live or unreadable process.');
+ const transition = join(lock, '.transition');
+ try { io.mkdirSync(transition, { mode: 0o700 }); }
+ catch { refuse('Maintenance recovery has an unfinished transition; stop all recovery processes and inspect its fence.'); }
+ const transitionStat = physical(stat(transition));
+ let retired = false;
+ try {
+ if (physical(stat(lock)) !== physical(observed) || readOwner(lock)?.bytes.equals(holder.bytes) !== true || !dead(holder.pid))
+ refuse('Maintenance recovery ownership changed during inspection.');
+ io.renameSync(lock, join(directory, `recovery-interrupted-${randomUUID()}`));
+ retired = true;
+ syncMaintenanceDirectory(directory, io);
+ } finally {
+ if (!retired && physical(stat(lock)) === physical(observed) && physical(stat(transition)) === transitionStat) io.rmdirSync(transition);
+ }
+ }
+ refuse('Maintenance recovery is already owned; retry after its owner finishes.');
+}
+
/** Offline operator disposition of a transaction, never settlement of its work. */
export async function recoverMaintenanceTransaction(reason, { data = PATHS.data, io = fs, processes = listProcesses } = {}) {
const { reason: confirmedReason } = maintenanceBeginSchema.parse({ reason });
@@ -29,68 +107,78 @@ export async function recoverMaintenanceTransaction(reason, { data = PATHS.data,
try { return io.lstatSync(path); }
catch (err) { if (err.code === 'ENOENT') return null; throw err; }
};
- const dirStat = stat(directory);
- if (dirStat && !dirStat.isDirectory()) refuse('The maintenance journal must be a real directory.');
- const originalStat = stat(file);
- if (originalStat && !originalStat.isFile()) refuse('state.json must be a regular file, not a link or directory.');
- const original = originalStat ? io.readFileSync(file) : null;
- // Use exactly the runtime schema; future or damaged journals require compatible
- // code/data recovery, never replacement with an empty journal.
- const state = original ? maintenanceJournalSchema.parse(JSON.parse(original.toString('utf8')))
- : { version: 1, revision: 0, hold: null, operations: [] };
- const lockStat = stat(lock);
- if (lockStat && !lockStat.isDirectory()) refuse('The transaction lock must be a real directory.');
- const pending = dirStat ? io.readdirSync(directory).filter(name => /^pending-.*\.json$/.test(name)).sort().map(name => {
- const stats = stat(join(directory, name));
- if (!stats?.isFile()) refuse('Pending journal evidence must be regular files.');
- return { name, size: stats.size };
- }) : [];
- await offline();
const identity = stats => stats && [stats.dev, stats.ino, stats.size, stats.mtimeMs, stats.ctimeMs].join(':');
- const current = stat(file);
- if (identity(current) !== identity(originalStat) || identity(stat(lock)) !== identity(lockStat)
- || (original && !io.readFileSync(file).equals(original)))
- refuse('The maintenance journal changed during inspection; inspect again before recovery.');
+ const existingDirectory = stat(directory);
+ if (existingDirectory && !existingDirectory.isDirectory()) refuse('The maintenance journal must be a real directory.');
assertNotRealDataWrite(directory, 'maintenance transaction recovery');
io.mkdirSync(directory, { recursive: true, mode: 0o700 });
- if (!lockStat) io.mkdirSync(lock, { mode: 0o700 });
- const at = new Date().toISOString();
- const archiveDir = join(directory, `recovered-${at.replace(/:/g, '-')}-${randomUUID()}`);
- io.mkdirSync(archiveDir, { mode: 0o700 });
- const durableWrite = (path, contents) => {
- const fd = io.openSync(path, 'wx', 0o600);
- try { io.writeFileSync(fd, contents); io.fsyncSync(fd); } finally { io.closeSync(fd); }
- };
- if (original) durableWrite(join(archiveDir, 'state.json'), original);
- if (!state.hold) {
- state.revision++;
- state.hold = { id: randomUUID(), revision: state.revision, owner: 'Transaction recovery',
- reason: confirmedReason, requestedAt: at };
- }
- maintenanceJournalSchema.parse(state);
- const receipt = { version: 1, reason: confirmedReason, at, lockMtime: lockStat?.mtime.toISOString() ?? null,
- pending, originalStateSha256: original ? createHash('sha256').update(original).digest('hex') : null,
- hold: state.hold };
- durableWrite(join(archiveDir, 'receipt.json'), `${JSON.stringify(receipt)}\n`);
- // Interrupted writes may never have flushed their data. A directory fsync
- // preserves names, not those bytes; retain the original fence on flush failure.
- for (const { name } of pending) {
- // Windows FlushFileBuffers requires write access; r+ never truncates evidence.
- const fd = io.openSync(join(directory, name), 'r+');
- try { io.fsyncSync(fd); } finally { io.closeSync(fd); }
- io.renameSync(join(directory, name), join(archiveDir, name));
+ const recoveryFence = claimRecoveryFence(directory, io, stat);
+ try {
+ const dirStat = stat(directory);
+ if (dirStat && !dirStat.isDirectory()) refuse('The maintenance journal must be a real directory.');
+ const originalStat = stat(file);
+ if (originalStat && !originalStat.isFile()) refuse('state.json must be a regular file, not a link or directory.');
+ const original = originalStat ? io.readFileSync(file) : null;
+ // Use exactly the runtime schema; future or damaged journals require compatible
+ // code/data recovery, never replacement with an empty journal.
+ const state = original ? maintenanceJournalSchema.parse(JSON.parse(original.toString('utf8')))
+ : { version: 1, revision: 0, hold: null, operations: [] };
+ const lockStat = stat(lock);
+ if (lockStat && !lockStat.isDirectory()) refuse('The transaction lock must be a real directory.');
+ const pending = dirStat ? io.readdirSync(directory).filter(name => /^pending-.*\.json$/.test(name)).sort().map(name => {
+ const stats = stat(join(directory, name));
+ if (!stats?.isFile()) refuse('Pending journal evidence must be regular files.');
+ return { name, size: stats.size };
+ }) : [];
+ await offline();
+ const current = stat(file);
+ if (identity(current) !== identity(originalStat) || identity(stat(lock)) !== identity(lockStat)
+ || !recoveryFence.owns() || (original && !io.readFileSync(file).equals(original)))
+ refuse('The maintenance journal changed during inspection; inspect again before recovery.');
+ if (!lockStat) io.mkdirSync(lock, { mode: 0o700 });
+ const transactionStat = lockStat || stat(lock);
+ const at = new Date().toISOString();
+ const archiveDir = join(directory, `recovered-${at.replace(/:/g, '-')}-${randomUUID()}`);
+ io.mkdirSync(archiveDir, { mode: 0o700 });
+ const durableWrite = (path, contents) => {
+ const fd = io.openSync(path, 'wx', 0o600);
+ try { io.writeFileSync(fd, contents); io.fsyncSync(fd); } finally { io.closeSync(fd); }
+ };
+ if (original) durableWrite(join(archiveDir, 'state.json'), original);
+ if (!state.hold) {
+ state.revision++;
+ state.hold = { id: randomUUID(), revision: state.revision, owner: 'Transaction recovery',
+ reason: confirmedReason, requestedAt: at };
+ }
+ maintenanceJournalSchema.parse(state);
+ const receipt = { version: 1, reason: confirmedReason, at, lockMtime: lockStat?.mtime.toISOString() ?? null,
+ pending, originalStateSha256: original ? createHash('sha256').update(original).digest('hex') : null,
+ hold: state.hold };
+ durableWrite(join(archiveDir, 'receipt.json'), `${JSON.stringify(receipt)}\n`);
+ // Interrupted writes may never have flushed their data. A directory fsync
+ // preserves names, not those bytes; retain the original fence on flush failure.
+ for (const { name } of pending) {
+ // Windows FlushFileBuffers requires write access; r+ never truncates evidence.
+ const fd = io.openSync(join(directory, name), 'r+');
+ try { io.fsyncSync(fd); } finally { io.closeSync(fd); }
+ io.renameSync(join(directory, name), join(archiveDir, name));
+ }
+ syncMaintenanceDirectory(archiveDir, io);
+ // Publish and fsync a hold BEFORE moving the admission lock. A crash at any
+ // step retains either the original fence or the deliberate recovery hold.
+ const staged = join(directory, `pending-${randomUUID()}.json`);
+ durableWrite(staged, `${JSON.stringify(state)}\n`);
+ io.renameSync(staged, file);
+ syncMaintenanceDirectory(directory, io);
+ if (identity(stat(lock)) !== identity(transactionStat) || !recoveryFence.owns())
+ refuse('The maintenance transaction changed during recovery; its replacement remains fenced.');
+ io.renameSync(lock, join(archiveDir, 'transaction'));
+ syncMaintenanceDirectory(archiveDir, io);
+ syncMaintenanceDirectory(directory, io);
+ return { archiveDir, hold: state.hold, pendingCount: pending.length };
+ } finally {
+ recoveryFence.release();
}
- syncMaintenanceDirectory(archiveDir, io);
- // Publish and fsync a hold BEFORE moving the admission lock. A crash at any
- // step retains either the original fence or the deliberate recovery hold.
- const staged = join(directory, `pending-${randomUUID()}.json`);
- durableWrite(staged, `${JSON.stringify(state)}\n`);
- io.renameSync(staged, file);
- syncMaintenanceDirectory(directory, io);
- io.renameSync(lock, join(archiveDir, 'transaction'));
- syncMaintenanceDirectory(archiveDir, io);
- syncMaintenanceDirectory(directory, io);
- return { archiveDir, hold: state.hold, pendingCount: pending.length };
}
if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
diff --git a/scripts/recover-maintenance-transaction.test.js b/scripts/recover-maintenance-transaction.test.js
index b71ceca462..d14775f8e5 100644
--- a/scripts/recover-maintenance-transaction.test.js
+++ b/scripts/recover-maintenance-transaction.test.js
@@ -1,4 +1,5 @@
-import { afterEach, describe, expect, it } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { spawn, spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -25,6 +26,121 @@ const interrupt = directory => {
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
describe('offline maintenance transaction recovery', () => {
+ it('serializes separate recovery processes and never archives a replacement runtime transaction', async () => {
+ const { data, directory } = fixture();
+ fs.mkdirSync(join(directory, 'transaction'));
+ fs.writeFileSync(join(directory, 'transaction', 'evidence'), 'original transaction');
+ const barrier = join(data, 'barrier');
+ fs.mkdirSync(barrier);
+ const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;
+ const source = `
+ import * as fs from 'node:fs';
+ import { join, basename } from 'node:path';
+ import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};
+ const [id, data, barrier] = process.argv.slice(1);
+ const io = { ...fs, mkdirSync(path, options) {
+ if (basename(path).startsWith('recovered-')) {
+ fs.writeFileSync(join(barrier, id + '-ready'), '');
+ const until = Date.now() + 10000;
+ while (!fs.existsSync(join(barrier, id + '-release'))) {
+ if (Date.now() >= until) throw new Error('Recovery fixture barrier timed out');
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+ }
+ }
+ return fs.mkdirSync(path, options);
+ } };
+ try {
+ const result = await recoverMaintenanceTransaction('Concurrent recovery fixture', {
+ data, io, processes: async () => ['portos-server', 'portos-cos'].map(name => ({ name, status: 'stopped', pid: 0 })),
+ });
+ process.stdout.write(JSON.stringify(result));
+ } catch (error) { process.stdout.write(JSON.stringify({ error: error.message })); }
+ `;
+ const children = [];
+ const launch = id => {
+ const child = spawn(process.execPath, ['--input-type=module', '-e', source, id, data, barrier], { stdio: ['ignore', 'pipe', 'pipe'] });
+ children.push(child);
+ let output = '';
+ let errors = '';
+ const entry = { finished: false, result: null };
+ child.stdout.on('data', bytes => { output += bytes; });
+ child.stderr.on('data', bytes => { errors += bytes; });
+ entry.done = new Promise((resolve, reject) => {
+ child.once('error', reject);
+ child.once('close', code => {
+ entry.finished = true;
+ try { entry.result = JSON.parse(output); resolve(entry.result); }
+ catch { reject(new Error(`Recovery fixture exited ${code}: ${errors}`)); }
+ });
+ });
+ return entry;
+ };
+ try {
+ const first = launch('first');
+ await vi.waitFor(() => expect(fs.existsSync(join(barrier, 'first-ready'))).toBe(true), { timeout: 5000 });
+ const second = launch('second');
+ await vi.waitFor(() => expect(second.finished || fs.existsSync(join(barrier, 'second-ready'))).toBe(true), { timeout: 5000 });
+ fs.writeFileSync(join(barrier, 'first-release'), '');
+ const recovered = await first.done;
+ expect(recovered.error).toBeUndefined();
+ fs.mkdirSync(join(directory, 'transaction'));
+ fs.writeFileSync(join(directory, 'transaction', 'runtime'), 'new runtime evidence');
+ fs.writeFileSync(join(barrier, 'second-release'), '');
+ expect((await second.done).error).toMatch(/recovery.*(?:owned|progress)/i);
+ expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8')).hold).toEqual(recovered.hold);
+ expect(fs.readFileSync(join(directory, 'transaction', 'runtime'), 'utf8')).toBe('new runtime evidence');
+ } finally {
+ for (const id of ['first', 'second']) fs.writeFileSync(join(barrier, `${id}-release`), '');
+ await Promise.all(children.map(child => new Promise(resolve => {
+ if (child.exitCode !== null || child.signalCode !== null) return resolve();
+ child.once('close', resolve);
+ child.kill('SIGKILL');
+ })));
+ }
+ });
+
+ it('reclaims only a proven dead recovery owner and preserves its fence evidence', async () => {
+ const { data, directory } = fixture();
+ fs.mkdirSync(join(directory, 'transaction'));
+ fs.writeFileSync(join(directory, 'transaction', 'original'), 'interrupted runtime evidence');
+ const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;
+ const child = spawnSync(process.execPath, ['--input-type=module', '-e', `
+ import * as fs from 'node:fs';
+ import { basename } from 'node:path';
+ import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};
+ const io = { ...fs, mkdirSync(path, options) {
+ if (basename(path).startsWith('recovered-')) process.kill(process.pid, 'SIGKILL');
+ return fs.mkdirSync(path, options);
+ } };
+ await recoverMaintenanceTransaction('Interrupted recovery fixture', {
+ data: process.argv[1], io, processes: async () => [],
+ });
+ `, data], { encoding: 'utf8', timeout: 5000 });
+ expect(child.error).toBeUndefined();
+ const owner = fs.readFileSync(join(directory, 'recovery', 'owner.json'));
+ const recovered = await recoverMaintenanceTransaction(reason, { data, processes: stopped });
+ const preserved = fs.readdirSync(directory).find(name => name.startsWith('recovery-interrupted-'));
+ expect(fs.readFileSync(join(directory, preserved, 'owner.json'))).toEqual(owner);
+ expect(fs.readFileSync(join(recovered.archiveDir, 'transaction', 'original'), 'utf8')).toBe('interrupted runtime evidence');
+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: recovered.hold });
+ });
+
+ it('preserves unknown recovery ownership and refuses journal symlinks before any mutation', async () => {
+ const { data, directory } = fixture();
+ fs.mkdirSync(join(directory, 'recovery'));
+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/ownership is unverified/);
+ expect(fs.readdirSync(directory)).toEqual(['recovery']);
+ fs.rmdirSync(join(directory, 'recovery'));
+ fs.rmdirSync(directory);
+ const target = join(data, 'outside');
+ fs.mkdirSync(target);
+ fs.writeFileSync(join(target, 'evidence'), 'untouched');
+ fs.symlinkSync(target, directory, process.platform === 'win32' ? 'junction' : 'dir');
+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/real directory/);
+ expect(fs.readdirSync(target)).toEqual(['evidence']);
+ expect(fs.readFileSync(join(target, 'evidence'), 'utf8')).toBe('untouched');
+ });
+
it('preserves interrupted bytes and the original state, then holds admission until deliberate resume', async () => {
const { data, directory, admission } = fixture();
// A previously published journal must be preserved byte-for-byte as evidence.
@@ -49,6 +165,24 @@ describe('offline maintenance transaction recovery', () => {
expect(restarted.resume({ id: result.hold.id, revision: result.hold.revision }).state).toBe('normal');
});
+ it('retains a transaction replaced after recovery publishes its hold', async () => {
+ const { data, directory } = fixture();
+ fs.mkdirSync(join(directory, 'transaction'));
+ fs.writeFileSync(join(directory, 'transaction', 'original'), 'original transaction evidence');
+ const io = { ...fs, renameSync(source, destination) {
+ fs.renameSync(source, destination);
+ if (destination === join(directory, 'state.json')) {
+ fs.renameSync(join(directory, 'transaction'), join(data, 'original-transaction'));
+ fs.mkdirSync(join(directory, 'transaction'));
+ fs.writeFileSync(join(directory, 'transaction', 'replacement'), 'new runtime evidence');
+ }
+ } };
+ await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped })).rejects.toThrow(/transaction changed/);
+ expect(fs.readFileSync(join(directory, 'transaction', 'replacement'), 'utf8')).toBe('new runtime evidence');
+ expect(fs.readFileSync(join(data, 'original-transaction', 'original'), 'utf8')).toBe('original transaction evidence');
+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
+ });
+
it('recovers an absent state conservatively and preserves existing holds and unresolved operations', async () => {
const absent = fixture();
interrupt(absent.directory);