Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: serialize maintenance recovery and preserve replacement transactions
[._.]/ Adam Eivy790c95209ae3
diff --git a/docs/STORAGE.md b/docs/STORAGE.mdindex efe98f9a1b..d76b0641b2 100644--- a/docs/STORAGE.md+++ b/docs/STORAGE.md@@ -99,8 +99,14 @@ state hash and retained hold under `data/workflow-maintenance/recovered-<timestaIt publishes a **Transaction recovery** hold before moving the lock when no holdexists; an existing hold and every operation/exclusive owner remain intact.A failed repair retains admission fencing and evidence; after addressing its-cause, repeat the confirmed command. There is no automatic expiry or PID-based-lock removal. Starting both apps clears their in-memory uncertainty, but does+cause, repeat the confirmed command. The runtime `transaction/` lock never expires+or gets removed by PID. Recovery commands serialize through a separate `recovery/`+fence with an atomically published owner. A proven dead recovery PID permits a+retry to preserve that fence under `recovery-interrupted-<id>/` and continue.+Unknown ownership or an interrupted `.transition` remains closed: keep both apps+stopped, stop every recovery command, inspect and preserve the whole `recovery/`+directory under a unique sibling name, then repeat the confirmed command. Never+remove the runtime `transaction/` evidence. Starting both apps clears their in-memory uncertainty, but doesnot clear the hold or certify that unresolved work finished.After restarting, refresh maintenance status, inspect/reconcile any remainingdiff --git a/scripts/recover-maintenance-transaction.mjs b/scripts/recover-maintenance-transaction.mjsindex a2c99b15b9..e6a2b09a4d 100644--- a/scripts/recover-maintenance-transaction.mjs+++ b/scripts/recover-maintenance-transaction.mjs@@ -10,6 +10,84 @@ import { assertNotRealDataWrite } from '../server/lib/testDataIsolation.js';const listProcesses = () => import('../server/services/pm2.js').then(pm2 => pm2.listMaintenanceProcesses());const refuse = message => { throw new Error(message); };+// Publish a complete owner with the fence, never an ownerless mkdir/write gap.+// Only a proven dead PID permits takeover; unknown/unfinished transitions stay+// closed for explicit operator inspection, as with the migration runner lock.+function claimRecoveryFence(directory, io, stat) {+ const lock = join(directory, 'recovery');+ const physical = value => value && [value.dev, value.ino, value.birthtimeMs].join(':');+ const readOwner = path => {+ const file = join(path, 'owner.json');+ const info = stat(file);+ if (!info?.isFile() || info.size > 4096) return null;+ try {+ const bytes = io.readFileSync(file);+ const owner = JSON.parse(bytes);+ return owner.version === 1 && Number.isSafeInteger(owner.pid) && owner.pid > 0 && typeof owner.token === 'string'+ ? { ...owner, bytes } : null;+ } catch { return null; }+ };+ const dead = pid => {+ try { process.kill(pid, 0); return false; }+ catch (error) { return error.code === 'ESRCH'; }+ };+ const owner = { version: 1, pid: process.pid, token: randomUUID() };+ const bytes = Buffer.from(`${JSON.stringify(owner)}\n`);+ for (let attempt = 0; attempt < 2; attempt++) {+ if (!stat(lock)) {+ const staged = join(directory, `recovery-acquire-${randomUUID()}`);+ io.mkdirSync(staged, { mode: 0o700 });+ let acquired = false;+ try {+ const fd = io.openSync(join(staged, 'owner.json'), 'wx', 0o600);+ try { io.writeFileSync(fd, bytes); io.fsyncSync(fd); } finally { io.closeSync(fd); }+ syncMaintenanceDirectory(staged, io);+ try { io.renameSync(staged, lock); acquired = true; }+ catch (error) { if (!stat(lock)) throw error; }+ } finally {+ if (!acquired) {+ io.unlinkSync(join(staged, 'owner.json'));+ io.rmdirSync(staged);+ }+ }+ if (acquired) {+ syncMaintenanceDirectory(directory, io);+ const observed = physical(stat(lock));+ const owns = () => physical(stat(lock)) === observed && readOwner(lock)?.bytes.equals(bytes) === true;+ return { owns, release() {+ if (!owns()) refuse('The maintenance recovery fence changed; preserve it for inspection.');+ // Rename the nonempty fence first; never expose an empty shared name+ // that another publisher could replace before our final rmdir.+ const released = join(directory, `recovery-release-${owner.token}`);+ io.renameSync(lock, released);+ io.unlinkSync(join(released, 'owner.json'));+ io.rmdirSync(released);+ syncMaintenanceDirectory(directory, io);+ } };+ }+ }+ const observed = stat(lock);+ const holder = observed?.isDirectory() ? readOwner(lock) : null;+ if (!holder) refuse('Maintenance recovery ownership is unverified; preserve its fence for inspection.');+ if (!dead(holder.pid)) refuse('Maintenance recovery is already owned by a live or unreadable process.');+ const transition = join(lock, '.transition');+ try { io.mkdirSync(transition, { mode: 0o700 }); }+ catch { refuse('Maintenance recovery has an unfinished transition; stop all recovery processes and inspect its fence.'); }+ const transitionStat = physical(stat(transition));+ let retired = false;+ try {+ if (physical(stat(lock)) !== physical(observed) || readOwner(lock)?.bytes.equals(holder.bytes) !== true || !dead(holder.pid))+ refuse('Maintenance recovery ownership changed during inspection.');+ io.renameSync(lock, join(directory, `recovery-interrupted-${randomUUID()}`));+ retired = true;+ syncMaintenanceDirectory(directory, io);+ } finally {+ if (!retired && physical(stat(lock)) === physical(observed) && physical(stat(transition)) === transitionStat) io.rmdirSync(transition);+ }+ }+ refuse('Maintenance recovery is already owned; retry after its owner finishes.');+}+/** Offline operator disposition of a transaction, never settlement of its work. */export async function recoverMaintenanceTransaction(reason, { data = PATHS.data, io = fs, processes = listProcesses } = {}) {const { reason: confirmedReason } = maintenanceBeginSchema.parse({ reason });@@ -29,68 +107,78 @@ export async function recoverMaintenanceTransaction(reason, { data = PATHS.data,try { return io.lstatSync(path); }catch (err) { if (err.code === 'ENOENT') return null; throw err; }};- const dirStat = stat(directory);- if (dirStat && !dirStat.isDirectory()) refuse('The maintenance journal must be a real directory.');- const originalStat = stat(file);- if (originalStat && !originalStat.isFile()) refuse('state.json must be a regular file, not a link or directory.');- const original = originalStat ? io.readFileSync(file) : null;- // Use exactly the runtime schema; future or damaged journals require compatible- // code/data recovery, never replacement with an empty journal.- const state = original ? maintenanceJournalSchema.parse(JSON.parse(original.toString('utf8')))- : { version: 1, revision: 0, hold: null, operations: [] };- const lockStat = stat(lock);- if (lockStat && !lockStat.isDirectory()) refuse('The transaction lock must be a real directory.');- const pending = dirStat ? io.readdirSync(directory).filter(name => /^pending-.*\.json$/.test(name)).sort().map(name => {- const stats = stat(join(directory, name));- if (!stats?.isFile()) refuse('Pending journal evidence must be regular files.');- return { name, size: stats.size };- }) : [];- await offline();const identity = stats => stats && [stats.dev, stats.ino, stats.size, stats.mtimeMs, stats.ctimeMs].join(':');- const current = stat(file);- if (identity(current) !== identity(originalStat) || identity(stat(lock)) !== identity(lockStat)- || (original && !io.readFileSync(file).equals(original)))- refuse('The maintenance journal changed during inspection; inspect again before recovery.');+ const existingDirectory = stat(directory);+ if (existingDirectory && !existingDirectory.isDirectory()) refuse('The maintenance journal must be a real directory.');assertNotRealDataWrite(directory, 'maintenance transaction recovery');io.mkdirSync(directory, { recursive: true, mode: 0o700 });- if (!lockStat) io.mkdirSync(lock, { mode: 0o700 });- const at = new Date().toISOString();- const archiveDir = join(directory, `recovered-${at.replace(/:/g, '-')}-${randomUUID()}`);- io.mkdirSync(archiveDir, { mode: 0o700 });- const durableWrite = (path, contents) => {- const fd = io.openSync(path, 'wx', 0o600);- try { io.writeFileSync(fd, contents); io.fsyncSync(fd); } finally { io.closeSync(fd); }- };- if (original) durableWrite(join(archiveDir, 'state.json'), original);- if (!state.hold) {- state.revision++;- state.hold = { id: randomUUID(), revision: state.revision, owner: 'Transaction recovery',- reason: confirmedReason, requestedAt: at };- }- maintenanceJournalSchema.parse(state);- const receipt = { version: 1, reason: confirmedReason, at, lockMtime: lockStat?.mtime.toISOString() ?? null,- pending, originalStateSha256: original ? createHash('sha256').update(original).digest('hex') : null,- hold: state.hold };- durableWrite(join(archiveDir, 'receipt.json'), `${JSON.stringify(receipt)}\n`);- // Interrupted writes may never have flushed their data. A directory fsync- // preserves names, not those bytes; retain the original fence on flush failure.- for (const { name } of pending) {- // Windows FlushFileBuffers requires write access; r+ never truncates evidence.- const fd = io.openSync(join(directory, name), 'r+');- try { io.fsyncSync(fd); } finally { io.closeSync(fd); }- io.renameSync(join(directory, name), join(archiveDir, name));+ const recoveryFence = claimRecoveryFence(directory, io, stat);+ try {+ const dirStat = stat(directory);+ if (dirStat && !dirStat.isDirectory()) refuse('The maintenance journal must be a real directory.');+ const originalStat = stat(file);+ if (originalStat && !originalStat.isFile()) refuse('state.json must be a regular file, not a link or directory.');+ const original = originalStat ? io.readFileSync(file) : null;+ // Use exactly the runtime schema; future or damaged journals require compatible+ // code/data recovery, never replacement with an empty journal.+ const state = original ? maintenanceJournalSchema.parse(JSON.parse(original.toString('utf8')))+ : { version: 1, revision: 0, hold: null, operations: [] };+ const lockStat = stat(lock);+ if (lockStat && !lockStat.isDirectory()) refuse('The transaction lock must be a real directory.');+ const pending = dirStat ? io.readdirSync(directory).filter(name => /^pending-.*\.json$/.test(name)).sort().map(name => {+ const stats = stat(join(directory, name));+ if (!stats?.isFile()) refuse('Pending journal evidence must be regular files.');+ return { name, size: stats.size };+ }) : [];+ await offline();+ const current = stat(file);+ if (identity(current) !== identity(originalStat) || identity(stat(lock)) !== identity(lockStat)+ || !recoveryFence.owns() || (original && !io.readFileSync(file).equals(original)))+ refuse('The maintenance journal changed during inspection; inspect again before recovery.');+ if (!lockStat) io.mkdirSync(lock, { mode: 0o700 });+ const transactionStat = lockStat || stat(lock);+ const at = new Date().toISOString();+ const archiveDir = join(directory, `recovered-${at.replace(/:/g, '-')}-${randomUUID()}`);+ io.mkdirSync(archiveDir, { mode: 0o700 });+ const durableWrite = (path, contents) => {+ const fd = io.openSync(path, 'wx', 0o600);+ try { io.writeFileSync(fd, contents); io.fsyncSync(fd); } finally { io.closeSync(fd); }+ };+ if (original) durableWrite(join(archiveDir, 'state.json'), original);+ if (!state.hold) {+ state.revision++;+ state.hold = { id: randomUUID(), revision: state.revision, owner: 'Transaction recovery',+ reason: confirmedReason, requestedAt: at };+ }+ maintenanceJournalSchema.parse(state);+ const receipt = { version: 1, reason: confirmedReason, at, lockMtime: lockStat?.mtime.toISOString() ?? null,+ pending, originalStateSha256: original ? createHash('sha256').update(original).digest('hex') : null,+ hold: state.hold };+ durableWrite(join(archiveDir, 'receipt.json'), `${JSON.stringify(receipt)}\n`);+ // Interrupted writes may never have flushed their data. A directory fsync+ // preserves names, not those bytes; retain the original fence on flush failure.+ for (const { name } of pending) {+ // Windows FlushFileBuffers requires write access; r+ never truncates evidence.+ const fd = io.openSync(join(directory, name), 'r+');+ try { io.fsyncSync(fd); } finally { io.closeSync(fd); }+ io.renameSync(join(directory, name), join(archiveDir, name));+ }+ syncMaintenanceDirectory(archiveDir, io);+ // Publish and fsync a hold BEFORE moving the admission lock. A crash at any+ // step retains either the original fence or the deliberate recovery hold.+ const staged = join(directory, `pending-${randomUUID()}.json`);+ durableWrite(staged, `${JSON.stringify(state)}\n`);+ io.renameSync(staged, file);+ syncMaintenanceDirectory(directory, io);+ if (identity(stat(lock)) !== identity(transactionStat) || !recoveryFence.owns())+ refuse('The maintenance transaction changed during recovery; its replacement remains fenced.');+ io.renameSync(lock, join(archiveDir, 'transaction'));+ syncMaintenanceDirectory(archiveDir, io);+ syncMaintenanceDirectory(directory, io);+ return { archiveDir, hold: state.hold, pendingCount: pending.length };+ } finally {+ recoveryFence.release();}- syncMaintenanceDirectory(archiveDir, io);- // Publish and fsync a hold BEFORE moving the admission lock. A crash at any- // step retains either the original fence or the deliberate recovery hold.- const staged = join(directory, `pending-${randomUUID()}.json`);- durableWrite(staged, `${JSON.stringify(state)}\n`);- io.renameSync(staged, file);- syncMaintenanceDirectory(directory, io);- io.renameSync(lock, join(archiveDir, 'transaction'));- syncMaintenanceDirectory(archiveDir, io);- syncMaintenanceDirectory(directory, io);- return { archiveDir, hold: state.hold, pendingCount: pending.length };}if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) {diff --git a/scripts/recover-maintenance-transaction.test.js b/scripts/recover-maintenance-transaction.test.jsindex b71ceca462..d14775f8e5 100644--- a/scripts/recover-maintenance-transaction.test.js+++ b/scripts/recover-maintenance-transaction.test.js@@ -1,4 +1,5 @@-import { afterEach, describe, expect, it } from 'vitest';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { spawn, spawnSync } from 'node:child_process';import * as fs from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';@@ -25,6 +26,121 @@ const interrupt = directory => {afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });describe('offline maintenance transaction recovery', () => {+ it('serializes separate recovery processes and never archives a replacement runtime transaction', async () => {+ const { data, directory } = fixture();+ fs.mkdirSync(join(directory, 'transaction'));+ fs.writeFileSync(join(directory, 'transaction', 'evidence'), 'original transaction');+ const barrier = join(data, 'barrier');+ fs.mkdirSync(barrier);+ const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;+ const source = `+ import * as fs from 'node:fs';+ import { join, basename } from 'node:path';+ import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};+ const [id, data, barrier] = process.argv.slice(1);+ const io = { ...fs, mkdirSync(path, options) {+ if (basename(path).startsWith('recovered-')) {+ fs.writeFileSync(join(barrier, id + '-ready'), '');+ const until = Date.now() + 10000;+ while (!fs.existsSync(join(barrier, id + '-release'))) {+ if (Date.now() >= until) throw new Error('Recovery fixture barrier timed out');+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);+ }+ }+ return fs.mkdirSync(path, options);+ } };+ try {+ const result = await recoverMaintenanceTransaction('Concurrent recovery fixture', {+ data, io, processes: async () => ['portos-server', 'portos-cos'].map(name => ({ name, status: 'stopped', pid: 0 })),+ });+ process.stdout.write(JSON.stringify(result));+ } catch (error) { process.stdout.write(JSON.stringify({ error: error.message })); }+ `;+ const children = [];+ const launch = id => {+ const child = spawn(process.execPath, ['--input-type=module', '-e', source, id, data, barrier], { stdio: ['ignore', 'pipe', 'pipe'] });+ children.push(child);+ let output = '';+ let errors = '';+ const entry = { finished: false, result: null };+ child.stdout.on('data', bytes => { output += bytes; });+ child.stderr.on('data', bytes => { errors += bytes; });+ entry.done = new Promise((resolve, reject) => {+ child.once('error', reject);+ child.once('close', code => {+ entry.finished = true;+ try { entry.result = JSON.parse(output); resolve(entry.result); }+ catch { reject(new Error(`Recovery fixture exited ${code}: ${errors}`)); }+ });+ });+ return entry;+ };+ try {+ const first = launch('first');+ await vi.waitFor(() => expect(fs.existsSync(join(barrier, 'first-ready'))).toBe(true), { timeout: 5000 });+ const second = launch('second');+ await vi.waitFor(() => expect(second.finished || fs.existsSync(join(barrier, 'second-ready'))).toBe(true), { timeout: 5000 });+ fs.writeFileSync(join(barrier, 'first-release'), '');+ const recovered = await first.done;+ expect(recovered.error).toBeUndefined();+ fs.mkdirSync(join(directory, 'transaction'));+ fs.writeFileSync(join(directory, 'transaction', 'runtime'), 'new runtime evidence');+ fs.writeFileSync(join(barrier, 'second-release'), '');+ expect((await second.done).error).toMatch(/recovery.*(?:owned|progress)/i);+ expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8')).hold).toEqual(recovered.hold);+ expect(fs.readFileSync(join(directory, 'transaction', 'runtime'), 'utf8')).toBe('new runtime evidence');+ } finally {+ for (const id of ['first', 'second']) fs.writeFileSync(join(barrier, `${id}-release`), '');+ await Promise.all(children.map(child => new Promise(resolve => {+ if (child.exitCode !== null || child.signalCode !== null) return resolve();+ child.once('close', resolve);+ child.kill('SIGKILL');+ })));+ }+ });++ it('reclaims only a proven dead recovery owner and preserves its fence evidence', async () => {+ const { data, directory } = fixture();+ fs.mkdirSync(join(directory, 'transaction'));+ fs.writeFileSync(join(directory, 'transaction', 'original'), 'interrupted runtime evidence');+ const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;+ const child = spawnSync(process.execPath, ['--input-type=module', '-e', `+ import * as fs from 'node:fs';+ import { basename } from 'node:path';+ import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};+ const io = { ...fs, mkdirSync(path, options) {+ if (basename(path).startsWith('recovered-')) process.kill(process.pid, 'SIGKILL');+ return fs.mkdirSync(path, options);+ } };+ await recoverMaintenanceTransaction('Interrupted recovery fixture', {+ data: process.argv[1], io, processes: async () => [],+ });+ `, data], { encoding: 'utf8', timeout: 5000 });+ expect(child.error).toBeUndefined();+ const owner = fs.readFileSync(join(directory, 'recovery', 'owner.json'));+ const recovered = await recoverMaintenanceTransaction(reason, { data, processes: stopped });+ const preserved = fs.readdirSync(directory).find(name => name.startsWith('recovery-interrupted-'));+ expect(fs.readFileSync(join(directory, preserved, 'owner.json'))).toEqual(owner);+ expect(fs.readFileSync(join(recovered.archiveDir, 'transaction', 'original'), 'utf8')).toBe('interrupted runtime evidence');+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: recovered.hold });+ });++ it('preserves unknown recovery ownership and refuses journal symlinks before any mutation', async () => {+ const { data, directory } = fixture();+ fs.mkdirSync(join(directory, 'recovery'));+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/ownership is unverified/);+ expect(fs.readdirSync(directory)).toEqual(['recovery']);+ fs.rmdirSync(join(directory, 'recovery'));+ fs.rmdirSync(directory);+ const target = join(data, 'outside');+ fs.mkdirSync(target);+ fs.writeFileSync(join(target, 'evidence'), 'untouched');+ fs.symlinkSync(target, directory, process.platform === 'win32' ? 'junction' : 'dir');+ await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/real directory/);+ expect(fs.readdirSync(target)).toEqual(['evidence']);+ expect(fs.readFileSync(join(target, 'evidence'), 'utf8')).toBe('untouched');+ });+it('preserves interrupted bytes and the original state, then holds admission until deliberate resume', async () => {const { data, directory, admission } = fixture();// A previously published journal must be preserved byte-for-byte as evidence.@@ -49,6 +165,24 @@ describe('offline maintenance transaction recovery', () => {expect(restarted.resume({ id: result.hold.id, revision: result.hold.revision }).state).toBe('normal');});+ it('retains a transaction replaced after recovery publishes its hold', async () => {+ const { data, directory } = fixture();+ fs.mkdirSync(join(directory, 'transaction'));+ fs.writeFileSync(join(directory, 'transaction', 'original'), 'original transaction evidence');+ const io = { ...fs, renameSync(source, destination) {+ fs.renameSync(source, destination);+ if (destination === join(directory, 'state.json')) {+ fs.renameSync(join(directory, 'transaction'), join(data, 'original-transaction'));+ fs.mkdirSync(join(directory, 'transaction'));+ fs.writeFileSync(join(directory, 'transaction', 'replacement'), 'new runtime evidence');+ }+ } };+ await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped })).rejects.toThrow(/transaction changed/);+ expect(fs.readFileSync(join(directory, 'transaction', 'replacement'), 'utf8')).toBe('new runtime evidence');+ expect(fs.readFileSync(join(data, 'original-transaction', 'original'), 'utf8')).toBe('original transaction evidence');+ expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });+ });+it('recovers an absent state conservatively and preserves existing holds and unresolved operations', async () => {const absent = fixture();interrupt(absent.directory);