Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.6 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from 'vitest';
import { spawn, spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
import { recoverMaintenanceTransaction } from './recover-maintenance-transaction.mjs';
import { createMaintenanceAdmission } from '../server/lib/maintenanceAdmission.js';

const roots = [];
const reason = 'Inspected interrupted transaction evidence';
const stopped = async () => ['portos-server', 'portos-cos'].map(name => ({ name, status: 'stopped', pid: 0 }));
function fixture() {
  const data = fs.mkdtempSync(join(tmpdir(), 'maintenance-recovery-'));
  roots.push(data);
  const admission = createMaintenanceAdmission(data);
  fs.mkdirSync(admission.directory, { recursive: true });
  return { data, admission, directory: admission.directory };
}
const interrupt = directory => {
  fs.mkdirSync(join(directory, 'transaction'));
  fs.writeFileSync(join(directory, 'transaction', 'evidence'), 'retained lock evidence');
  fs.writeFileSync(join(directory, 'pending-fixture.json'), '');
  fs.writeFileSync(join(directory, 'pending-other.json'), '{"partial":true}');
};
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });

describe('offline maintenance transaction recovery', () => {
  it('serializes separate recovery processes and never archives a replacement runtime transaction', async () => {
    const { data, directory } = fixture();
    fs.mkdirSync(join(directory, 'transaction'));
    fs.writeFileSync(join(directory, 'transaction', 'evidence'), 'original transaction');
    const barrier = join(data, 'barrier');
    fs.mkdirSync(barrier);
    const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;
    const source = `
      import * as fs from 'node:fs';
      import { join, basename } from 'node:path';
      import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};
      const [id, data, barrier] = process.argv.slice(1);
      const io = { ...fs, mkdirSync(path, options) {
        if (basename(path).startsWith('recovered-')) {
          fs.writeFileSync(join(barrier, id + '-ready'), '');
          const until = Date.now() + 10000;
          while (!fs.existsSync(join(barrier, id + '-release'))) {
            if (Date.now() >= until) throw new Error('Recovery fixture barrier timed out');
            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
          }
        }
        return fs.mkdirSync(path, options);
      } };
      try {
        const result = await recoverMaintenanceTransaction('Concurrent recovery fixture', {
          data, io, processes: async () => ['portos-server', 'portos-cos'].map(name => ({ name, status: 'stopped', pid: 0 })),
        });
        process.stdout.write(JSON.stringify(result));
      } catch (error) { process.stdout.write(JSON.stringify({ error: error.message })); }
    `;
    const children = [];
    const launch = id => {
      const child = spawn(process.execPath, ['--input-type=module', '-e', source, id, data, barrier], { stdio: ['ignore', 'pipe', 'pipe'] });
      children.push(child);
      let output = '';
      let errors = '';
      const entry = { finished: false, result: null };
      child.stdout.on('data', bytes => { output += bytes; });
      child.stderr.on('data', bytes => { errors += bytes; });
      entry.done = new Promise((resolve, reject) => {
        child.once('error', reject);
        child.once('close', code => {
          entry.finished = true;
          try { entry.result = JSON.parse(output); resolve(entry.result); }
          catch { reject(new Error(`Recovery fixture exited ${code}: ${errors}`)); }
        });
      });
      return entry;
    };
    try {
      const first = launch('first');
      await vi.waitFor(() => expect(fs.existsSync(join(barrier, 'first-ready'))).toBe(true), { timeout: 5000 });
      const second = launch('second');
      await vi.waitFor(() => expect(second.finished || fs.existsSync(join(barrier, 'second-ready'))).toBe(true), { timeout: 5000 });
      fs.writeFileSync(join(barrier, 'first-release'), '');
      const recovered = await first.done;
      expect(recovered.error).toBeUndefined();
      fs.mkdirSync(join(directory, 'transaction'));
      fs.writeFileSync(join(directory, 'transaction', 'runtime'), 'new runtime evidence');
      fs.writeFileSync(join(barrier, 'second-release'), '');
      expect((await second.done).error).toMatch(/recovery.*(?:owned|progress)/i);
      expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8')).hold).toEqual(recovered.hold);
      expect(fs.readFileSync(join(directory, 'transaction', 'runtime'), 'utf8')).toBe('new runtime evidence');
    } finally {
      for (const id of ['first', 'second']) fs.writeFileSync(join(barrier, `${id}-release`), '');
      await Promise.all(children.map(child => new Promise(resolve => {
        if (child.exitCode !== null || child.signalCode !== null) return resolve();
        child.once('close', resolve);
        child.kill('SIGKILL');
      })));
    }
  });

  it('reclaims only a proven dead recovery owner and preserves its fence evidence', async () => {
    const { data, directory } = fixture();
    fs.mkdirSync(join(directory, 'transaction'));
    fs.writeFileSync(join(directory, 'transaction', 'original'), 'interrupted runtime evidence');
    const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;
    const child = spawnSync(process.execPath, ['--input-type=module', '-e', `
      import * as fs from 'node:fs';
      import { basename } from 'node:path';
      import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};
      const io = { ...fs, mkdirSync(path, options) {
        if (basename(path).startsWith('recovered-')) process.kill(process.pid, 'SIGKILL');
        return fs.mkdirSync(path, options);
      } };
      await recoverMaintenanceTransaction('Interrupted recovery fixture', {
        data: process.argv[1], io, processes: async () => [],
      });
    `, data], { encoding: 'utf8', timeout: 5000 });
    expect(child.error).toBeUndefined();
    const owner = fs.readFileSync(join(directory, 'recovery', 'owner.json'));
    const recovered = await recoverMaintenanceTransaction(reason, { data, processes: stopped });
    const preserved = fs.readdirSync(directory).find(name => name.startsWith('recovery-interrupted-'));
    expect(fs.readFileSync(join(directory, preserved, 'owner.json'))).toEqual(owner);
    expect(fs.readFileSync(join(recovered.archiveDir, 'transaction', 'original'), 'utf8')).toBe('interrupted runtime evidence');
    expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: recovered.hold });
  });

  it('lets the winning recovery release while a stale contender inspects its takeover marker', async () => {
    const { data, directory } = fixture();
    const dead = spawnSync(process.execPath, ['-e', ''], { timeout: 5000 });
    expect(dead.status).toBe(0);
    const fence = join(directory, 'recovery');
    fs.mkdirSync(fence);
    const originalOwner = JSON.stringify({ version: 1, pid: dead.pid, token: randomUUID() });
    fs.writeFileSync(join(fence, 'owner.json'), originalOwner);
    const barrier = join(data, 'barrier');
    fs.mkdirSync(barrier);
    const module = new URL('./recover-maintenance-transaction.mjs', import.meta.url).href;
    const source = `
      import * as fs from 'node:fs';
      import { join, basename } from 'node:path';
      import { recoverMaintenanceTransaction } from ${JSON.stringify(module)};
      const [data, barrier] = process.argv.slice(1);
      const wait = name => {
        const until = Date.now() + 10000;
        while (!fs.existsSync(join(barrier, name))) {
          if (Date.now() >= until) throw new Error('Recovery fixture barrier timed out');
          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
        }
      };
      const io = { ...fs, mkdirSync(path, options) {
        if (basename(path).startsWith('recovered-')) {
          fs.writeFileSync(join(barrier, 'ready'), '');
          wait('release');
        }
        return fs.mkdirSync(path, options);
      } };
      try {
        wait('start');
        const result = await recoverMaintenanceTransaction('Winning recovery fixture', { data, io, processes: async () => [] });
        process.stdout.write(JSON.stringify(result));
      } catch (error) { process.stdout.write(JSON.stringify({ error: error.message })); }
      finally { fs.writeFileSync(join(barrier, 'finished'), ''); }
    `;
    const child = spawn(process.execPath, ['--input-type=module', '-e', source, data, barrier], { stdio: ['ignore', 'pipe', 'pipe'] });
    let output = '';
    let errors = '';
    child.stdout.on('data', bytes => { output += bytes; });
    child.stderr.on('data', bytes => { errors += bytes; });
    const done = new Promise((resolve, reject) => {
      child.once('error', reject);
      child.once('close', code => {
        try { resolve(JSON.parse(output)); }
        catch { reject(new Error(`Recovery fixture exited ${code}: ${errors}`)); }
      });
    });
    let replacementOwner;
    let transitionFd;
    const io = { ...fs, openSync(path, ...args) {
      const transition = path === join(directory, 'recovery-transition');
      if (transition) {
        // The other process takes ownership after our inspection, immediately
        // before our takeover marker is created.
        fs.writeFileSync(join(barrier, 'start'), '');
        const until = Date.now() + 10000;
        while (!fs.existsSync(join(barrier, 'ready'))) {
          if (Date.now() >= until) throw new Error('Winning recovery did not take ownership');
          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
        }
        replacementOwner = fs.readFileSync(join(fence, 'owner.json'));
      }
      const fd = fs.openSync(path, ...args);
      if (transition) transitionFd = fd;
      return fd;
    }, fstatSync(fd) {
      if (fd === transitionFd) {
        // Let the winner release after our marker is created but before our
        // descriptor inspection and cleanup. Its fence must remain removable.
        fs.writeFileSync(join(barrier, 'release'), '');
        const until = Date.now() + 10000;
        while (!fs.existsSync(join(barrier, 'finished'))) {
          if (Date.now() >= until) throw new Error('Winning recovery did not finish');
          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
        }
      }
      return fs.fstatSync(fd);
    } };
    try {
      await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped }))
        .rejects.toThrow('Maintenance recovery ownership changed during inspection.');
      expect(JSON.parse(replacementOwner).pid).toBe(child.pid);
      fs.writeFileSync(join(barrier, 'release'), '');
      const result = await done;
      expect(result.error).toBeUndefined();
      expect(fs.existsSync(fence)).toBe(false);
      expect(fs.existsSync(join(directory, 'recovery-transition'))).toBe(false);
      expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: result.hold });
      const retired = fs.readdirSync(directory).find(name => name.startsWith('recovery-interrupted-'));
      expect(fs.readFileSync(join(directory, retired, 'owner.json'), 'utf8')).toBe(originalOwner);
    } finally {
      fs.writeFileSync(join(barrier, 'start'), '');
      fs.writeFileSync(join(barrier, 'release'), '');
      await done;
    }
  });

  it('preserves a foreign transition that replaces its marker before descriptor inspection', async () => {
    const { data, directory } = fixture();
    const dead = spawnSync(process.execPath, ['-e', ''], { timeout: 5000 });
    expect(dead.status).toBe(0);
    const fence = join(directory, 'recovery');
    fs.mkdirSync(fence);
    fs.writeFileSync(join(fence, 'owner.json'), JSON.stringify({ version: 1, pid: dead.pid, token: randomUUID() }));
    const transition = join(directory, 'recovery-transition');
    const displaced = join(data, 'displaced-transition');
    const replacement = JSON.stringify({ version: 1, pid: process.pid, token: randomUUID() });
    const io = { ...fs, openSync(path, ...args) {
      const fd = fs.openSync(path, ...args);
      if (path === transition) {
        fs.renameSync(transition, displaced);
        fs.writeFileSync(transition, 'foreign transition evidence');
        fs.writeFileSync(join(fence, 'owner.json'), replacement);
      }
      return fd;
    } };
    await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped }))
      .rejects.toThrow('Maintenance recovery ownership changed during inspection.');
    expect(fs.readFileSync(transition, 'utf8')).toBe('foreign transition evidence');
    expect(fs.readFileSync(join(fence, 'owner.json'), 'utf8')).toBe(replacement);
    expect(fs.readFileSync(displaced)).toHaveLength(0);
    expect(fs.existsSync(join(directory, 'state.json'))).toBe(false);
  });

  it('keeps legacy and standalone unfinished transitions closed without changing their evidence', async () => {
    const dead = spawnSync(process.execPath, ['-e', ''], { timeout: 5000 });
    expect(dead.status).toBe(0);
    for (const kind of ['legacy-directory', 'legacy-file', 'standalone-file']) {
      const { data, directory } = fixture();
      const fence = join(directory, 'recovery');
      const legacy = kind.startsWith('legacy-');
      if (legacy) {
        fs.mkdirSync(fence);
        fs.writeFileSync(join(fence, 'owner.json'), JSON.stringify({ version: 1, pid: dead.pid, token: randomUUID() }));
      }
      const marker = legacy ? join(fence, '.transition') : join(directory, 'recovery-transition');
      if (kind === 'legacy-directory') fs.mkdirSync(marker);
      else fs.writeFileSync(marker, 'interrupted transition evidence');
      await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/unfinished transition/);
      if (kind === 'legacy-directory') expect(fs.lstatSync(marker).isDirectory()).toBe(true);
      else expect(fs.readFileSync(marker, 'utf8')).toBe('interrupted transition evidence');
      expect(fs.existsSync(join(directory, 'state.json'))).toBe(false);
    }
  });

  it('preserves unknown recovery ownership and refuses journal symlinks before any mutation', async () => {
    const { data, directory } = fixture();
    fs.mkdirSync(join(directory, 'recovery'));
    await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/ownership is unverified/);
    expect(fs.readdirSync(directory)).toEqual(['recovery']);
    fs.rmdirSync(join(directory, 'recovery'));
    fs.rmdirSync(directory);
    const target = join(data, 'outside');
    fs.mkdirSync(target);
    fs.writeFileSync(join(target, 'evidence'), 'untouched');
    fs.symlinkSync(target, directory, process.platform === 'win32' ? 'junction' : 'dir');
    await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow(/real directory/);
    expect(fs.readdirSync(target)).toEqual(['evidence']);
    expect(fs.readFileSync(join(target, 'evidence'), 'utf8')).toBe('untouched');
  });

  it('preserves interrupted bytes and the original state, then holds admission until deliberate resume', async () => {
    const { data, directory, admission } = fixture();
    // A previously published journal must be preserved byte-for-byte as evidence.
    const work = admission.admit('agent', 'fixture');
    await work.finish();
    const before = fs.readFileSync(join(directory, 'state.json'));
    interrupt(directory);
    const since = fs.lstatSync(join(directory, 'transaction')).mtime.toISOString();
    const result = await recoverMaintenanceTransaction(reason, { data, processes: stopped });
    expect(fs.readFileSync(join(result.archiveDir, 'state.json'))).toEqual(before);
    expect(fs.readFileSync(join(result.archiveDir, 'transaction', 'evidence'), 'utf8')).toBe('retained lock evidence');
    expect(fs.readFileSync(join(result.archiveDir, 'pending-fixture.json'))).toHaveLength(0);
    expect(fs.readFileSync(join(result.archiveDir, 'pending-other.json'), 'utf8')).toBe('{"partial":true}');
    expect(JSON.parse(fs.readFileSync(join(result.archiveDir, 'receipt.json'), 'utf8'))).toMatchObject({
      reason, lockMtime: since, pending: [{ name: 'pending-fixture.json', size: 0 }, { name: 'pending-other.json', size: 16 }],
      hold: { owner: 'Transaction recovery' },
    });
    expect(fs.readdirSync(directory).sort()).toEqual([result.archiveDir.split(/[\\/]/).at(-1), 'state.json'].sort());
    const restarted = createMaintenanceAdmission(data);
    expect(restarted.status()).toMatchObject({ state: 'ready', hold: result.hold });
    expect(restarted.tryAdmit('agent', 'new')).toBeNull();
    expect(restarted.resume({ id: result.hold.id, revision: result.hold.revision }).state).toBe('normal');
  });

  it('retains a transaction replaced after recovery publishes its hold', async () => {
    const { data, directory } = fixture();
    fs.mkdirSync(join(directory, 'transaction'));
    fs.writeFileSync(join(directory, 'transaction', 'original'), 'original transaction evidence');
    const io = { ...fs, renameSync(source, destination) {
      fs.renameSync(source, destination);
      if (destination === join(directory, 'state.json')) {
        fs.renameSync(join(directory, 'transaction'), join(data, 'original-transaction'));
        fs.mkdirSync(join(directory, 'transaction'));
        fs.writeFileSync(join(directory, 'transaction', 'replacement'), 'new runtime evidence');
      }
    } };
    await expect(recoverMaintenanceTransaction(reason, { data, io, processes: stopped })).rejects.toThrow(/transaction changed/);
    expect(fs.readFileSync(join(directory, 'transaction', 'replacement'), 'utf8')).toBe('new runtime evidence');
    expect(fs.readFileSync(join(data, 'original-transaction', 'original'), 'utf8')).toBe('original transaction evidence');
    expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
  });

  it('recovers an absent state conservatively and preserves existing holds and unresolved operations', async () => {
    const absent = fixture();
    interrupt(absent.directory);
    await recoverMaintenanceTransaction(reason, { data: absent.data, processes: stopped });
    expect(createMaintenanceAdmission(absent.data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });

    const { data, directory, admission } = fixture();
    const work = admission.admit('settlement', 'Unfinished save');
    work.markUnsettled();
    const hold = admission.begin({ reason: 'Existing hold', owner: 'Operator' }).hold;
    const before = JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'));
    interrupt(directory);
    await recoverMaintenanceTransaction(reason, { data, processes: stopped });
    expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'))).toEqual(before);
    expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'draining', hold, blockers: [{ unsettled: true }] });
  });

  it('refuses live, transitioning or unreadable PM2 ownership and invalid journals without changing evidence', async () => {
    const { data, directory } = fixture();
    interrupt(directory);
    for (const inventory of [null, [{}], [{ name: 'portos-server', status: 'online', pid: 123 }],
      [{ name: 'portos-cos', status: 'launching', pid: 0 }], [{ name: 'portos-cos', status: 'stopped', pid: 123 }]]) {
      await expect(recoverMaintenanceTransaction(reason, { data, processes: async () => inventory })).rejects.toThrow(/PM2/);
    }
    for (const text of ['', '{bad json', '{"version":999}']) {
      fs.writeFileSync(join(directory, 'state.json'), text);
      await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped })).rejects.toThrow();
      expect(fs.readFileSync(join(directory, 'state.json'), 'utf8')).toBe(text);
    }
    expect(fs.readdirSync(directory).sort()).toEqual(['pending-fixture.json', 'pending-other.json', 'state.json', 'transaction']);
    expect(fs.readFileSync(join(directory, 'transaction', 'evidence'), 'utf8')).toBe('retained lock evidence');
  });

  it('preserves an exclusive owner and its resume fence', async () => {
    const { data, directory, admission } = fixture();
    const hold = admission.begin({ reason: 'Exclusive work', owner: 'Operator' }).hold;
    admission.claimReady({ id: hold.id, revision: hold.revision, operation: {
      operationId: randomUUID(), requestId: randomUUID(), peerInstanceId: randomUUID(),
      hostInstanceId: randomUUID(), grantId: randomUUID(), grantGeneration: 1, scope: 'execution-v1',
      pairBinding: 'a'.repeat(64), intent: { action: 'portos.update' }, receiverVersion: '1.2.3', evidenceDigest: 'b'.repeat(64),
    } }, admission.observeIdle());
    const before = JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'));
    interrupt(directory);
    await recoverMaintenanceTransaction(reason, { data, processes: stopped });
    expect(JSON.parse(fs.readFileSync(join(directory, 'state.json'), 'utf8'))).toEqual(before);
    const restarted = createMaintenanceAdmission(data);
    expect(restarted.status()).toMatchObject({ state: 'draining', hold });
    expect(() => restarted.resume({ id: hold.id, revision: hold.revision })).toThrow(/exclusive maintenance/);
  });

  it('keeps admission closed if recovery publication fails, and a confirmed retry preserves partial recovery', async () => {
    const { data, directory, admission } = fixture();
    interrupt(directory);
    const io = { ...fs, renameSync: (source, destination) => {
      if (destination === join(directory, 'state.json')) throw Object.assign(new Error('fixture disk full'), { code: 'ENOSPC' });
      return fs.renameSync(source, destination);
    } };
    await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped, io })).rejects.toThrow('fixture disk full');
    expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
    const partial = fs.readdirSync(directory).find(name => name.startsWith('recovered-'));
    expect(fs.readFileSync(join(directory, partial, 'pending-fixture.json'))).toHaveLength(0);
    await recoverMaintenanceTransaction(reason, { data, processes: stopped });
    expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });
    expect(fs.existsSync(join(directory, partial, 'receipt.json'))).toBe(true);
  });

  it('refuses to release the original fence when interrupted pending evidence cannot be flushed', async () => {
    const { data, directory, admission } = fixture();
    interrupt(directory);
    const descriptors = new Map();
    const io = { ...fs,
      openSync: (path, ...args) => {
        const fd = fs.openSync(path, ...args);
        descriptors.set(fd, path);
        return fd;
      },
      closeSync: fd => { descriptors.delete(fd); fs.closeSync(fd); },
      fsyncSync: fd => {
        if (descriptors.get(fd) === join(directory, 'pending-fixture.json')) throw new Error('fixture evidence flush failed');
        fs.fsyncSync(fd);
      },
    };
    await expect(recoverMaintenanceTransaction(reason, { data, processes: stopped, io })).rejects.toThrow('fixture evidence flush failed');
    expect(admission.status()).toMatchObject({ state: 'unavailable', cause: 'interrupted-transaction' });
    expect(fs.readFileSync(join(directory, 'pending-fixture.json'))).toHaveLength(0);
    expect(fs.readFileSync(join(directory, 'pending-other.json'), 'utf8')).toBe('{"partial":true}');
    expect(descriptors.size).toBe(0);
    await recoverMaintenanceTransaction(reason, { data, processes: stopped });
    expect(createMaintenanceAdmission(data).status()).toMatchObject({ state: 'ready', hold: { owner: 'Transaction recovery' } });
  });
});