Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.7 KB
Raw Blame
// The real detached worker, end to end: supervisor launch, producer stop,
// writer reconciliation, db.sh dump/import, mode commit, restart with target
// proof, and release. PM2 is replaced at its module boundary by a loader hook
// in the worker process only; pg_dump/psql are disposable stubs; the restarted
// "server" is a surrogate running the real boot handshake against a stubbed
// pg module. No live daemon, database or install data is touched.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { spawn, spawnSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { delimiter, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { createDatabaseMaintenanceJournal } from './databaseMaintenanceJournal.js';
import { createDatabaseWriterRegistry } from './databaseWriterRegistry.js';
import { STUB_DUMP_COMPLETE, installDatabaseStubs } from '../test/fixtures/databaseTransferStubs.js';
import { installCutoverStubs } from '../test/fixtures/databaseCutoverStubs.js';
import { isProcessAlive } from '../test/processAlive.js';

const cutoverFixtureUrl = new URL('../test/fixtures/databaseCutoverStubs.js', import.meta.url).href;

const detachedUrl = new URL('./detachedSpawn.js', import.meta.url).href;
const source = { mode: 'native', host: 'native.example.invalid', port: 5432, database: 'example_test', user: 'example' };
const target = { ...source, mode: 'docker', port: 5561 };
let root;
let journal;
let env;
let stubs;
let strays;
let cutover;

// Writes a PM2 module stand-in with file-backed state and a loader hook that
// substitutes it for server/services/pm2.js.
function installPm2Stub() {
  const state = join(stubs.dir, 'pm2.json');
  writeFileSync(state, JSON.stringify([
    { name: 'portos-cos', pmId: 12, pid: 1012, status: 'online', cwd: root, script: 'server/cos-runner/index.js' },
    { name: 'portos-server', pmId: 13, pid: 1013, status: 'online', cwd: root, script: 'server/start.js' },
  ]));
  const stubModule = join(stubs.dir, 'pm2.mjs');
  const processAliveUrl = new URL('../test/processAlive.js', import.meta.url).href;
  writeFileSync(stubModule, `import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
import { launchSurrogateServer, restartedServerEndpoint } from ${JSON.stringify(cutoverFixtureUrl)};
import { isProcessAlive } from ${JSON.stringify(processAliveUrl)};
const state = ${JSON.stringify(state)};
const alive = isProcessAlive;
export async function listMaintenanceProcesses() {
  return JSON.parse(readFileSync(state, 'utf8')).map(row => row.surrogate && !alive(row.pid) ? { ...row, status: 'errored', pid: 0 } : row);
}
export async function restartMaintenanceProducer(name) {
  const rows = JSON.parse(readFileSync(state, 'utf8'));
  const row = rows.find(value => value.name === name);
  appendFileSync(${JSON.stringify(join(stubs.dir, 'events.log'))}, 'restart ' + name + '\\n');
  row.surrogate = name === 'portos-server';
  row.pid = row.surrogate ? launchSurrogateServer(${JSON.stringify(root)}, ${JSON.stringify(stubs.dir)}, restartedServerEndpoint(${JSON.stringify(root)}, ${JSON.stringify(stubs.dir)})) : 2012;
  row.status = 'online';
  writeFileSync(state, JSON.stringify(rows));
  return { success: true };
}
export async function stopApp(id) {
  const rows = JSON.parse(readFileSync(state, 'utf8'));
  const row = rows.find(value => value.pmId === id);
  row.status = 'stopped'; row.pid = 0;
  writeFileSync(state, JSON.stringify(rows));
  appendFileSync(${JSON.stringify(join(stubs.dir, 'events.log'))}, 'stop ' + row.name + '\\n');
  return { success: true };
}\n`);
  // In-thread registerHooks: Node 26 deprecates module.register() with a
  // stderr warning that breaks the exact-stderr assertions below.
  const register = join(stubs.dir, 'register.mjs');
  writeFileSync(register, `import { registerHooks } from 'node:module';
registerHooks({ resolve(specifier, context, next) {
  const result = next(specifier, context);
  return result.url.endsWith('/server/services/pm2.js') ? { url: ${JSON.stringify(pathToFileURL(stubModule).href)}, format: 'module', shortCircuit: true } : result;
} });\n`);
  return pathToFileURL(register).href;
}

beforeEach(() => {
  root = mkdtempSync(join(tmpdir(), 'maintenance-worker-'));
  writeFileSync(join(root, '.portos-disposable-root'), '');
  mkdirSync(join(root, 'server', 'cos-runner'), { recursive: true });
  writeFileSync(join(root, 'server', 'start.js'), '');
  writeFileSync(join(root, 'server', 'cos-runner', 'index.js'), '');
  journal = createDatabaseMaintenanceJournal(join(root, 'data'));
  stubs = installDatabaseStubs(root);
  cutover = installCutoverStubs(root, stubs.dir, { source, target });
  writeFileSync(join(root, '.env'), 'PGMODE=native\n');
  strays = [];
  mkdirSync(join(root, 'tmp'));
  env = { ...process.env, NODE_ENV: 'test', PORTOS_DATA_ROOT: root,
    TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PATH: `${stubs.bin}${delimiter}${process.env.PATH}`,
    PGPASSWORD: 'example-password', PGHOST: 'inherited.example.invalid', PGHOSTADDR: '192.0.2.10',
    NODE_OPTIONS: `--import=${installPm2Stub()}` };
  delete env.VITEST;
});
afterEach(async () => {
  for (const group of strays) { try { process.kill(-group, 'SIGKILL'); } catch { /* already gone */ } }
  const surrogates = cutover.surrogatePids();
  for (const pid of surrogates) { try { process.kill(pid, 'SIGKILL'); } catch { /* already gone */ } }
  // These servers deliberately outlive the coordinator. Their parent handles
  // are gone, so teardown proves exit via the zombie-aware test probe before
  // removing any files they could still write.
  await vi.waitFor(() => expect(surrogates.some(isProcessAlive)).toBe(false), { timeout: 15_000, interval: 20 });
  rmSync(root, { recursive: true, force: true });
});

function run(code) {
  return new Promise((resolve, reject) => {
    const child = spawn(process.execPath, ['--input-type=module', '-e', code], {
      env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 30_000,
    });
    let stdout = '';
    let stderr = '';
    child.stdout.on('data', chunk => { stdout += chunk; });
    child.stderr.on('data', chunk => { stderr += chunk; });
    child.once('error', reject);
    child.once('close', status => resolve({ status, stdout, stderr }));
  });
}
// Exit status of one worker launch. A worker that RELEASED admission moved its
// control directory with the fence, so its supervisor can publish no exit
// receipt; that outcome reports `released` once the worker process is gone.
//
// The `released` interval below never clears on its own while the control dir
// still exists (the common case for every non-release outcome, including a
// refusal). `close` breaks out of it via `process.exit()`, so `error` must do
// the same — otherwise a detached-spawn `error` (e.g. the Windows supervisor
// missing its PID deadline under CI load, #9059) leaves this script running
// forever, and only the outer `run()` spawn's own 30s timeout ever reaps it —
// surfacing as a SIGTERM'd `exitCode: null` instead of the real failure.
function launch(id, token) {
  return run(`import {existsSync} from 'node:fs';
    import {spawnDatabaseMaintenanceWorker} from ${JSON.stringify(detachedUrl)};
    try {
      const child=await spawnDatabaseMaintenanceWorker(${JSON.stringify(id)},${JSON.stringify(token)});
      child.stdout.on('data',chunk=>process.stdout.write(chunk));
      child.stderr.on('data',chunk=>process.stderr.write(chunk));
      child.on('error',()=>{process.exitCode=1;process.exit();});
      child.on('close',code=>{process.exitCode=code;process.exit();});
      const released=setInterval(()=>{
        if(existsSync(${JSON.stringify(controlDirFor(token))})) return;
        try{process.kill(child.pid,0);}catch{clearInterval(released);setTimeout(()=>{process.stdout.write('\\nRELEASED\\n',()=>process.exit(0));});}
      },50);
    } catch { process.exitCode=1; }`);
}
// Release archives the control directory before printing the final result.
// The live tailer may not have opened its old path yet; after proven exit, the
// exact archived log is the authoritative result, independent of forwarding.
const releasedResult = (id, token) => {
  const log = readFileSync(join(root, 'data', 'database-maintenance-completed', id, 'worker-' + token, 'stdout.log'), 'utf8');
  const result = log.split('\n').find(line => line.startsWith('{'));
  if (!result) throw new Error('Released worker archive has no JSON result');
  return JSON.parse(result);
};
const isReleased = outcome => outcome.status === 0 && outcome.stdout.includes('RELEASED');

// Operator status: bounded stage evidence, never endpoints, paths or tokens.
const status = () => {
  const result = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/database-maintenance.mjs', import.meta.url)), 'status'],
    { env, encoding: 'utf8' });
  expect(result.status, result.stderr).toBe(0);
  expect(result.stdout).not.toMatch(/example\.invalid|example-password|maintenance-worker-/);
  return JSON.parse(result.stdout);
};
const controlDirFor = token => join(root, 'data', 'database-maintenance', 'worker-' + token);
// The operator CLI's same-operation recovery, as a separate process.
const recover = id => new Promise((resolve, reject) => {
  const child = spawn(process.execPath, [fileURLToPath(new URL('../../scripts/database-maintenance.mjs', import.meta.url)), 'recover', id],
    { env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 30_000 });
  let stdout = '';
  let stderr = '';
  child.stdout.on('data', chunk => { stdout += chunk; });
  child.stderr.on('data', chunk => { stderr += chunk; });
  child.once('error', reject);
  child.once('close', status => resolve({ status, stdout, stderr }));
});
const importsStarted = () => stubs.events().filter(line => line === 'import-start').length;

// Bounded, synthetic-only failure evidence, read BEFORE afterEach removes the
// disposable root (#8904): every fixture event, the operator's bounded stage
// and coordinator state, and the stderr tail of each owned worker (active or
// archived) and each surrogate server.
const tail = path => (existsSync(path) ? readFileSync(path, 'utf8').slice(-1_500) : '');
function evidence() {
  const operator = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/database-maintenance.mjs', import.meta.url)), 'status'],
    { env, encoding: 'utf8', timeout: 10_000 });
  const completed = join(root, 'data', 'database-maintenance-completed');
  const controlParents = [join(root, 'data', 'database-maintenance'),
    ...(existsSync(completed) ? readdirSync(completed).map(id => join(completed, id)) : [])];
  const workers = controlParents.flatMap(parent => (existsSync(parent) ? readdirSync(parent) : [])
    .filter(name => name.startsWith('worker-')).map(name => join(parent, name, 'stderr.log')));
  return [
    `events: ${JSON.stringify(stubs.events())}`,
    `status (exit ${operator.status}): ${(operator.stdout || operator.stderr).trim().slice(0, 1_500)}`,
    ...workers.map((path, index) => `worker ${index} stderr: ${tail(path)}`),
    `surrogate stderr: ${tail(join(stubs.dir, 'surrogate-stderr.log'))}`,
  ].join('\n');
}
// vi.waitFor that appends the evidence above when its deadline expires.
async function waitWithEvidence(assertion, timeout) {
  try {
    await vi.waitFor(assertion, { timeout, interval: 50 });
  } catch (err) {
    err.message += `\n--- maintenance evidence ---\n${evidence()}`;
    throw err;
  }
}

describe.skipIf(process.platform === 'win32')('owned maintenance worker', () => {
  it.each([[source, target], [target, source]])('cuts over the recorded operation once and releases only after the restarted server proves the target (%#)', async (from, to) => {
    writeFileSync(join(root, '.env'), `PGMODE=${from.mode}\n`);
    const operation = journal.begin({ source: from, target: to });
    const token = journal.acquireCoordinator(operation.id);
    const outcomes = await Promise.all([launch(operation.id, token), launch(operation.id, token)]);
    const released = outcomes.filter(isReleased);
    expect(released, outcomes.map(value => value.stderr).join('\n')).toHaveLength(1);
    expect(outcomes.filter(value => value.status === 1)).toHaveLength(1);
    expect(releasedResult(operation.id, token)).toEqual({ id: operation.id, stage: 'released', source: from.mode, target: to.mode,
      importCommitted: true, sourceRetained: true, restartVerified: true, cosRestarted: true });
    await waitWithEvidence(() => expect(stubs.events()).toContain(`server booted ${to.port}`), 15_000);
    expect(stubs.events().slice(0, 6)).toEqual(['stop portos-cos', 'stop portos-server', 'dump writer=none',
      'import-start', 'import-commit', 'restart portos-server']);
    expect(stubs.events()).toContain('restart portos-cos');
    expect(stubs.invocations('pg_dump')).toEqual([`pg_dump -h ${from.host} -p ${from.port} -U ${from.user} -d ${from.database} --no-owner --no-privileges --no-comments --if-exists --clean`]);
    expect(stubs.invocations('psql')).toEqual([`psql -h ${to.host} -p ${to.port} -U ${to.user} -d ${to.database} -v ON_ERROR_STOP=1 --single-transaction`]);
    expect(stubs.receivedVariables()).toEqual(['PGPASSWORD']);
    expect(stubs.imported()).toBe(STUB_DUMP_COMPLETE);
    expect(readFileSync(join(root, '.env'), 'utf8')).toBe(`PGMODE=${to.mode}\n`);
    expect(journal.read()).toBeNull();
    expect(status()).toEqual({ stage: 'idle', lastCutover: { id: operation.id, source: from.mode, target: to.mode } });
    expect(createDatabaseWriterRegistry(join(root, 'data')).read()).toEqual([]);
    // The one-use owner cannot run again, and its evidence is archived.
    expect((await launch(operation.id, token)).status).toBe(1);
    expect(readdirSync(join(root, 'data', 'database-maintenance-completed', operation.id, 'worker-' + token)))
      .toEqual(expect.arrayContaining(['owner.json', 'started.json', 'group.json', 'stdout.log', 'stderr.log', 'pid']));
  }, 90_000);

  it('relaunches the recorded operation once; a contended operator recovery reports running or fenced, retry', async () => {
    const operation = journal.begin({ source, target });
    const token = journal.acquireCoordinator(operation.id);
    stubs.setMode('dump', 'fail');
    expect((await launch(operation.id, token)).status).toBe(1);
    expect(status()).toMatchObject({ stage: 'exporting', coordinator: { state: 'exited', exitCode: 1 } });
    // Hold the successor at export so a slow competing CLI cannot arrive
    // after release. Ownership changes between checks deliberately fail
    // closed: that contender must retry, not assume both calls exit zero.
    stubs.setMode('dump', 'pause');
    try {
      const outcomes = await Promise.all([recover(operation.id), recover(operation.id)]);
      const successful = outcomes.filter(value => value.status === 0);
      expect(successful.filter(value => JSON.parse(value.stdout).recovery === 'launched'),
        outcomes.map(value => value.stderr).join('\n')).toHaveLength(1);
      for (const outcome of outcomes) {
        if (outcome.status === 0) {
          expect(JSON.parse(outcome.stdout)).toMatchObject({ id: operation.id, recovery: expect.stringMatching(/^(launched|running)$/) });
        } else {
          expect(outcome.status).toBe(1);
          expect(outcome.stdout).toBe('');
          expect(outcome.stderr.trim()).toBe('Database maintenance is fenced; journal recovery is required.');
        }
      }
      await waitWithEvidence(() => expect(stubs.started('dump')).toBe(true), 30_000);
      const retry = await recover(operation.id);
      expect(retry.status, retry.stderr).toBe(0);
      expect(JSON.parse(retry.stdout)).toMatchObject({ id: operation.id, recovery: 'running' });
    } finally {
      stubs.release('dump');
    }
    await waitWithEvidence(() => expect(stubs.events()).toContain(`server booted ${target.port}`), 30_000);
    await waitWithEvidence(() => expect(journal.read()).toBeNull(), 10_000);
    // One recovered worker: one export retry, one import, one server restart.
    expect(stubs.invocations('pg_dump')).toHaveLength(2);
    expect(importsStarted()).toBe(1);
    expect(stubs.events().filter(line => line === 'restart portos-server')).toHaveLength(1);
    // A released operation has nothing left to recover.
    expect((await recover(operation.id)).status).toBe(1);
  }, 90_000);

  // Regression caught: a successor importing while an import started by a
  // killed predecessor (whose supervisor already published an exit) still runs.
  it('never lets a recovered coordinator import while a killed predecessor\'s import still runs', async () => {
    const operation = journal.begin({ source, target });
    const token = journal.acquireCoordinator(operation.id);
    stubs.setMode('import', 'pause');
    const first = launch(operation.id, token);
    await vi.waitFor(() => expect(stubs.started('import')).toBe(true), { timeout: 30_000, interval: 50 });
    const { pgid } = JSON.parse(readFileSync(join(controlDirFor(token), 'group.json'), 'utf8'));
    strays.push(pgid);
    process.kill(Number(readFileSync(join(controlDirFor(token), 'pid'), 'utf8')), 'SIGKILL');
    expect((await first).status).not.toBe(78);
    await vi.waitFor(() => expect(journal.coordinatorStatus(operation.id).state).toBe('exited'), { timeout: 10_000, interval: 50 });

    // Neither knowing the token nor passing extra spawn options authorizes a
    // different executable, arguments, control directory, or ordinary child.
    const ran = join(root, 'ordinary-ran');
    const ordinary = await run(`import {spawnDetached} from ${JSON.stringify(detachedUrl)};
      const child=await spawnDetached(process.execPath,['-e',${JSON.stringify(`require('fs').writeFileSync(${JSON.stringify(ran)},'bad')`)}],{
        controlDir:${JSON.stringify(join(root, 'ordinary'))}, maintenanceCoordinator:{id:${JSON.stringify(operation.id)},token:${JSON.stringify(token)}}});
      child.on('error',()=>{process.exitCode=1;}); child.on('close',()=>{process.exitCode=2;});`);
    expect(ordinary.status).toBe(1);
    expect(existsSync(ran)).toBe(false);

    const second = journal.recoverCoordinator(operation.id, token, randomUUID());
    const refused = await launch(operation.id, second);
    expect(refused.status).toBe(1);
    expect(refused.stderr).toMatch(/previous coordinator's dump or import process is still running/);
    expect(importsStarted()).toBe(1);
    expect(status()).toEqual({ id: operation.id, stage: 'importing', source: 'native', target: 'docker',
      coordinator: { state: 'exited', exitCode: 1 }, transfer: { dump: 'recorded', import: 'pending' } });

    process.kill(-pgid, 'SIGKILL');
    stubs.setMode('import', 'ok');
    const third = journal.recoverCoordinator(operation.id, second, randomUUID());
    const done = await launch(operation.id, third);
    expect(isReleased(done), done.stderr).toBe(true);
    expect(stubs.invocations('pg_dump')).toHaveLength(1);
    expect(importsStarted()).toBe(2);
    expect(status()).toMatchObject({ stage: 'idle', lastCutover: { id: operation.id } });
    expect(stubs.imported()).toBe(STUB_DUMP_COMPLETE);
    expect(journal.read()).toBeNull();
  }, 90_000);

  it('retains unresolved inventory and refuses to export', async () => {
    createDatabaseWriterRegistry(join(root, 'data')).reserve(join(root, 'pending-writer'));
    const operation = journal.begin({ source, target });
    const token = journal.acquireCoordinator(operation.id);
    const outcome = await launch(operation.id, token);
    expect(outcome.status).toBe(1);
    expect(outcome.stdout).toBe('');
    expect(outcome.stderr).toMatch(/1 admitted launch\(es\) have not recorded a process identity/);
    expect(createDatabaseWriterRegistry(join(root, 'data')).read()).toHaveLength(1);
    expect(journal.read()).toEqual({ ...operation, stage: 'quiescing' });
    expect(stubs.invocations('pg_dump')).toEqual([]);
  }, 40_000);

});

// Ownership and entry refusals happen before any platform-specific transfer step.
describe('owned maintenance worker entry', () => {
  it('rejects a foreign owner and cannot relaunch an interrupted reservation', async () => {
    const operation = journal.begin({ source, target });
    const token = journal.acquireCoordinator(operation.id);
    expect((await launch(operation.id, randomUUID())).status).toBe(1);
    expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'unregistered' });
    const controlDir = journal.reserveCoordinatorWorker(operation.id, token);
    writeFileSync(join(controlDir, 'stdout.log'), 'retained evidence');
    expect((await launch(operation.id, token)).status).toBe(1);
    expect(readFileSync(join(controlDir, 'stdout.log'), 'utf8')).toBe('retained evidence');
    expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'awaiting-exit' });
    expect(() => journal.recoverCoordinator(operation.id, token, randomUUID())).toThrow();
    expect(stubs.events()).toEqual([]);
  }, 40_000);

  it('refuses damaged inventory without printing local evidence', async () => {
    const registry = createDatabaseWriterRegistry(join(root, 'data'));
    registry.reserve(join(root, 'pending-writer'));
    const [writer] = registry.read();
    writeFileSync(join(root, 'data', 'database-writers', writer.id, 'reservation.json'), 'private-example-marker');
    const operation = journal.begin({ source, target });
    const token = journal.acquireCoordinator(operation.id);
    // This contract is the worker entry refusal, before platform-specific
    // transfer. Reserve the real one-use owner and invoke the fixed worker
    // directly; detached supervisor launch is exercised by the tests above.
    journal.reserveCoordinatorWorker(operation.id, token);
    const workerUrl = new URL('../../scripts/database-maintenance-worker.mjs', import.meta.url).href;
    const outcome = await run(`process.argv=[process.execPath,'worker',${JSON.stringify(operation.id)},${JSON.stringify(token)}];
      await import(${JSON.stringify(workerUrl)});`);
    expect(outcome.status).toBe(1);
    expect(outcome.stdout).toBe('');
    expect(outcome.stderr).toContain('ownership, writer, or transfer evidence is incomplete');
    expect(outcome.stderr).not.toContain('private-example-marker');
    expect(outcome.stderr).not.toContain(root);
    expect(outcome.stderr).not.toContain(token);
    expect(journal.read()).toEqual({ ...operation, stage: 'quiescing' });
    expect(stubs.invocations('pg_dump')).toEqual([]);
    expect(() => journal.assertAdmission()).toThrow();
  }, 40_000);

  it('rejects repeated direct worker entry', async () => {
    const operation = journal.begin({ source, target });
    const token = journal.acquireCoordinator(operation.id);
    const controlDir = journal.reserveCoordinatorWorker(operation.id, token);
    journal.enterCoordinatorWorker(operation.id, token);
    expect(() => journal.enterCoordinatorWorker(operation.id, token)).toThrow();
    expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'awaiting-exit' });
    // The stopped reservation is never repaired or reused automatically.
    expect(readFileSync(join(controlDir, 'started.json'), 'utf8')).toContain(token);
  });
});


describe('released worker result evidence', () => {
  it('reads exact archived output when live forwarding missed the renamed directory', () => {
    const id = randomUUID();
    const token = randomUUID();
    const archive = join(root, 'data', 'database-maintenance-completed', id, 'worker-' + token);
    mkdirSync(archive, { recursive: true });
    const result = { id, stage: 'released', restartVerified: true };
    writeFileSync(join(archive, 'stdout.log'), JSON.stringify(result) + '\n');
    expect(releasedResult(id, token)).toEqual(result);
    expect(() => releasedResult(id, randomUUID())).toThrow();
    writeFileSync(join(archive, 'stdout.log'), '');
    expect(() => releasedResult(id, token)).toThrow('no JSON result');
    writeFileSync(join(archive, 'stdout.log'), '{invalid}');
    expect(() => releasedResult(id, token)).toThrow(SyntaxError);
  });
});