Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// Offline cutover past the transfer: forward-only mode commit, restart of the
// recorded server with the committed configuration, the restarted process's
// own target proof, and admission release. PM2 is mocked at its module
// boundary; pg_dump/psql are disposable stubs; the "server" is a surrogate
// process running the REAL boot handshake and boot fence against a stubbed pg
// module. No live database, PM2 daemon, install configuration or data.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { spawn } from 'node:child_process';
import { join } from 'node:path';
import { STUB_DUMP_COMPLETE, installDatabaseStubs } from '../test/fixtures/databaseTransferStubs.js';
import { installCutoverStubs, restartedServerEndpoint } from '../test/fixtures/databaseCutoverStubs.js';
import { snapshotProcesses } from '../lib/detachedSpawn.js';
const context = vi.hoisted(() => ({ root: undefined, list: null, stop: null, restart: null }));
vi.mock('../lib/paths.js', async original => {
const actual = await original();
return { ...actual, PATHS: new Proxy(actual.PATHS, {
get: (target, key) => {
if (key === 'data' && context.root) return context.root + '/data';
if (key === 'installRoot' && context.root) return context.root;
return target[key];
},
}) };
});
vi.mock('./pm2.js', () => ({
listMaintenanceProcesses: (...args) => context.list(...args),
stopApp: (...args) => context.stop(...args),
restartMaintenanceProducer: (...args) => context.restart(...args),
}));
const { createDatabaseMaintenanceJournal } = await import('../lib/databaseMaintenanceJournal.js');
const { createDatabaseAuthority } = await import('../lib/databaseAuthority.js');
const { runDatabaseTransfer } = await import('./databaseMaintenanceTransfer.js');
const { runDatabaseCutover } = await import('./databaseMaintenanceCutover.js');
const native = { mode: 'native', host: 'db.example.invalid', port: 5432, database: 'example_test', user: 'example' };
const docker = { mode: 'docker', host: 'db.example.invalid', port: 5561, database: 'example_test', user: 'example' };
// `fast` shortens the proof deadline for a run that is MEANT to hit it. A run
// that must succeed uses `patient` instead: its restarted server is a real
// subprocess whose startup is CPU-bound — 1-2s idle, but 4-5s once the host is
// ~13x oversubscribed, with a valid proof published just AFTER a 4s deadline
// (#9368). Racing a clock against that startup made recovery flaky without
// exercising the cutover; `patient` returns the moment the proof lands, so it
// costs nothing when the host is healthy.
import { isProcessAlive } from '../test/processAlive.js';
const fast = { graceMs: 1500, pollMs: 20, proofTimeoutMs: 4000 };
const patient = { ...fast, proofTimeoutMs: 30_000 };
const alive = isProcessAlive;
const processStart = async pid => process.platform === 'win32' ? null
: (await snapshotProcesses()).find(row => row.pid === pid)?.startedAt;
const savedEnv = { ...process.env };
let root;
let journal;
let stubs;
let cutover;
let rows;
let operation;
let token;
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), 'database-cutover-'));
writeFileSync(join(root, '.portos-disposable-root'), '');
mkdirSync(join(root, 'server', 'cos-runner'), { recursive: true });
writeFileSync(join(root, 'server', 'start.js'), '');
writeFileSync(join(root, 'server', 'cos-runner', 'index.js'), '');
context.root = root;
journal = createDatabaseMaintenanceJournal(join(root, 'data'));
stubs = installDatabaseStubs(root);
rows = [
{ name: 'portos-cos', pmId: 12, pid: 1012, status: 'online', cwd: root, script: 'server/cos-runner/index.js' },
{ name: 'portos-server', pmId: 13, pid: 1013, status: 'online', cwd: root, script: 'server/start.js' },
];
// A surrogate server that exited (refused boot) reads back as errored.
context.list = vi.fn(async () => rows.map(row => (row.surrogate && !alive(row.pid)
? { ...row, status: 'errored', pid: 0 } : { ...row })));
context.stop = vi.fn(async id => {
const row = rows.find(value => value.pmId === id);
stubs.event('stop ' + row.name);
row.status = 'stopped'; row.pid = 0;
return { success: true };
});
// Emulates `pm2 restart ecosystem.config.cjs --only <name> --update-env`:
// the server receives the endpoint the saved configuration names NOW.
context.restart = vi.fn(async name => {
stubs.event('restart ' + name);
const row = rows.find(value => value.name === name);
row.pid = name === 'portos-server' ? cutover.launchServer(restartedServerEndpoint(root, stubs.dir)) : 2012;
row.surrogate = name === 'portos-server';
row.status = 'online';
return { success: true };
});
// The worker inherits its launcher's pre-cutover routing variables; they
// must not redirect the saved-configuration probe or the PM2 restart.
Object.assign(process.env, { PATH: `${stubs.bin}:${savedEnv.PATH}`, PGPASSWORD: 'example-password',
PGHOST: 'inherited.example.invalid', PGPORT: '1', PGUSER: 'inherited', PGDATABASE: 'inherited',
PORTOS_NATIVE_PGPORT: '2', PGPORT_DOCKER: '3' });
});
afterEach(async () => {
for (const key of Object.keys(process.env)) if (!(key in savedEnv)) delete process.env[key];
Object.assign(process.env, savedEnv);
await cutover?.stopSurrogates();
context.root = undefined;
rmSync(root, { recursive: true, force: true });
});
function begin(source = native, target = docker) {
cutover = installCutoverStubs(root, stubs.dir, { source, target });
// Other saved settings must survive the mode commit byte for byte.
writeFileSync(join(root, '.env'), `PGPASSWORD=example-password\nPGMODE=${source.mode}\nPORTOS_EXAMPLE=kept\n`);
operation = journal.begin({ source, target });
token = journal.acquireCoordinator(operation.id);
journal.reserveCoordinatorWorker(operation.id, token);
}
// Same-operation recovery exactly as a detached supervisor receipt permits it.
function successor() {
writeFileSync(join(root, 'data', 'database-maintenance', 'worker-' + token, 'exit'), '1\n');
token = journal.prepareRecovery(operation.id);
journal.reserveCoordinatorWorker(operation.id, token);
}
// Defaults to `patient`; pass `fast` only where the run is expected to time out.
const run = async (options = patient) => {
try {
return await runDatabaseCutover(operation.id, token, options);
} catch (err) {
err.message += `\n--- cutover evidence ---\n${await evidence()}`;
throw err;
}
};
const savedMode = () => /^PGMODE=(\S+)/m.exec(readFileSync(join(root, '.env'), 'utf8'))?.[1];
const serverEvents = () => stubs.events().filter(line => line.startsWith('server '));
// Bounded, synthetic-only failure evidence, read BEFORE afterEach removes the
// disposable root (#8929): every fixture event, the operation's stage and
// coordinator state, the authority record's operation, each surrogate
// server's liveness, and the tail of their timestamped stderr trace — enough
// to tell a refused boot from one still waiting (or never started).
const bounded = read => { try { return read(); } catch (err) { return 'unreadable (' + (err.code ?? 'error') + ')'; } };
async function evidence() {
// A signal-0 probe cannot distinguish a booting process from one suspended
// before its first line. Keep only fixture-owned identities, never commands
// or the rest of the host's process table.
const processes = await snapshotProcesses().catch(() => null);
const surrogates = (cutover?.surrogatePids() ?? []).slice(-20).map(pid => {
const row = processes?.find(value => value.pid === pid);
return { pid, alive: alive(pid), state: processes === null ? 'unavailable' : row?.state ?? 'gone',
startedAt: row?.startedAt ?? null,
targetProof: processes === null ? 'unavailable'
: row ? bounded(() => journal.readTargetProof(operation.id, pid, row.startedAt) !== null) : 'process-gone' };
});
const log = join(stubs.dir, 'surrogate-stderr.log');
return [
`events (last 80): ${JSON.stringify(stubs.events().slice(-80))}`,
`fenced: ${bounded(() => journal.isFenced())}; stage: ${bounded(() => journal.read()?.stage ?? 'none')}`,
`coordinator: ${bounded(() => JSON.stringify(journal.coordinatorStatus(operation.id)))}`,
`authority for this operation: ${bounded(() => createDatabaseAuthority(join(root, 'data')).read()?.operationId === operation.id)}`,
`surrogates: ${JSON.stringify(surrogates)}`,
`surrogate stderr: ${existsSync(log) ? readFileSync(log, 'utf8').slice(-3_000) : ''}`,
].join('\n');
}
// vi.waitFor that appends the evidence above when its deadline expires.
async function waitWithEvidence(assertion) {
try {
await vi.waitFor(assertion, { timeout: 15_000, interval: 20 });
} catch (err) {
err.message += `\n--- cutover evidence ---\n${await evidence()}`;
throw err;
}
}
const waitForEvent = (line) => waitWithEvidence(() => expect(stubs.events()).toContain(line));
const waitForProof = (pid) => waitWithEvidence(async () => expect(journal.readTargetProof(operation.id, pid, await processStart(pid))).not.toBeNull());
describe.skipIf(process.platform === 'win32')('offline database cutover', () => {
it.each([[native, docker], [docker, native]])('commits mode and releases only after the restarted server proves the target (%#)', async (source, target) => {
begin(source, target);
const result = await run();
expect(result).toEqual({ id: operation.id, stage: 'released', source: source.mode, target: target.mode,
importCommitted: true, sourceRetained: true, restartVerified: true, cosRestarted: true });
await waitForEvent(`server booted ${target.port}`);
// Regression caught: a restart before the committed import, or a release
// (CoS back online) before the restarted server's own proof.
expect(stubs.events().slice(0, 6)).toEqual(['stop portos-cos', 'stop portos-server', 'dump writer=none',
'import-start', 'import-commit', 'restart portos-server']);
expect(stubs.events().indexOf('restart portos-cos')).toBeGreaterThan(stubs.events().indexOf('restart portos-server'));
// PM2 re-evaluates the ecosystem with the RECORDED endpoints, not inherited overrides.
const [, restartEnv] = context.restart.mock.calls.find(([name]) => name === 'portos-server');
expect(restartEnv).toMatchObject({ PGHOST: target.host, PGUSER: target.user, PGDATABASE: target.database,
PORTOS_NATIVE_PGPORT: String(native.port), PGPORT_DOCKER: String(docker.port) });
expect(restartEnv.PGPORT).toBeUndefined();
expect(readFileSync(join(root, '.env'), 'utf8'))
.toBe(`PGPASSWORD=example-password\nPGMODE=${target.mode}\nPORTOS_EXAMPLE=kept\n`);
// Admission is open; the authority record retires the source endpoint.
expect(journal.read()).toBeNull();
expect(() => journal.assertAdmission()).not.toThrow();
expect(createDatabaseAuthority(join(root, 'data')).read()).toMatchObject({ operationId: operation.id, source, target });
expect(existsSync(join(root, 'data', 'database-maintenance-completed', operation.id, 'operation.json'))).toBe(true);
// The source recovery copy is retained.
expect(readFileSync(join(root, 'data', 'db-dumps', `portos-maintenance-${operation.id}.sql`), 'utf8')).toBe(STUB_DUMP_COMPLETE);
// Release admits no process still pointed at the retired source.
cutover.launchServer(source);
await waitForEvent('server refused DATABASE_RETIRED_BACKEND');
expect(serverEvents().filter(line => line.startsWith('server booted'))).toEqual([`server booted ${target.port}`]);
}, 60_000);
it.each(['wrong pool', 'unhealthy target'])('stays fenced when the restarted server cannot prove the target (%s), then recovers the same operation', async (fault) => {
begin();
if (fault === 'wrong pool') cutover.overrideRestartPool(native);
else cutover.setHealth('unhealthy');
await expect(run(fast)).rejects.toThrow(/did not prove the target backend[\s\S]*--- cutover evidence ---[\s\S]*events \(last 80\)[\s\S]*coordinator:[\s\S]*surrogate stderr:/);
await waitForEvent('server refused DATABASE_MAINTENANCE');
// Mode is committed forward, but PM2 `online`/saved mode is not success.
expect(journal.read()).toEqual({ ...operation, stage: 'verifying' });
expect(savedMode()).toBe('docker');
expect(() => journal.assertAdmission()).toThrow();
expect(createDatabaseAuthority(join(root, 'data')).read()).toBeNull();
expect(context.restart).not.toHaveBeenCalledWith('portos-cos');
// Operator repairs the backend; same-operation recovery restarts the
// crashed server once more and never re-exports, re-imports or reverses.
cutover.clearRestartPool();
cutover.setHealth('healthy');
successor();
expect(await run()).toMatchObject({ stage: 'released', restartVerified: true });
await waitForEvent(`server booted ${docker.port}`);
expect(stubs.invocations('pg_dump')).toHaveLength(1);
expect(stubs.invocations('psql')).toHaveLength(1);
expect(journal.read()).toBeNull();
}, 60_000);
it('diagnoses a suspended server before startup, stays fenced, then recovers the same operation', async () => {
begin();
const restart = context.restart.getMockImplementation();
context.restart.mockImplementation(async (...args) => {
const result = await restart(...args);
if (args[0] === 'portos-server') {
// Delay the actual process, not a mock handshake or a longer deadline.
process.kill(rows.find(row => row.name === 'portos-server').pid, 'SIGSTOP');
}
return result;
});
await expect(run(fast)).rejects.toThrow(/did not prove the target backend[\s\S]*"state":"T[^"]*"[\s\S]*"targetProof":false/);
expect(journal.read().stage).toBe('verifying');
expect(() => journal.assertAdmission()).toThrow();
expect(context.restart).not.toHaveBeenCalledWith('portos-cos');
// The already-started incarnation publishes its own proof after the
// operator unblocks startup; recovery must neither restart nor re-import.
const pid = rows.find(row => row.name === 'portos-server').pid;
process.kill(pid, 'SIGCONT');
await waitForProof(pid);
successor();
expect(await run()).toMatchObject({ stage: 'released', restartVerified: true });
await waitForEvent(`server booted ${docker.port}`);
expect(context.restart.mock.calls.filter(([name]) => name === 'portos-server')).toHaveLength(1);
expect(stubs.invocations('pg_dump')).toHaveLength(1);
expect(stubs.invocations('psql')).toHaveLength(1);
}, 60_000);
it('waits out a restarted server slower than the short proof deadline instead of racing it', async () => {
begin();
const restart = context.restart.getMockImplementation();
let resume;
context.restart.mockImplementation(async (...args) => {
const result = await restart(...args);
if (args[0] === 'portos-server') {
// Hold the real process past `fast`'s deadline, as a CPU-starved host
// does (#9368): the proof cannot exist until this timer resumes it.
const { pid } = rows.find(row => row.name === 'portos-server');
process.kill(pid, 'SIGSTOP');
resume = setTimeout(() => process.kill(pid, 'SIGCONT'), fast.proofTimeoutMs + 500);
}
return result;
});
try {
// Regression caught: a success-expected run abandoning a live, merely
// slow server at the refusal deadline (the old flaky recovery).
expect(await run()).toMatchObject({ stage: 'released', restartVerified: true });
} finally {
clearTimeout(resume);
}
await waitForEvent(`server booted ${docker.port}`);
expect(context.restart.mock.calls.filter(([name]) => name === 'portos-server')).toHaveLength(1);
expect(stubs.invocations('pg_dump')).toHaveLength(1);
expect(stubs.invocations('psql')).toHaveLength(1);
}, 60_000);
it.each(['before', 'after', 'reverted'])('recovers a mode commit interrupted %s the .env write without guessing or reversing', async (point) => {
begin();
await runDatabaseTransfer(operation.id, token, fast);
// The crashed worker published `committing`, then died.
journal.transition(operation.id, token, 'importing', 'committing');
const env = readFileSync(join(root, '.env'), 'utf8');
if (point === 'after') writeFileSync(join(root, '.env'), env.replace('PGMODE=native', 'PGMODE=docker'));
if (point === 'reverted') writeFileSync(join(root, '.env'), env.replace('PGMODE=native\n', ''));
successor();
expect(await run()).toMatchObject({ stage: 'released', source: 'native', target: 'docker' });
// The recorded target — never the file's current contents — decides.
expect(savedMode()).toBe('docker');
expect(readFileSync(join(root, '.env'), 'utf8')).toMatch(/PORTOS_EXAMPLE=kept/);
expect(stubs.invocations('psql')).toHaveLength(1);
await waitForEvent(`server booted ${docker.port}`);
}, 60_000);
it('keeps a verified-but-unreleased operation fenced until recovery releases it', async () => {
begin();
cutover.setHealth('unhealthy');
await expect(run(fast)).rejects.toThrow(/did not prove/);
cutover.setHealth('healthy');
// A healthy restarted server proves the target and waits at the fence.
const pid = cutover.launchServer(docker);
rows.find(row => row.name === 'portos-server').pid = pid;
await waitForProof(pid);
// A worker crashes between verification and release.
successor();
journal.enterCoordinatorWorker(operation.id, token);
journal.transition(operation.id, token, 'verifying', 'verified');
await new Promise(resolve => setTimeout(resolve, 200));
expect(serverEvents().filter(line => line.startsWith('server booted'))).toEqual([]);
expect(() => journal.assertAdmission()).toThrow();
successor();
expect(await run()).toMatchObject({ stage: 'released' });
await waitForEvent(`server booted ${docker.port}`);
expect(journal.read()).toBeNull();
// Repeated recovery of a released operation has nothing to reopen.
expect(() => journal.prepareRecovery(operation.id)).toThrow();
}, 60_000);
it('never accepts a proof written by an earlier process whose pid was reused', async () => {
begin();
cutover.setHealth('unhealthy');
await expect(run(fast)).rejects.toThrow(/did not prove/);
// PM2 now reports a process that never verified anything, holding a pid for
// which an earlier (verified) process left a proof with its own start time.
const impostor = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: true });
impostor.unref();
try {
const server = rows.find(row => row.name === 'portos-server');
Object.assign(server, { pid: impostor.pid, status: 'online', surrogate: true });
journal.recordTargetProof(operation.id, impostor.pid, Date.now() - 3_600_000);
successor();
await expect(run(fast)).rejects.toThrow(/did not prove the target backend/);
expect(journal.read().stage).toBe('verifying');
expect(() => journal.assertAdmission()).toThrow();
// The new incarnation can publish its own proof without colliding with
// immutable evidence left by the earlier holder of this PID.
const startedAt = await processStart(impostor.pid);
expect(journal.recordTargetProof(operation.id, impostor.pid, startedAt)).toMatchObject({ startedAt });
expect(journal.readTargetProof(operation.id, impostor.pid, startedAt)).toMatchObject({ startedAt });
} finally {
impostor.kill('SIGKILL');
}
}, 60_000);
it('refuses to release onto a saved configuration edited away from the target', async () => {
begin();
cutover.setHealth('unhealthy');
await expect(run(fast)).rejects.toThrow(/did not prove/);
cutover.setHealth('healthy');
const pid = cutover.launchServer(docker);
rows.find(row => row.name === 'portos-server').pid = pid;
await waitForProof(pid);
successor();
journal.enterCoordinatorWorker(operation.id, token);
journal.transition(operation.id, token, 'verifying', 'verified');
writeFileSync(join(root, '.env'), readFileSync(join(root, '.env'), 'utf8').replace('PGMODE=docker', 'PGMODE=native'));
successor();
await expect(run()).rejects.toThrow(/saved configuration does not resolve/);
expect(journal.read().stage).toBe('verified');
expect(() => journal.assertAdmission()).toThrow();
}, 60_000);
it('reuses one persisted successor across repeated recovery requests', async () => {
begin();
writeFileSync(join(root, 'data', 'database-maintenance', 'worker-' + token, 'exit'), '1\n');
const first = journal.prepareRecovery(operation.id);
// A retry before the successor launched gets the SAME token, not a second owner.
expect(journal.prepareRecovery(operation.id)).toBe(first);
journal.reserveCoordinatorWorker(operation.id, first);
expect(journal.prepareRecovery(operation.id)).toBeNull();
expect(() => journal.reserveCoordinatorWorker(operation.id, first)).toThrow();
expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'awaiting-exit' });
});
});