Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// The real detached worker, end to end: supervisor launch, producer stop,
// writer reconciliation, db.sh dump/import, mode commit, restart with target
// proof, and release. PM2 is replaced at its module boundary by a loader hook
// in the worker process only; pg_dump/psql are disposable stubs; the restarted
// "server" is a surrogate running the real boot handshake against a stubbed
// pg module. No live daemon, database or install data is touched.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { spawn, spawnSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { delimiter, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { createDatabaseMaintenanceJournal } from './databaseMaintenanceJournal.js';
import { createDatabaseWriterRegistry } from './databaseWriterRegistry.js';
import { STUB_DUMP_COMPLETE, installDatabaseStubs } from '../test/fixtures/databaseTransferStubs.js';
import { installCutoverStubs } from '../test/fixtures/databaseCutoverStubs.js';
import { isProcessAlive } from '../test/processAlive.js';
const cutoverFixtureUrl = new URL('../test/fixtures/databaseCutoverStubs.js', import.meta.url).href;
const detachedUrl = new URL('./detachedSpawn.js', import.meta.url).href;
const source = { mode: 'native', host: 'native.example.invalid', port: 5432, database: 'example_test', user: 'example' };
const target = { ...source, mode: 'docker', port: 5561 };
let root;
let journal;
let env;
let stubs;
let strays;
let cutover;
// Writes a PM2 module stand-in with file-backed state and a loader hook that
// substitutes it for server/services/pm2.js.
function installPm2Stub() {
const state = join(stubs.dir, 'pm2.json');
writeFileSync(state, JSON.stringify([
{ name: 'portos-cos', pmId: 12, pid: 1012, status: 'online', cwd: root, script: 'server/cos-runner/index.js' },
{ name: 'portos-server', pmId: 13, pid: 1013, status: 'online', cwd: root, script: 'server/start.js' },
]));
const stubModule = join(stubs.dir, 'pm2.mjs');
const processAliveUrl = new URL('../test/processAlive.js', import.meta.url).href;
writeFileSync(stubModule, `import { appendFileSync, readFileSync, writeFileSync } from 'node:fs';
import { launchSurrogateServer, restartedServerEndpoint } from ${JSON.stringify(cutoverFixtureUrl)};
import { isProcessAlive } from ${JSON.stringify(processAliveUrl)};
const state = ${JSON.stringify(state)};
const alive = isProcessAlive;
export async function listMaintenanceProcesses() {
return JSON.parse(readFileSync(state, 'utf8')).map(row => row.surrogate && !alive(row.pid) ? { ...row, status: 'errored', pid: 0 } : row);
}
export async function restartMaintenanceProducer(name) {
const rows = JSON.parse(readFileSync(state, 'utf8'));
const row = rows.find(value => value.name === name);
appendFileSync(${JSON.stringify(join(stubs.dir, 'events.log'))}, 'restart ' + name + '\\n');
row.surrogate = name === 'portos-server';
row.pid = row.surrogate ? launchSurrogateServer(${JSON.stringify(root)}, ${JSON.stringify(stubs.dir)}, restartedServerEndpoint(${JSON.stringify(root)}, ${JSON.stringify(stubs.dir)})) : 2012;
row.status = 'online';
writeFileSync(state, JSON.stringify(rows));
return { success: true };
}
export async function stopApp(id) {
const rows = JSON.parse(readFileSync(state, 'utf8'));
const row = rows.find(value => value.pmId === id);
row.status = 'stopped'; row.pid = 0;
writeFileSync(state, JSON.stringify(rows));
appendFileSync(${JSON.stringify(join(stubs.dir, 'events.log'))}, 'stop ' + row.name + '\\n');
return { success: true };
}\n`);
// In-thread registerHooks: Node 26 deprecates module.register() with a
// stderr warning that breaks the exact-stderr assertions below.
const register = join(stubs.dir, 'register.mjs');
writeFileSync(register, `import { registerHooks } from 'node:module';
registerHooks({ resolve(specifier, context, next) {
const result = next(specifier, context);
return result.url.endsWith('/server/services/pm2.js') ? { url: ${JSON.stringify(pathToFileURL(stubModule).href)}, format: 'module', shortCircuit: true } : result;
} });\n`);
return pathToFileURL(register).href;
}
beforeEach(() => {
root = mkdtempSync(join(tmpdir(), 'maintenance-worker-'));
writeFileSync(join(root, '.portos-disposable-root'), '');
mkdirSync(join(root, 'server', 'cos-runner'), { recursive: true });
writeFileSync(join(root, 'server', 'start.js'), '');
writeFileSync(join(root, 'server', 'cos-runner', 'index.js'), '');
journal = createDatabaseMaintenanceJournal(join(root, 'data'));
stubs = installDatabaseStubs(root);
cutover = installCutoverStubs(root, stubs.dir, { source, target });
writeFileSync(join(root, '.env'), 'PGMODE=native\n');
strays = [];
mkdirSync(join(root, 'tmp'));
env = { ...process.env, NODE_ENV: 'test', PORTOS_DATA_ROOT: root,
TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PATH: `${stubs.bin}${delimiter}${process.env.PATH}`,
PGPASSWORD: 'example-password', PGHOST: 'inherited.example.invalid', PGHOSTADDR: '192.0.2.10',
NODE_OPTIONS: `--import=${installPm2Stub()}` };
delete env.VITEST;
});
afterEach(async () => {
for (const group of strays) { try { process.kill(-group, 'SIGKILL'); } catch { /* already gone */ } }
const surrogates = cutover.surrogatePids();
for (const pid of surrogates) { try { process.kill(pid, 'SIGKILL'); } catch { /* already gone */ } }
// These servers deliberately outlive the coordinator. Their parent handles
// are gone, so teardown proves exit via the zombie-aware test probe before
// removing any files they could still write.
await vi.waitFor(() => expect(surrogates.some(isProcessAlive)).toBe(false), { timeout: 15_000, interval: 20 });
rmSync(root, { recursive: true, force: true });
});
function run(code) {
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--input-type=module', '-e', code], {
env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 30_000,
});
let stdout = '';
let stderr = '';
child.stdout.on('data', chunk => { stdout += chunk; });
child.stderr.on('data', chunk => { stderr += chunk; });
child.once('error', reject);
child.once('close', status => resolve({ status, stdout, stderr }));
});
}
// Exit status of one worker launch. A worker that RELEASED admission moved its
// control directory with the fence, so its supervisor can publish no exit
// receipt; that outcome reports `released` once the worker process is gone.
//
// The `released` interval below never clears on its own while the control dir
// still exists (the common case for every non-release outcome, including a
// refusal). `close` breaks out of it via `process.exit()`, so `error` must do
// the same — otherwise a detached-spawn `error` (e.g. the Windows supervisor
// missing its PID deadline under CI load, #9059) leaves this script running
// forever, and only the outer `run()` spawn's own 30s timeout ever reaps it —
// surfacing as a SIGTERM'd `exitCode: null` instead of the real failure.
function launch(id, token) {
return run(`import {existsSync} from 'node:fs';
import {spawnDatabaseMaintenanceWorker} from ${JSON.stringify(detachedUrl)};
try {
const child=await spawnDatabaseMaintenanceWorker(${JSON.stringify(id)},${JSON.stringify(token)});
child.stdout.on('data',chunk=>process.stdout.write(chunk));
child.stderr.on('data',chunk=>process.stderr.write(chunk));
child.on('error',()=>{process.exitCode=1;process.exit();});
child.on('close',code=>{process.exitCode=code;process.exit();});
const released=setInterval(()=>{
if(existsSync(${JSON.stringify(controlDirFor(token))})) return;
try{process.kill(child.pid,0);}catch{clearInterval(released);setTimeout(()=>{process.stdout.write('\\nRELEASED\\n',()=>process.exit(0));});}
},50);
} catch { process.exitCode=1; }`);
}
// Release archives the control directory before printing the final result.
// The live tailer may not have opened its old path yet; after proven exit, the
// exact archived log is the authoritative result, independent of forwarding.
const releasedResult = (id, token) => {
const log = readFileSync(join(root, 'data', 'database-maintenance-completed', id, 'worker-' + token, 'stdout.log'), 'utf8');
const result = log.split('\n').find(line => line.startsWith('{'));
if (!result) throw new Error('Released worker archive has no JSON result');
return JSON.parse(result);
};
const isReleased = outcome => outcome.status === 0 && outcome.stdout.includes('RELEASED');
// Operator status: bounded stage evidence, never endpoints, paths or tokens.
const status = () => {
const result = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/database-maintenance.mjs', import.meta.url)), 'status'],
{ env, encoding: 'utf8' });
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).not.toMatch(/example\.invalid|example-password|maintenance-worker-/);
return JSON.parse(result.stdout);
};
const controlDirFor = token => join(root, 'data', 'database-maintenance', 'worker-' + token);
// The operator CLI's same-operation recovery, as a separate process.
const recover = id => new Promise((resolve, reject) => {
const child = spawn(process.execPath, [fileURLToPath(new URL('../../scripts/database-maintenance.mjs', import.meta.url)), 'recover', id],
{ env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 30_000 });
let stdout = '';
let stderr = '';
child.stdout.on('data', chunk => { stdout += chunk; });
child.stderr.on('data', chunk => { stderr += chunk; });
child.once('error', reject);
child.once('close', status => resolve({ status, stdout, stderr }));
});
const importsStarted = () => stubs.events().filter(line => line === 'import-start').length;
// Bounded, synthetic-only failure evidence, read BEFORE afterEach removes the
// disposable root (#8904): every fixture event, the operator's bounded stage
// and coordinator state, and the stderr tail of each owned worker (active or
// archived) and each surrogate server.
const tail = path => (existsSync(path) ? readFileSync(path, 'utf8').slice(-1_500) : '');
function evidence() {
const operator = spawnSync(process.execPath, [fileURLToPath(new URL('../../scripts/database-maintenance.mjs', import.meta.url)), 'status'],
{ env, encoding: 'utf8', timeout: 10_000 });
const completed = join(root, 'data', 'database-maintenance-completed');
const controlParents = [join(root, 'data', 'database-maintenance'),
...(existsSync(completed) ? readdirSync(completed).map(id => join(completed, id)) : [])];
const workers = controlParents.flatMap(parent => (existsSync(parent) ? readdirSync(parent) : [])
.filter(name => name.startsWith('worker-')).map(name => join(parent, name, 'stderr.log')));
return [
`events: ${JSON.stringify(stubs.events())}`,
`status (exit ${operator.status}): ${(operator.stdout || operator.stderr).trim().slice(0, 1_500)}`,
...workers.map((path, index) => `worker ${index} stderr: ${tail(path)}`),
`surrogate stderr: ${tail(join(stubs.dir, 'surrogate-stderr.log'))}`,
].join('\n');
}
// vi.waitFor that appends the evidence above when its deadline expires.
async function waitWithEvidence(assertion, timeout) {
try {
await vi.waitFor(assertion, { timeout, interval: 50 });
} catch (err) {
err.message += `\n--- maintenance evidence ---\n${evidence()}`;
throw err;
}
}
describe.skipIf(process.platform === 'win32')('owned maintenance worker', () => {
it.each([[source, target], [target, source]])('cuts over the recorded operation once and releases only after the restarted server proves the target (%#)', async (from, to) => {
writeFileSync(join(root, '.env'), `PGMODE=${from.mode}\n`);
const operation = journal.begin({ source: from, target: to });
const token = journal.acquireCoordinator(operation.id);
const outcomes = await Promise.all([launch(operation.id, token), launch(operation.id, token)]);
const released = outcomes.filter(isReleased);
expect(released, outcomes.map(value => value.stderr).join('\n')).toHaveLength(1);
expect(outcomes.filter(value => value.status === 1)).toHaveLength(1);
expect(releasedResult(operation.id, token)).toEqual({ id: operation.id, stage: 'released', source: from.mode, target: to.mode,
importCommitted: true, sourceRetained: true, restartVerified: true, cosRestarted: true });
await waitWithEvidence(() => expect(stubs.events()).toContain(`server booted ${to.port}`), 15_000);
expect(stubs.events().slice(0, 6)).toEqual(['stop portos-cos', 'stop portos-server', 'dump writer=none',
'import-start', 'import-commit', 'restart portos-server']);
expect(stubs.events()).toContain('restart portos-cos');
expect(stubs.invocations('pg_dump')).toEqual([`pg_dump -h ${from.host} -p ${from.port} -U ${from.user} -d ${from.database} --no-owner --no-privileges --no-comments --if-exists --clean`]);
expect(stubs.invocations('psql')).toEqual([`psql -h ${to.host} -p ${to.port} -U ${to.user} -d ${to.database} -v ON_ERROR_STOP=1 --single-transaction`]);
expect(stubs.receivedVariables()).toEqual(['PGPASSWORD']);
expect(stubs.imported()).toBe(STUB_DUMP_COMPLETE);
expect(readFileSync(join(root, '.env'), 'utf8')).toBe(`PGMODE=${to.mode}\n`);
expect(journal.read()).toBeNull();
expect(status()).toEqual({ stage: 'idle', lastCutover: { id: operation.id, source: from.mode, target: to.mode } });
expect(createDatabaseWriterRegistry(join(root, 'data')).read()).toEqual([]);
// The one-use owner cannot run again, and its evidence is archived.
expect((await launch(operation.id, token)).status).toBe(1);
expect(readdirSync(join(root, 'data', 'database-maintenance-completed', operation.id, 'worker-' + token)))
.toEqual(expect.arrayContaining(['owner.json', 'started.json', 'group.json', 'stdout.log', 'stderr.log', 'pid']));
}, 90_000);
it('relaunches the recorded operation once; a contended operator recovery reports running or fenced, retry', async () => {
const operation = journal.begin({ source, target });
const token = journal.acquireCoordinator(operation.id);
stubs.setMode('dump', 'fail');
expect((await launch(operation.id, token)).status).toBe(1);
expect(status()).toMatchObject({ stage: 'exporting', coordinator: { state: 'exited', exitCode: 1 } });
// Hold the successor at export so a slow competing CLI cannot arrive
// after release. Ownership changes between checks deliberately fail
// closed: that contender must retry, not assume both calls exit zero.
stubs.setMode('dump', 'pause');
try {
const outcomes = await Promise.all([recover(operation.id), recover(operation.id)]);
const successful = outcomes.filter(value => value.status === 0);
expect(successful.filter(value => JSON.parse(value.stdout).recovery === 'launched'),
outcomes.map(value => value.stderr).join('\n')).toHaveLength(1);
for (const outcome of outcomes) {
if (outcome.status === 0) {
expect(JSON.parse(outcome.stdout)).toMatchObject({ id: operation.id, recovery: expect.stringMatching(/^(launched|running)$/) });
} else {
expect(outcome.status).toBe(1);
expect(outcome.stdout).toBe('');
expect(outcome.stderr.trim()).toBe('Database maintenance is fenced; journal recovery is required.');
}
}
await waitWithEvidence(() => expect(stubs.started('dump')).toBe(true), 30_000);
const retry = await recover(operation.id);
expect(retry.status, retry.stderr).toBe(0);
expect(JSON.parse(retry.stdout)).toMatchObject({ id: operation.id, recovery: 'running' });
} finally {
stubs.release('dump');
}
await waitWithEvidence(() => expect(stubs.events()).toContain(`server booted ${target.port}`), 30_000);
await waitWithEvidence(() => expect(journal.read()).toBeNull(), 10_000);
// One recovered worker: one export retry, one import, one server restart.
expect(stubs.invocations('pg_dump')).toHaveLength(2);
expect(importsStarted()).toBe(1);
expect(stubs.events().filter(line => line === 'restart portos-server')).toHaveLength(1);
// A released operation has nothing left to recover.
expect((await recover(operation.id)).status).toBe(1);
}, 90_000);
// Regression caught: a successor importing while an import started by a
// killed predecessor (whose supervisor already published an exit) still runs.
it('never lets a recovered coordinator import while a killed predecessor\'s import still runs', async () => {
const operation = journal.begin({ source, target });
const token = journal.acquireCoordinator(operation.id);
stubs.setMode('import', 'pause');
const first = launch(operation.id, token);
await vi.waitFor(() => expect(stubs.started('import')).toBe(true), { timeout: 30_000, interval: 50 });
const { pgid } = JSON.parse(readFileSync(join(controlDirFor(token), 'group.json'), 'utf8'));
strays.push(pgid);
process.kill(Number(readFileSync(join(controlDirFor(token), 'pid'), 'utf8')), 'SIGKILL');
expect((await first).status).not.toBe(78);
await vi.waitFor(() => expect(journal.coordinatorStatus(operation.id).state).toBe('exited'), { timeout: 10_000, interval: 50 });
// Neither knowing the token nor passing extra spawn options authorizes a
// different executable, arguments, control directory, or ordinary child.
const ran = join(root, 'ordinary-ran');
const ordinary = await run(`import {spawnDetached} from ${JSON.stringify(detachedUrl)};
const child=await spawnDetached(process.execPath,['-e',${JSON.stringify(`require('fs').writeFileSync(${JSON.stringify(ran)},'bad')`)}],{
controlDir:${JSON.stringify(join(root, 'ordinary'))}, maintenanceCoordinator:{id:${JSON.stringify(operation.id)},token:${JSON.stringify(token)}}});
child.on('error',()=>{process.exitCode=1;}); child.on('close',()=>{process.exitCode=2;});`);
expect(ordinary.status).toBe(1);
expect(existsSync(ran)).toBe(false);
const second = journal.recoverCoordinator(operation.id, token, randomUUID());
const refused = await launch(operation.id, second);
expect(refused.status).toBe(1);
expect(refused.stderr).toMatch(/previous coordinator's dump or import process is still running/);
expect(importsStarted()).toBe(1);
expect(status()).toEqual({ id: operation.id, stage: 'importing', source: 'native', target: 'docker',
coordinator: { state: 'exited', exitCode: 1 }, transfer: { dump: 'recorded', import: 'pending' } });
process.kill(-pgid, 'SIGKILL');
stubs.setMode('import', 'ok');
const third = journal.recoverCoordinator(operation.id, second, randomUUID());
const done = await launch(operation.id, third);
expect(isReleased(done), done.stderr).toBe(true);
expect(stubs.invocations('pg_dump')).toHaveLength(1);
expect(importsStarted()).toBe(2);
expect(status()).toMatchObject({ stage: 'idle', lastCutover: { id: operation.id } });
expect(stubs.imported()).toBe(STUB_DUMP_COMPLETE);
expect(journal.read()).toBeNull();
}, 90_000);
it('retains unresolved inventory and refuses to export', async () => {
createDatabaseWriterRegistry(join(root, 'data')).reserve(join(root, 'pending-writer'));
const operation = journal.begin({ source, target });
const token = journal.acquireCoordinator(operation.id);
const outcome = await launch(operation.id, token);
expect(outcome.status).toBe(1);
expect(outcome.stdout).toBe('');
expect(outcome.stderr).toMatch(/1 admitted launch\(es\) have not recorded a process identity/);
expect(createDatabaseWriterRegistry(join(root, 'data')).read()).toHaveLength(1);
expect(journal.read()).toEqual({ ...operation, stage: 'quiescing' });
expect(stubs.invocations('pg_dump')).toEqual([]);
}, 40_000);
});
// Ownership and entry refusals happen before any platform-specific transfer step.
describe('owned maintenance worker entry', () => {
it('rejects a foreign owner and cannot relaunch an interrupted reservation', async () => {
const operation = journal.begin({ source, target });
const token = journal.acquireCoordinator(operation.id);
expect((await launch(operation.id, randomUUID())).status).toBe(1);
expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'unregistered' });
const controlDir = journal.reserveCoordinatorWorker(operation.id, token);
writeFileSync(join(controlDir, 'stdout.log'), 'retained evidence');
expect((await launch(operation.id, token)).status).toBe(1);
expect(readFileSync(join(controlDir, 'stdout.log'), 'utf8')).toBe('retained evidence');
expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'awaiting-exit' });
expect(() => journal.recoverCoordinator(operation.id, token, randomUUID())).toThrow();
expect(stubs.events()).toEqual([]);
}, 40_000);
it('refuses damaged inventory without printing local evidence', async () => {
const registry = createDatabaseWriterRegistry(join(root, 'data'));
registry.reserve(join(root, 'pending-writer'));
const [writer] = registry.read();
writeFileSync(join(root, 'data', 'database-writers', writer.id, 'reservation.json'), 'private-example-marker');
const operation = journal.begin({ source, target });
const token = journal.acquireCoordinator(operation.id);
// This contract is the worker entry refusal, before platform-specific
// transfer. Reserve the real one-use owner and invoke the fixed worker
// directly; detached supervisor launch is exercised by the tests above.
journal.reserveCoordinatorWorker(operation.id, token);
const workerUrl = new URL('../../scripts/database-maintenance-worker.mjs', import.meta.url).href;
const outcome = await run(`process.argv=[process.execPath,'worker',${JSON.stringify(operation.id)},${JSON.stringify(token)}];
await import(${JSON.stringify(workerUrl)});`);
expect(outcome.status).toBe(1);
expect(outcome.stdout).toBe('');
expect(outcome.stderr).toContain('ownership, writer, or transfer evidence is incomplete');
expect(outcome.stderr).not.toContain('private-example-marker');
expect(outcome.stderr).not.toContain(root);
expect(outcome.stderr).not.toContain(token);
expect(journal.read()).toEqual({ ...operation, stage: 'quiescing' });
expect(stubs.invocations('pg_dump')).toEqual([]);
expect(() => journal.assertAdmission()).toThrow();
}, 40_000);
it('rejects repeated direct worker entry', async () => {
const operation = journal.begin({ source, target });
const token = journal.acquireCoordinator(operation.id);
const controlDir = journal.reserveCoordinatorWorker(operation.id, token);
journal.enterCoordinatorWorker(operation.id, token);
expect(() => journal.enterCoordinatorWorker(operation.id, token)).toThrow();
expect(journal.coordinatorStatus(operation.id)).toEqual({ state: 'awaiting-exit' });
// The stopped reservation is never repaired or reused automatically.
expect(readFileSync(join(controlDir, 'started.json'), 'utf8')).toContain(token);
});
});
describe('released worker result evidence', () => {
it('reads exact archived output when live forwarding missed the renamed directory', () => {
const id = randomUUID();
const token = randomUUID();
const archive = join(root, 'data', 'database-maintenance-completed', id, 'worker-' + token);
mkdirSync(archive, { recursive: true });
const result = { id, stage: 'released', restartVerified: true };
writeFileSync(join(archive, 'stdout.log'), JSON.stringify(result) + '\n');
expect(releasedResult(id, token)).toEqual(result);
expect(() => releasedResult(id, randomUUID())).toThrow();
writeFileSync(join(archive, 'stdout.log'), '');
expect(() => releasedResult(id, token)).toThrow('no JSON result');
writeFileSync(join(archive, 'stdout.log'), '{invalid}');
expect(() => releasedResult(id, token)).toThrow(SyntaxError);
});
});