Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { spawnSync } from 'node:child_process';
import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { delimiter, join } from 'node:path';
import { pathToFileURL } from 'node:url';
import { afterAll, describe, expect, it as vitestIt } from 'vitest';
import { setTimeout as delay } from 'node:timers/promises';
import { ownTestBodies } from '../server/lib/mockPathsDataRoot.js';
// Where PID 1 does not reap orphans (some containers), a killed orphan stays a
// zombie that still answers signal 0; on Linux its /proc state says it exited.
const exited = (pid) => {
try { process.kill(pid, 0); } catch { return true; }
if (process.platform !== 'linux') return false;
try { return /^[ZX]$/.test(readFileSync(`/proc/${pid}/stat`, 'utf8').replace(/^.*\) /s, '')[0]); } catch { return true; }
};
const { it, drain } = ownTestBodies(vitestIt);
afterAll(drain);
const supervisor = new URL('./watch-ci-tests.js', import.meta.url).href;
const reporter = new URL('./vitestRunDiagnostics.js', import.meta.url).href;
describe.skipIf(process.platform === 'win32')('independent browser test watchdog', () => {
for (const [stalled, status, denied, orphan] of [[false, 0, false, false], [false, 17, false, false], [true, 17, false, false], [true, 17, true, false], [false, 0, false, true]]) {
it(`preserves status or interrupts a stopped launcher with prior progress (stalled=${stalled}, status=${status}, denied=${denied}, orphan=${orphan})`, async () => {
const root = mkdtempSync(join(tmpdir(), 'ci-watchdog-'));
const reports = join(root, 'reports');
mkdirSync(reports);
try {
const npm = join(root, 'npm.mjs');
writeFileSync(npm, `
import reporter from ${JSON.stringify(reporter)};
import { writeFileSync } from 'node:fs';
import { spawn } from 'node:child_process';
const hooks = reporter();
hooks.onInit({ onClose() {} });
hooks.onTestModuleQueued({ moduleId: 'server/lib/synthetic.test.js' });
const orphan = ${orphan} ? spawn(process.execPath, ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000); process.send('ready');"], { stdio: ['ignore', 'ignore', 'ignore', 'ipc'] }) : null;
writeFileSync(${JSON.stringify(join(root, 'owned.json'))}, JSON.stringify({ worker: process.pid, launcher: process.ppid, orphan: orphan?.pid }));
if (orphan) { await new Promise(resolve => orphan.once('message', resolve)); orphan.disconnect(); orphan.unref(); }
// Fixture handshake complete: the scenario deadline starts here.
writeFileSync(${JSON.stringify(join(root, 'ready'))}, '');
if (${stalled}) {
// Stop the launcher's event loop: its own signal handler cannot help.
// The independent supervisor must retire the inherited known group.
process.on('SIGTERM', () => {});
process.kill(process.ppid, 'SIGSTOP');
setInterval(() => {}, 1000);
} else process.exitCode = ${status};
`);
writeFileSync(join(root, 'npm'), `#!${process.execPath}\nimport(${JSON.stringify(pathToFileURL(npm).href)});`, { mode: 0o755 });
const result = spawnSync(process.execPath, ['--input-type=module', '-e', `
import { existsSync } from 'node:fs';
import { watchTests } from ${JSON.stringify(supervisor)};
const kill = process.kill;
if (${denied}) process.kill = (pid, signal) => {
if (pid < 0 && signal === 'SIGKILL') throw Object.assign(Error('synthetic denied signal'), { code: 'EPERM' });
return kill(pid, signal);
};
// Launcher startup under load is not the behavior under test: the deadline
// clock starts at the fixture's ready marker. Only the stalled scenarios
// depend on the deadline firing (124); the others get headroom so teardown
// after the worker exits cannot race it. A gate that never opens is bounded.
const deadlineGate = async () => {
const until = Date.now() + 8000;
while (!existsSync(${JSON.stringify(join(root, 'ready'))}) && Date.now() < until) await new Promise(r => setTimeout(r, 20));
};
try { process.exitCode = await watchTests('server', { deadlineMs: ${stalled ? 1500 : 5000}, progressMs: 300, graceMs: 200, deadlineGate }); }
catch (error) { console.error('Synthetic supervisor failed closed: ' + error.message); process.exitCode = 1; }
`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env,
PATH: `${root}${delimiter}${process.env.PATH}`, PORTOS_CI_DIAGNOSTICS_DIR: reports,
// The supervised launcher is the real run-ci-tests.js: pin its plan so the
// job's own CI_TEST_MODE/CI_SHARD/memo settings cannot put planning work
// in front of the fake npm the deadlines below are sized for.
CI_TEST_MODE: 'full', CI_TEST_FILES: '[]', CI_TEST_SOURCES: '[]', CI_SHARD: '', PORTOS_TEST_MEMO: 'off',
GITHUB_STEP_SUMMARY: '', GITHUB_ENV: '', NODE_DISABLE_COMPILE_CACHE: '1' } });
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(denied || orphan ? 1 : stalled ? 124 : status);
const owned = JSON.parse(readFileSync(join(root, 'owned.json'), 'utf8'));
if (!denied) for (const pid of [owned.worker, owned.launcher, ...(owned.orphan ? [owned.orphan] : [])]) expect(exited(pid), `pid ${pid}`).toBe(true);
if (stalled) {
expect(result.stdout).toContain('CI test watchdog: 0s elapsed');
const progress = result.stderr.indexOf('server/lib/synthetic.test.js');
expect(progress).toBeGreaterThanOrEqual(0);
expect(progress).toBeLessThan(result.stderr.indexOf('subordinate deadline exceeded'));
if (denied) {
expect(result.stderr).toContain('owned-runner settlement unverified');
expect(result.stderr).not.toContain('owned launcher group stopped');
} else {
expect(result.stderr).toContain('owned launcher group stopped');
expect(result.stderr).toContain('independently detached browser groups remain unverified');
}
expect(result.stderr).not.toContain(root);
} else {
expect(result.stderr).not.toContain('subordinate deadline exceeded');
if (orphan) {
expect(result.stderr).toContain('launcher closed with inherited group members remaining');
expect(result.stderr).toContain('owned launcher group stopped');
}
}
} finally {
// Even an assertion or supervisor regression must retire this fixture's
// positively recorded group, including a launcher stopped with SIGSTOP.
const ownedPath = join(root, 'owned.json');
if (!existsSync(ownedPath)) throw Error('Synthetic watchdog ownership unavailable; scratch retained.');
{
const { launcher } = JSON.parse(readFileSync(ownedPath, 'utf8'));
try { process.kill(-launcher, 'SIGKILL'); }
catch (error) { if (error.code !== 'ESRCH') throw error; }
const until = Date.now() + 5000;
while (true) {
// macOS answers EPERM while a killed group leader is a not-yet-reaped
// zombie; it settles to ESRCH, so keep polling inside the bounded wait.
try { process.kill(-launcher, 0); }
catch (error) { if (error.code === 'ESRCH') break; if (error.code !== 'EPERM') throw error; }
if (Date.now() >= until) throw Error('Synthetic watchdog group settlement unverified; scratch retained.');
await delay(25);
}
}
rmSync(root, { recursive: true, force: true });
}
}, 15000);
}
});