Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: start watchdog deadline after fixture readiness so orphan settlement is deterministic (#13122)

Adam Eivyae6a982ca738
diff --git a/scripts/watch-ci-tests.js b/scripts/watch-ci-tests.js
index 9d649e8dfc..fe8e4c0ebe 100644
--- a/scripts/watch-ci-tests.js
+++ b/scripts/watch-ci-tests.js
@@ -12,6 +12,9 @@ import { drainBrowserProcessGroup } from '../server/services/htmlComposition/bro
// prevent both execution and delivery of these logs; that remains inconclusive.
export async function watchTests(scope, {
env = process.env, deadlineMs = 12 * 60_000, progressMs = 60_000, graceMs = 5000, settleMs = 5000,
+ // Test seam: the deadline clock starts when this resolves instead of at spawn,
+ // so a fixture can exclude variable launcher startup. Production omits it.
+ deadlineGate,
} = {}) {
if (process.platform === 'win32') throw Error('Browser test supervision requires POSIX process groups.');
if (!['server', 'client'].includes(scope)) throw Error('Expected a server or client test workspace.');
@@ -69,11 +72,19 @@ export async function watchTests(scope, {
console.log(`CI test watchdog: ${Math.floor((Date.now() - started) / 1000)}s elapsed; deadline=${deadlineMs / 1000}s.`);
collect().catch(() => console.error('CI test watchdog: progress unavailable.'));
}, progressMs);
- const deadline = setTimeout(() => {
- timedOut = true;
- console.error('CI test watchdog: subordinate deadline exceeded; this step will fail.');
- requestStop();
- }, deadlineMs);
+ let deadline, gateSettled = false;
+ const armDeadline = () => {
+ if (gateSettled) return;
+ gateSettled = true;
+ deadline = setTimeout(() => {
+ timedOut = true;
+ console.error('CI test watchdog: subordinate deadline exceeded; this step will fail.');
+ requestStop();
+ }, deadlineMs);
+ };
+ // A failed gate arms the deadline at once: supervision never fails open.
+ if (deadlineGate) Promise.resolve().then(deadlineGate).then(armDeadline, armDeadline);
+ else armDeadline();
try {
const { error } = await completed;
if (error) throw error;
@@ -92,6 +103,7 @@ export async function watchTests(scope, {
if (spawnError) throw spawnError;
return timedOut ? 124 : interrupted ? 143 : leftovers ? 1 : closeResult.code ?? 1;
} finally {
+ gateSettled = true;
clearInterval(progress);
clearTimeout(deadline);
process.removeListener('SIGINT', interrupt);
diff --git a/scripts/watch-ci-tests.test.js b/scripts/watch-ci-tests.test.js
index bed7723a6d..322ea330da 100644
--- a/scripts/watch-ci-tests.test.js
+++ b/scripts/watch-ci-tests.test.js
@@ -31,6 +31,8 @@ describe.skipIf(process.platform === 'win32')('independent browser test watchdog
const orphan = ${orphan} ? spawn(process.execPath, ['-e', "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000); process.send('ready');"], { stdio: ['ignore', 'ignore', 'ignore', 'ipc'] }) : null;
writeFileSync(${JSON.stringify(join(root, 'owned.json'))}, JSON.stringify({ worker: process.pid, launcher: process.ppid, orphan: orphan?.pid }));
if (orphan) { await new Promise(resolve => orphan.once('message', resolve)); orphan.disconnect(); orphan.unref(); }
+ // Fixture handshake complete: the scenario deadline starts here.
+ writeFileSync(${JSON.stringify(join(root, 'ready'))}, '');
if (${stalled}) {
// Stop the launcher's event loop: its own signal handler cannot help.
// The independent supervisor must retire the inherited known group.
@@ -41,13 +43,22 @@ describe.skipIf(process.platform === 'win32')('independent browser test watchdog
`);
writeFileSync(join(root, 'npm'), `#!${process.execPath}\nimport(${JSON.stringify(pathToFileURL(npm).href)});`, { mode: 0o755 });
const result = spawnSync(process.execPath, ['--input-type=module', '-e', `
+ import { existsSync } from 'node:fs';
import { watchTests } from ${JSON.stringify(supervisor)};
const kill = process.kill;
if (${denied}) process.kill = (pid, signal) => {
if (pid < 0 && signal === 'SIGKILL') throw Object.assign(Error('synthetic denied signal'), { code: 'EPERM' });
return kill(pid, signal);
};
- try { process.exitCode = await watchTests('server', { deadlineMs: 1500, progressMs: 300, graceMs: 200 }); }
+ // Launcher startup under load is not the behavior under test: the deadline
+ // clock starts at the fixture's ready marker. Only the stalled scenarios
+ // depend on the deadline firing (124); the others get headroom so teardown
+ // after the worker exits cannot race it. A gate that never opens is bounded.
+ const deadlineGate = async () => {
+ const until = Date.now() + 8000;
+ while (!existsSync(${JSON.stringify(join(root, 'ready'))}) && Date.now() < until) await new Promise(r => setTimeout(r, 20));
+ };
+ try { process.exitCode = await watchTests('server', { deadlineMs: ${stalled ? 1500 : 5000}, progressMs: 300, graceMs: 200, deadlineGate }); }
catch (error) { console.error('Synthetic supervisor failed closed: ' + error.message); process.exitCode = 1; }
`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env,
PATH: `${root}${delimiter}${process.env.PATH}`, PORTOS_CI_DIAGNOSTICS_DIR: reports,
@@ -91,8 +102,10 @@ describe.skipIf(process.platform === 'win32')('independent browser test watchdog
catch (error) { if (error.code !== 'ESRCH') throw error; }
const until = Date.now() + 5000;
while (true) {
+ // macOS answers EPERM while a killed group leader is a not-yet-reaped
+ // zombie; it settles to ESRCH, so keep polling inside the bounded wait.
try { process.kill(-launcher, 0); }
- catch (error) { if (error.code === 'ESRCH') break; throw error; }
+ catch (error) { if (error.code === 'ESRCH') break; if (error.code !== 'EPERM') throw error; }
if (Date.now() >= until) throw Error('Synthetic watchdog group settlement unverified; scratch retained.');
await delay(25);
}