Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

6.2 KB
Raw Blame
#!/usr/bin/env node

import { spawn } from 'node:child_process';
import { setTimeout as delay } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { collectTestDiagnostics } from './ci-test-diagnostics.js';
import { isDirectlyInvoked } from './lib/directInvocation.js';
import { drainBrowserProcessGroup } from '../server/services/htmlComposition/browserProcessGroup.js';

// A separate event loop owns this deadline: a blocked launcher/test worker must
// not prevent evidence collection or termination. Whole-runner loss can still
// prevent both execution and delivery of these logs; that remains inconclusive.
export async function watchTests(scope, {
  env = process.env, deadlineMs = 12 * 60_000, progressMs = 60_000, graceMs = 5000, settleMs = 5000,
  // Test seam: the deadline clock starts when this resolves instead of at spawn,
  // so a fixture can exclude variable launcher startup. Production omits it.
  deadlineGate,
} = {}) {
  if (process.platform === 'win32') throw Error('Browser test supervision requires POSIX process groups.');
  if (!['server', 'client'].includes(scope)) throw Error('Expected a server or client test workspace.');
  const root = env.PORTOS_CI_DIAGNOSTICS_DIR;
  if (!root) throw Error('Browser test supervision requires durable diagnostics.');
  const child = spawn(process.execPath, [fileURLToPath(new URL('./run-ci-tests.js', import.meta.url)), scope], {
    detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, PORTOS_CI_TEST_SUPERVISED_GROUP: '1' },
  });
  child.stdout.on('data', chunk => process.stdout.write(chunk));
  child.stderr.on('data', chunk => process.stderr.write(chunk));
  const started = Date.now();
  let timedOut = false, interrupted = false, stopping, spawnError, closeResult, finish, leftovers = false;
  const completed = new Promise(resolve => { finish = resolve; });
  child.once('error', error => { spawnError = error; });
  child.once('close', (code, signal) => { closeResult = { code, signal }; finish({}); });
  let collecting;
  const collect = () => collecting ??= collectTestDiagnostics(root, {
    // Linux browser jobs have no Windows crash query. Keep periodic output
    // bounded and sanitized through the same durable collector as final failure.
    diagnose: async () => {}, writeSummary: () => {},
  }).finally(() => { collecting = undefined; });
  const stop = () => stopping ??= (async () => {
    console.error('CI test watchdog: collecting evidence before owned-runner shutdown.');
    await collect().catch(() => console.error('CI test watchdog: deadline evidence unavailable.'));
    // Let the launcher's existing interruption path settle first, then retire
    // its entire known-owned group even if that launcher's event loop is stuck.
    child.kill('SIGTERM');
    await delay(graceMs);
    try { process.kill(-child.pid, 'SIGKILL'); }
    catch (error) { if (error.code !== 'ESRCH') throw error; }
    console.error('CI test watchdog: owned launcher group termination requested.');
    const until = Date.now() + settleMs;
    while (!closeResult) {
      if (Date.now() >= until) throw Error('Owned launcher close unverified after SIGKILL.');
      await delay(25);
    }
    await drainBrowserProcessGroup(child.pid, performance.now() + settleMs, Math.min(graceMs, settleMs / 2));
    console.error('CI test watchdog: owned launcher group stopped; independently detached browser groups remain unverified.');
    await collect();
  })();
  const requestStop = () => stop().catch(error => {
    // A refused signal or unresponsive OS cannot prove quiescence. Fail closed
    // with evidence; the workflow's outer deadline remains the final fallback.
    const code = ['EPERM', 'EACCES', 'EINVAL', 'ENOSYS'].includes(error.code) ? error.code : 'unavailable';
    console.error(`CI test watchdog: owned-runner settlement unverified; code=${code}; outer step cleanup still required.`);
    child.stdout.destroy();
    child.stderr.destroy();
    child.unref();
    finish({ error });
  });
  const interrupt = () => { interrupted = true; requestStop(); };
  process.on('SIGINT', interrupt);
  process.on('SIGTERM', interrupt);
  const progress = setInterval(() => {
    console.log(`CI test watchdog: ${Math.floor((Date.now() - started) / 1000)}s elapsed; deadline=${deadlineMs / 1000}s.`);
    collect().catch(() => console.error('CI test watchdog: progress unavailable.'));
  }, progressMs);
  let deadline, gateSettled = false;
  const armDeadline = () => {
    if (gateSettled) return;
    gateSettled = true;
    deadline = setTimeout(() => {
      timedOut = true;
      console.error('CI test watchdog: subordinate deadline exceeded; this step will fail.');
      requestStop();
    }, deadlineMs);
  };
  // A failed gate arms the deadline at once: supervision never fails open.
  if (deadlineGate) Promise.resolve().then(deadlineGate).then(armDeadline, armDeadline);
  else armDeadline();
  try {
    const { error } = await completed;
    if (error) throw error;
    if (!stopping && child.pid) {
      // Only a positively closed, retained group is eligible for this existing
      // numeric/state-only observer; it also detects stdio-detached leftovers.
      const state = { error: 'none' };
      await drainBrowserProcessGroup(child.pid, performance.now() + settleMs, Math.min(graceMs, settleMs / 2), state);
      if (state.term) {
        leftovers = true;
        console.error('CI test watchdog: launcher closed with inherited group members remaining; this step will fail.');
        console.error('CI test watchdog: owned launcher group stopped; independently detached browser groups remain unverified.');
      }
    }
    if (stopping) await stopping;
    if (spawnError) throw spawnError;
    return timedOut ? 124 : interrupted ? 143 : leftovers ? 1 : closeResult.code ?? 1;
  } finally {
    gateSettled = true;
    clearInterval(progress);
    clearTimeout(deadline);
    process.removeListener('SIGINT', interrupt);
    process.removeListener('SIGTERM', interrupt);
    await collecting?.catch(() => console.error('CI test watchdog: final progress unavailable.'));
  }
}

if (isDirectlyInvoked(import.meta.url)) {
  try { process.exitCode = await watchTests(process.argv[2]); }
  catch { console.error('CI test watchdog failed; owned-runner settlement is unverified.'); process.exitCode = 1; }
}