The design language that makes your AI harness better at design.
JavaScript
#!/usr/bin/env node
import { spawn } from 'node:child_process';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { DEFAULT_SUITES, OPT_IN_SUITES, SUITES, expandSuites } from './test-suites.mjs';
import { createGroupShutdown, trackChildExit } from './lib/process-group.mjs';
import {
REPO_ENV,
REPO_PATH_ENV,
RUN_ID_ENV,
alive,
findLiveServers,
killLiveServers,
makeRunId,
repoMarker,
} from './lib/live-server-processes.mjs';
const REPO_ROOT = path.resolve(fileURLToPath(new URL('..', import.meta.url)));
// Global wall-clock backstop for any one command. Even with per-test timeouts
// and client-side network deadlines in place, a wedged tool or an orphaned
// grandchild can keep a runner alive forever; this cap guarantees the sweep
// terminates. Per-suite `wallClockMs` overrides it; the env var overrides both.
const DEFAULT_WALL_CLOCK_MS = Number(process.env.IMPECCABLE_TEST_WALL_CLOCK_MS) || 1_200_000;
const args = process.argv.slice(2);
if (args.includes('--help') || args.includes('-h')) {
printHelp();
process.exit(0);
}
if (args.includes('--list')) {
printSuites();
process.exit(0);
}
if (args.includes('--cleanup')) {
process.exit(cleanupRepoServers());
}
/**
* Suite commands run in their own process group, which buys two things: the
* wall-clock cap can SIGKILL a wedged tree whole (the runner, its per-file
* `node --test` workers, and any grandchildren or browsers they left open),
* and a Ctrl-C can end that same tree deterministically instead of orphaning
* it. Nothing in this file may use spawnSync: a blocked event loop cannot run
* the signal handlers that make either guarantee, and cannot reap the child it
* is waiting on either.
*/
const shutdown = createGroupShutdown();
for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
process.on(sig, () => { void shutdown.onSignal(exitCodeForSignal(sig)); });
}
// Nothing can be awaited here, so this is the one path that does not wait.
process.on('exit', () => shutdown.onExit());
/** The shell convention for "killed by signal N": 130 SIGINT, 143 SIGTERM, 129 SIGHUP. */
function exitCodeForSignal(sig) {
return 128 + (os.constants.signals[sig] ?? 0);
}
const requestedSuites = args.filter((arg) => !arg.startsWith('-'));
let suites;
try {
suites = expandSuites(requestedSuites);
} catch (err) {
console.error(err.message);
process.exit(1);
}
await main();
async function main() {
for (const suiteName of suites) {
const suite = SUITES[suiteName];
console.log(`\n## test:${suiteName}`);
console.log(suite.description);
for (const command of suite.commands) {
await runCommand(command, suiteName);
}
}
}
async function runCommand(command, suiteName) {
const runId = makeRunId(REPO_ROOT);
const env = {
...process.env,
[RUN_ID_ENV]: runId,
// The hash is what matching uses; the path rides along for a human reading
// `ps -E` output and is never matched on.
[REPO_ENV]: repoMarker(REPO_ROOT),
[REPO_PATH_ENV]: REPO_ROOT,
...(command.env || {}),
};
const wallClockMs = command.wallClockMs ?? DEFAULT_WALL_CLOCK_MS;
if (command.runner === 'bun') {
await runProcess('bun', ['test', ...command.files], { env, wallClockMs });
} else if (command.runner === 'node') {
// One invocation for the whole file list: node --test runs each file in
// its own child process regardless, so isolation is unchanged, but the
// runner-per-file spawn overhead is gone and files execute concurrently.
// Measured on the live suite (38 files): 52s serial-per-file vs 18s
// batched at concurrency 4. Suites can pin `concurrency: 1` if their
// tests ever contend for a shared resource.
const nodeArgs = ['--test', `--test-concurrency=${command.concurrency ?? 4}`];
if (command.timeoutMs) nodeArgs.push(`--test-timeout=${command.timeoutMs}`);
if (command.forceExit) nodeArgs.push('--test-force-exit');
nodeArgs.push(...command.files);
await runProcess(process.execPath, nodeArgs, { env, wallClockMs });
} else {
throw new Error(`Unsupported test runner "${command.runner}"`);
}
await assertNoLeakedServers(runId, suiteName);
}
function runProcess(cmd, args, { env, wallClockMs }) {
console.log(`$ ${formatCommand(cmd, args)}`);
return new Promise((resolve) => {
const child = spawn(cmd, args, {
// Own process group: the wall-clock cap and the shutdown handler can
// then take down the runner, every test file it forked, and anything
// those forked, in one signal.
detached: true,
// stdin is deliberately not inherited. A detached child is a background
// process group on the terminal, and a background read of the tty stops
// the process with SIGTTIN. No suite reads the runner's stdin.
stdio: ['ignore', 'inherit', 'inherit'],
env,
});
// Registered before the handlers below, so `hasExited` is already set by
// the time they run and a shutdown mid-exit does not signal a dead pid.
const running = shutdown.track(trackChildExit(child));
let timedOut = false;
const timer = wallClockMs
? setTimeout(() => {
timedOut = true;
console.error(
`\n[run-tests] wall-clock cap of ${wallClockMs}ms exceeded for "${formatCommand(cmd, args)}"; ` +
'killing the process group (SIGKILL).',
);
// A test blocked in a synchronous spawnSync cannot be reached by
// node's --test-timeout, so this is the guaranteed end of the tree.
// No graceful phase: the cap has already been generous.
try { process.kill(-running.child.pid, 'SIGKILL'); }
catch { try { running.child.kill('SIGKILL'); } catch { /* already gone */ } }
}, wallClockMs)
: null;
child.on('error', (err) => {
if (timer) clearTimeout(timer);
shutdown.release();
console.error(err.message);
process.exit(1);
});
child.on('exit', (code, signal) => {
if (timer) clearTimeout(timer);
shutdown.release();
if (shutdown.shuttingDown) return;
if (timedOut) {
// A wedged suite is one of the ways servers are left behind, so sweep
// before reporting rather than walking away from them.
assertNoLeakedServers(env[RUN_ID_ENV], null).finally(() => process.exit(1));
return;
}
if (signal) {
console.error(`[run-tests] "${formatCommand(cmd, args)}" killed by signal ${signal}`);
assertNoLeakedServers(env[RUN_ID_ENV], null).finally(() => process.exit(1));
return;
}
if (code !== 0) {
// Leaked servers are still worth reporting on a failing suite: a
// failure before teardown is one of the ways they are left behind.
assertNoLeakedServers(env[RUN_ID_ENV], null).finally(() => process.exit(code || 1));
return;
}
resolve();
});
});
}
/**
* Fail the run when a suite left live servers behind.
*
* The whole point of the guard is that a leak shows up in the run that caused
* it rather than as a wedged port days later, so it is an error, not a warning.
* The leaked servers are killed either way, so the next suite still gets its
* ports.
*/
async function assertNoLeakedServers(runId, suiteName) {
if (!runId || process.env.IMPECCABLE_SKIP_LEAK_CHECK === '1') return;
// A server asked to stop needs a moment to actually go.
let leaked = [];
for (let attempt = 0; attempt < 10; attempt += 1) {
leaked = findLiveServers({ runId });
if (!leaked.length) return;
await new Promise((r) => setTimeout(r, 200));
}
killLiveServers(leaked);
const label = suiteName ? `test:${suiteName}` : 'the suite';
console.error(`\nLeaked live servers: ${label} left ${leaked.length} live server process(es) running.`);
for (const { pid, command } of leaked) console.error(` pid ${pid} ${command}`);
console.error('They have been killed. A live server outliving its suite means a teardown path');
console.error('was skipped; see tests/lib/live-servers.mjs for how servers are meant to be tracked.');
console.error('Set IMPECCABLE_SKIP_LEAK_CHECK=1 to bypass this check.');
process.exit(1);
}
/**
* `bun run test:cleanup`: kill live servers this checkout's tests left behind.
*
* Scoped to servers carrying this checkout's `IMPECCABLE_TEST_REPO` marker, so
* a live session the developer started themselves in this same repo is not a
* candidate. A server from a run that predates the marker is not found here and
* has to be killed by hand.
*/
function cleanupRepoServers() {
const leaked = findLiveServers({ repo: REPO_ROOT });
if (!leaked.length) {
console.log('No leftover live servers from this repo\'s test runs.');
return 0;
}
for (const { pid, command } of leaked) console.log(`killing pid ${pid} ${command}`);
killLiveServers(leaked);
const survivors = leaked.filter(({ pid }) => alive(pid));
if (survivors.length) {
console.error(`Could not kill ${survivors.length} process(es): ${survivors.map((p) => p.pid).join(', ')}`);
return 1;
}
console.log(`Killed ${leaked.length} leftover live server process(es).`);
return 0;
}
function formatCommand(cmd, args) {
const bin = cmd === process.execPath ? 'node' : cmd;
return [bin, ...args].join(' ');
}
function printHelp() {
console.log(`Usage: node scripts/run-tests.mjs [suite...]
Aliases:
default ${DEFAULT_SUITES.join(', ')}
all-local ${DEFAULT_SUITES.join(', ')}
all ${[...DEFAULT_SUITES, ...OPT_IN_SUITES].join(', ')}
Run with --list to see suite contents.
Run with --cleanup to kill live servers a previous run left behind.`);
}
function printSuites() {
for (const [name, suite] of Object.entries(SUITES)) {
const marker = suite.optIn ? ' (opt-in)' : '';
console.log(`\n${name}${marker}`);
console.log(` ${suite.description}`);
for (const command of suite.commands) {
console.log(` ${command.runner}:`);
for (const file of command.files) console.log(` ${file}`);
}
}
}