Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
feat(gate): one machine pool of test-file tokens, longest first [C7]
Corey Olsonc36419de4bfe
diff --git a/bin/run-tests.js b/bin/run-tests.jsindex aed6761e..c1d259a6 100644--- a/bin/run-tests.js+++ b/bin/run-tests.js@@ -16,6 +16,7 @@ import { channel } from 'node:diagnostics_channel';import { AGENT_SHELL_MARKERS, SSH_SHELL_MARKERS } from '../src/person.js';import { isLandingWorkerIdentityTimeout, landingWorkerIdentityQuery, parseLandingWorkerIdentity } from '../src/landing-worker-identity.js';import { foreignNodeOptions, isolateTestMachine } from '../src/test-isolation.js';+import { TEST_FILE_POOL_ENV, longestFirst, takeTestFileToken, testFilePoolValue } from '../src/test-file-pool.js';const runner = fileURLToPath(import.meta.url);const LANDING_RUN_ENV = 'PULLBOARD_LANDING_RUN';const LANDING_WORKER_STARTS_ENV = 'PULLBOARD_LANDING_WORKER_STARTS';@@ -225,6 +226,8 @@ function testEnvironment(sandbox, gateMarkers = {}) {env.PULLBOARD_MACHINE_HOME = join(sandbox, 'machine');// A test finds its runner by this pid, never by parsing ps text; a nested runner sets its own (#1073).env.PULLBOARD_TEST_RUNNER_PID = String(process.pid);+ // #1046: every file of every run takes a token from one machine pool; a nested run keeps its parent's loan.+ env[TEST_FILE_POOL_ENV] = testFilePoolValue(process.env);isolateTestMachine(env, sandbox); // #1068: no settings, resources, runners or host markers of the real machine.// init and resume start a background view by default; a test opts back in by deleting this.env.PULLBOARD_VIEW_BACKGROUND = '0';@@ -1300,6 +1303,36 @@ function explicitTestFiles(args) {return files;}+/**+ * Start the longest test files first, by the last landing's recorded wall times (#1046), so the slowest file+ * never starts last and holds the run open alone. Options and their values keep their places.+ *+ * @param {string[]} args+ * @returns {Promise<string[]>}+ */+async function longestFilesFirst(args) {+ const at = [];+ let takesValue = false;+ for (let index = 0; index < args.length; index += 1) {+ const argument = args[index];+ if (takesValue) { takesValue = false; continue; }+ if (argument === '--') { for (let rest = index + 1; rest < args.length; rest += 1) at.push(rest); break; }+ if (argument.startsWith('-')) {+ if (!argument.includes('=')) takesValue = optionTakesValue(argument);+ continue;+ }+ at.push(index);+ }+ if (at.length < 2) return args;+ const { readFileTimes } = await import('../src/landing-shards.js');+ const times = readFileTimes(process.cwd());+ if (!Object.keys(times).length) return args;+ const ordered = longestFirst(at.map(index => args[index]), times, file => relative(process.cwd(), resolve(file)));+ const result = [...args];+ at.forEach((index, position) => { result[index] = ordered[position]; });+ return result;+}+/** Admit only explicit regular in-root files; directories and uncertain options remain ordinary. */function namesOnlyTestFiles(args) {let takesValue = false;@@ -1407,6 +1440,7 @@ async function executeTests(args, sandbox, gateMarkers, fullSuite, concurrency)}if (files) selectedArgs = [...args, ...files];}+ selectedArgs = await longestFilesFirst(selectedArgs);const result = process.env[LANDING_RUN_ENV] === '1'? await runLandingTests(selectedArgs, env, concurrency): runTests(selectedArgs, env, concurrency);@@ -1599,5 +1633,7 @@ if (process.argv[1] && resolve(process.argv[1]) === runner) {process.env.NODE_OPTIONS = [process.env.NODE_OPTIONS, `--import=${guard.href}`].filter(Boolean).join(' ');}}+ // After any browser wait, so a file never holds a core while it queues for a browser (#1046).+ if (process.env.NODE_TEST_CONTEXT === 'child-v8') await takeTestFileToken();if (!landingWorker) installTestWatchdog();}diff --git a/src/test-file-pool.js b/src/test-file-pool.jsnew file mode 100644index 00000000..805cd86e--- /dev/null+++ b/src/test-file-pool.js@@ -0,0 +1,96 @@+/**+ * One machine-wide pool of test-file tokens [C7]: every test-file process takes a token before its tests run,+ * so the files running at once on a machine never exceed its cores, however many runs share it.+ *+ * Measured 22 Oct: 28 runs at --test-concurrency=10 kept 54 file processes live on 14 cores, and passing files+ * ran 18.5 times slower in landings than alone. A per-run limit cannot see the other runs; this pool can.+ *+ * A test file that starts its own run-tests and waits on it lends its token to that nested run: the nested+ * files take the lent token one at a time, never a second token from the pool, so N saturated outer files+ * with nested runs still run at most N files at once, and a nested run can never wait on its own parent.+ */+import { availableParallelism } from 'node:os';+import { randomUUID } from 'node:crypto';+import { machineHome } from './machine-home.js';++/** Names the pool a test-file process takes from, and the token its parent lent, as JSON. */+export const TEST_FILE_POOL_ENV = 'PULLBOARD_TEST_FILE_POOL';+/** Overrides the pool size; a test models a smaller machine with it. */+export const TEST_FILE_CORES_ENV = 'PULLBOARD_TEST_FILE_CORES';+/** The resource row every top-level test file takes from. */+const POOL_NAME = 'test-files';++/**+ * Read the pool a test-file process belongs to, or null outside a run-tests run.+ *+ * @param {NodeJS.ProcessEnv} env+ * @returns {{ home: string, cores: number, loan?: string } | null}+ */+export function testFilePool(env = process.env) {+ const value = env[TEST_FILE_POOL_ENV];+ if (!value) return null;+ try {+ const pool = JSON.parse(value);+ if (typeof pool?.home !== 'string' || !Number.isSafeInteger(pool.cores) || pool.cores < 1) return null;+ if (pool.loan !== undefined && typeof pool.loan !== 'string') return null;+ return pool;+ } catch { return null; }+}++/**+ * The pool value a run passes to its test files: an inherited one (a nested run keeps its parent's pool and+ * loan), or this machine's pool sized to its cores. One variable carries both, so no copy of the environment+ * can keep the pool and drop the loan, which would let a nested file wait on a token its parent holds.+ *+ * @param {NodeJS.ProcessEnv} env+ * @returns {string}+ */+export function testFilePoolValue(env = process.env) {+ if (testFilePool(env)) return env[TEST_FILE_POOL_ENV];+ const override = Number(env[TEST_FILE_CORES_ENV]);+ const cores = Number.isSafeInteger(override) && override > 0 ? override : Math.max(1, availableParallelism());+ return JSON.stringify({ home: machineHome(), cores });+}++/**+ * Take this test-file process's token and hold it until the process exits. A file with a lent token takes+ * that one; any other takes one from the machine pool. The file then lends its own token to anything it starts.+ *+ * @param {NodeJS.ProcessEnv} env - Changed in place: descendants see this file's loan.+ * @returns {Promise<void>}+ */+export async function takeTestFileToken(env = process.env) {+ const pool = testFilePool(env);+ if (!pool) return;+ const { takeResource } = await import('./resources.js');+ // The test file runs with a private machine home; the pool lives in the real one, named by the run.+ const ownHome = env.PULLBOARD_MACHINE_HOME;+ env.PULLBOARD_MACHINE_HOME = pool.home;+ let lease;+ try {+ lease = await takeResource(pool.loan+ ? { name: pool.loan, capacity: 1, kind: 'test-file' }+ : { name: POOL_NAME, capacity: pool.cores, kind: 'test-file' });+ } finally {+ if (ownHome === undefined) delete env.PULLBOARD_MACHINE_HOME;+ else env.PULLBOARD_MACHINE_HOME = ownHome;+ }+ process.once('exit', () => lease.release());+ env[TEST_FILE_POOL_ENV] = JSON.stringify({ home: pool.home, cores: pool.cores, loan: `test-file-loan:${randomUUID()}` });+}++/**+ * Order test files longest-first by their last recorded wall time, so the slowest file never starts last.+ * A file with no record keeps its place after the timed ones; ties keep the given order.+ *+ * @param {string[]} files - As given to the runner, relative to `cwd` or absolute.+ * @param {Record<string, number>} times - Wall time in ms by repository-relative path.+ * @param {(file: string) => string} key - Maps a given file to its repository-relative path.+ * @returns {string[]}+ */+export function longestFirst(files, times, key) {+ const ms = file => times[key(file)];+ return files.map((file, index) => ({ file, index, time: ms(file) }))+ .sort((a, b) => (Number.isFinite(b.time) ? b.time : -1) - (Number.isFinite(a.time) ? a.time : -1) || a.index - b.index)+ .map(entry => entry.file);+}diff --git a/test/gate-admission.test.js b/test/gate-admission.test.jsnew file mode 100644index 00000000..5fa3a251--- /dev/null+++ b/test/gate-admission.test.js@@ -0,0 +1,95 @@+/** Test files on one machine never outnumber its cores, and the longest files start first [C7]. */+import assert from 'node:assert/strict';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { fileURLToPath } from 'node:url';+import { test } from 'node:test';+import { runFixtureChild, runFixtureChildAsync } from './fixture-child.js';+import { TEST_FILE_CORES_ENV, TEST_FILE_POOL_ENV } from '../src/test-file-pool.js';++const runner = fileURLToPath(new URL('../bin/run-tests.js', import.meta.url));++/** A fixture test file that logs when its body starts and ends, holding its token for `ms`. */+function timedFile(name, ms, body = '') {+ return `import { appendFileSync } from 'node:fs';+import { test } from 'node:test';+const mark = at => appendFileSync(process.env.FIXTURE_LOG, JSON.stringify({ run: process.env.FIXTURE_RUN, file: ${JSON.stringify(name)}, at, t: Date.now() }) + '\\n');+test(${JSON.stringify(name)}, async () => {+ mark('start');+ ${body}+ await new Promise(done => setTimeout(done, ${ms}));+ mark('end');+});+`;+}++/** A Git fixture repository with its own machine home, and an environment whose pool has `cores` tokens. */+function fixture(t, cores) {+ const root = mkdtempSync(join(tmpdir(), 'pullboard-file-pool-'));+ t.after(() => rmSync(root, { recursive: true, force: true }));+ runFixtureChild('git', ['init', '-q'], { cwd: root });+ writeFileSync(join(root, 'package.json'), JSON.stringify({ type: 'module' }));+ const machine = join(root, '.machine');+ mkdirSync(machine);+ // Four gate slots, so the four runs overlap and only the file pool can hold them to the cores.+ writeFileSync(join(machine, 'settings.json'), JSON.stringify({ gateSlots: 4, browserSlots: 1 }));+ const env = { ...process.env, PULLBOARD_MACHINE_HOME: machine, [TEST_FILE_CORES_ENV]: String(cores), FIXTURE_LOG: join(root, 'log.jsonl') };+ // This test file holds a token of its own run's pool; the fixture runs get a pool of their own.+ delete env[TEST_FILE_POOL_ENV];+ return { root, env, log: () => readFileSync(env.FIXTURE_LOG, 'utf8').trim().split('\n').map(line => JSON.parse(line)) };+}++/** The most file bodies running at one moment in a start/end log. */+function peak(events) {+ let live = 0;+ let most = 0;+ for (const event of [...events].sort((a, b) => a.t - b.t || (a.at === 'end' ? -1 : 1))) {+ live += event.at === 'start' ? 1 : -1;+ most = Math.max(most, live);+ }+ return most;+}++test('four runs never hold more test files than cores, and the longest-recorded file starts first [C7]', async (t) => {+ const { root, env, log } = fixture(t, 4);+ const files = ['a.test.js', 'b.test.js', 'c.test.js', 'd.test.js', 'e.test.js', 'z-slow.test.js'];+ for (const file of files) writeFileSync(join(root, file), timedFile(file, 300));+ const runs = await Promise.all([1, 2, 3, 4].map(run => runFixtureChildAsync(process.execPath, [runner, ...files],+ { cwd: root, env: { ...env, FIXTURE_RUN: String(run) } })));+ for (const result of runs) assert.equal(result.status, 0, result.failure ?? result.stdout);+ const events = log();+ assert.equal(events.filter(event => event.at === 'start').length, 24, 'every file of every run ran');+ assert.ok(peak(events) <= 4, `at most 4 files at once on 4 cores, saw ${peak(events)}`);++ // The last landing timed z-slow longest, so it starts first though it sorts last.+ mkdirSync(join(root, '.git', 'pullboard'), { recursive: true });+ writeFileSync(join(root, '.git', 'pullboard', 'landing-file-times.json'),+ JSON.stringify({ version: 1, files: { 'z-slow.test.js': 9000, 'c.test.js': 500, 'a.test.js': 10 } }));+ writeFileSync(join(root, 'pullboard.json'), JSON.stringify({ testConcurrency: { files: 1 } }));+ writeFileSync(env.FIXTURE_LOG, '');+ const ordered = runFixtureChild(process.execPath, [runner, ...files], { cwd: root, env: { ...env, FIXTURE_RUN: 'order', PULLBOARD_LANDING_RUN: '1' }, encoding: 'utf8' });+ assert.equal(ordered.status, 0, ordered.failure ?? ordered.stdout);+ assert.deepEqual(log().filter(event => event.at === 'start').map(event => event.file),+ ['z-slow.test.js', 'c.test.js', 'a.test.js', 'b.test.js', 'd.test.js', 'e.test.js'], 'timed files longest first, then the rest in order');+});++test('a file that waits on a nested run lends it its token, so saturated parents still run at most cores files [C7]', async (t) => {+ const { root, env, log } = fixture(t, 2);+ for (const file of ['inner-a.test.js', 'inner-b.test.js']) writeFileSync(join(root, file), timedFile(file, 300));+ // Each outer file runs both inner files through run-tests and waits; its own body does no work meanwhile.+ const nested = `const { spawnSync } = await import('node:child_process');+ const inner = spawnSync(process.execPath, [${JSON.stringify(runner)}, 'inner-a.test.js', 'inner-b.test.js'], { cwd: ${JSON.stringify(root)}, env: { ...process.env, FIXTURE_RUN: 'inner-' + process.pid }, encoding: 'utf8' });+ if (inner.status !== 0) throw new Error(inner.stdout + inner.stderr);`;+ for (const file of ['outer-a.test.js', 'outer-b.test.js']) {+ writeFileSync(join(root, file), timedFile(file, 0, nested).replace("mark('start');", '').replace("mark('end');", ''));+ }+ const result = await runFixtureChildAsync(process.execPath, [runner, 'outer-a.test.js', 'outer-b.test.js'], { cwd: root, env: { ...env, FIXTURE_RUN: 'outer' } });+ assert.equal(result.status, 0, result.failure ?? result.stdout);+ const events = log();+ assert.equal(events.filter(event => event.at === 'start').length, 4, 'both inner files ran under both outer files');+ assert.ok(peak(events) <= 2, `two saturated outer files with nested runs still run at most 2 files, saw ${peak(events)}`);+ for (const run of new Set(events.map(event => event.run))) {+ assert.equal(peak(events.filter(event => event.run === run)), 1, 'a nested run takes only the token its parent lent');+ }+});