Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(gate): a nested run lends its parent's place, never one token [C7]
Corey Olson0f549067fa9d
diff --git a/bin/run-tests.js b/bin/run-tests.jsindex c1d259a6..4ddf1aac 100644--- a/bin/run-tests.js+++ b/bin/run-tests.js@@ -16,7 +16,7 @@ import { channel } from 'node:diagnostics_channel';import { AGENT_SHELL_MARKERS, SSH_SHELL_MARKERS } from '../src/person.js';import { isLandingWorkerIdentityTimeout, landingWorkerIdentityQuery, parseLandingWorkerIdentity } from '../src/landing-worker-identity.js';import { foreignNodeOptions, isolateTestMachine } from '../src/test-isolation.js';-import { TEST_FILE_POOL_ENV, longestFirst, takeTestFileToken, testFilePoolValue } from '../src/test-file-pool.js';+import { TEST_FILE_POOL_ENV, lendParentToken, longestFirst, takeTestFileToken, testFilePoolValue } from '../src/test-file-pool.js';const runner = fileURLToPath(import.meta.url);const LANDING_RUN_ENV = 'PULLBOARD_LANDING_RUN';const LANDING_WORKER_STARTS_ENV = 'PULLBOARD_LANDING_WORKER_STARTS';@@ -1466,6 +1466,8 @@ async function main(args) {process.exitCode = 1;return;}+ // A run started by a test file stands in for that file while it lives, so the file waits without a core (#1046).+ await lendParentToken();const sharding = shardArguments(args);if (sharding.error) {process.stderr.write(`run-tests: ${sharding.error}\n`);diff --git a/src/test-file-pool.js b/src/test-file-pool.jsindex 805cd86e..865e782e 100644--- a/src/test-file-pool.js+++ b/src/test-file-pool.js@@ -5,26 +5,28 @@* Measured 22 Oct: 28 runs at --test-concurrency=10 kept 54 file processes live on 14 cores, and passing files* ran 18.5 times slower in landings than alone. A per-run limit cannot see the other runs; this pool can.*- * A test file that starts its own run-tests and waits on it lends its token to that nested run: the nested- * files take the lent token one at a time, never a second token from the pool, so N saturated outer files- * with nested runs still run at most N files at once, and a nested run can never wait on its own parent.+ * A test file that starts its own run-tests and waits on it lends its token to that nested run: while the nested+ * run lives, its parent counts as waiting, not running, so the nested files take the parent's place and never a+ * second core. Running files are holders minus lending parents, and that stays at or under the cores. Two nested+ * runs of one parent share its one place; neither can wait forever on the token its own parent holds.*/+import { mkdirSync } from 'node:fs';import { availableParallelism } from 'node:os';-import { randomUUID } from 'node:crypto';+import { join } from 'node:path';import { machineHome } from './machine-home.js';-/** Names the pool a test-file process takes from, and the token its parent lent, as JSON. */+/** Names the pool a test-file process takes from, and the file process that started this run, as JSON. */export const TEST_FILE_POOL_ENV = 'PULLBOARD_TEST_FILE_POOL';/** Overrides the pool size; a test models a smaller machine with it. */export const TEST_FILE_CORES_ENV = 'PULLBOARD_TEST_FILE_CORES';-/** The resource row every top-level test file takes from. */-const POOL_NAME = 'test-files';+// Budget: a waiting file looks again this often; admission is one short transaction.+const POLL_MS = 20;/*** Read the pool a test-file process belongs to, or null outside a run-tests run.** @param {NodeJS.ProcessEnv} env- * @returns {{ home: string, cores: number, loan?: string } | null}+ * @returns {{ home: string, cores: number, lender?: number } | null}*/export function testFilePool(env = process.env) {const value = env[TEST_FILE_POOL_ENV];@@ -32,51 +34,138 @@ export function testFilePool(env = process.env) {try {const pool = JSON.parse(value);if (typeof pool?.home !== 'string' || !Number.isSafeInteger(pool.cores) || pool.cores < 1) return null;- if (pool.loan !== undefined && typeof pool.loan !== 'string') return null;+ if (pool.lender !== undefined && !Number.isSafeInteger(pool.lender)) return null;return pool;} catch { return null; }}/**- * The pool value a run passes to its test files: an inherited one (a nested run keeps its parent's pool and- * loan), or this machine's pool sized to its cores. One variable carries both, so no copy of the environment- * can keep the pool and drop the loan, which would let a nested file wait on a token its parent holds.+ * The pool value a run passes to its test files: an inherited one, so a nested run shares its parent's pool, or+ * this machine's pool sized to its cores.** @param {NodeJS.ProcessEnv} env* @returns {string}*/export function testFilePoolValue(env = process.env) {- if (testFilePool(env)) return env[TEST_FILE_POOL_ENV];+ const pool = testFilePool(env);+ if (pool) return JSON.stringify({ home: pool.home, cores: pool.cores });const override = Number(env[TEST_FILE_CORES_ENV]);const cores = Number.isSafeInteger(override) && override > 0 ? override : Math.max(1, availableParallelism());return JSON.stringify({ home: machineHome(), cores });}/**- * Take this test-file process's token and hold it until the process exits. A file with a lent token takes- * that one; any other takes one from the machine pool. The file then lends its own token to anything it starts.+ * Load SQLite without its experimental warning: the runner's own output must stay exactly what its tests pin,+ * and a run that never nests never loads it at all.+ */+async function databaseSync() {+ const emitWarning = process.emitWarning;+ process.emitWarning = (warning, ...rest) => {+ if (!String(warning).includes('SQLite is an experimental feature')) emitWarning.call(process, warning, ...rest);+ };+ try { return (await import('node:sqlite')).DatabaseSync; }+ finally { process.emitWarning = emitWarning; }+}++/** Open the pool's database, creating its tables on first use. */+async function openPool(home) {+ mkdirSync(home, { recursive: true });+ const DatabaseSync = await databaseSync();+ const db = new DatabaseSync(join(home, 'test-file-pool.sqlite'));+ db.exec(`PRAGMA busy_timeout = 10000;+ CREATE TABLE IF NOT EXISTS file_holder(pid INTEGER PRIMARY KEY, since INTEGER NOT NULL);+ CREATE TABLE IF NOT EXISTS file_waiter(ticket INTEGER PRIMARY KEY AUTOINCREMENT, pid INTEGER NOT NULL UNIQUE);+ CREATE TABLE IF NOT EXISTS file_loan(runner INTEGER PRIMARY KEY, lender INTEGER NOT NULL);`);+ return db;+}++/** Whether a process is alive; a process another user owns still is. */+function alive(pid) {+ try { process.kill(pid, 0); return true; }+ catch (error) { return error.code === 'EPERM'; }+}++/** Run one write transaction that no other process interleaves. */+function immediate(db, work) {+ db.exec('BEGIN IMMEDIATE');+ try {+ const result = work();+ db.exec('COMMIT');+ return result;+ } catch (error) {+ db.exec('ROLLBACK');+ throw error;+ }+}++/** Drop rows whose process has gone, so a crashed file or run never holds a place. */+function prune(db) {+ for (const { pid } of db.prepare('SELECT pid FROM file_holder').all()) {+ if (!alive(pid)) db.prepare('DELETE FROM file_holder WHERE pid = ?').run(pid);+ }+ for (const { pid } of db.prepare('SELECT pid FROM file_waiter').all()) {+ if (!alive(pid)) db.prepare('DELETE FROM file_waiter WHERE pid = ?').run(pid);+ }+ for (const { runner } of db.prepare('SELECT runner FROM file_loan').all()) {+ if (!alive(runner)) db.prepare('DELETE FROM file_loan WHERE runner = ?').run(runner);+ }+}++/**+ * Admit this process when a core is free and every earlier waiter fits first. Running files are holders less the+ * holders lending to a live nested run.+ */+function admit(db, cores) {+ return immediate(db, () => {+ prune(db);+ const holders = db.prepare('SELECT count(*) AS n FROM file_holder').get().n;+ const lending = db.prepare('SELECT count(DISTINCT lender) AS n FROM file_loan WHERE lender IN (SELECT pid FROM file_holder)').get().n;+ const free = cores - (holders - lending);+ const mine = db.prepare('SELECT ticket FROM file_waiter WHERE pid = ?').get(process.pid)?.ticket;+ const ahead = db.prepare('SELECT count(*) AS n FROM file_waiter WHERE ticket < ?').get(mine).n;+ if (ahead >= free) return false;+ db.prepare('DELETE FROM file_waiter WHERE pid = ?').run(process.pid);+ db.prepare('INSERT OR REPLACE INTO file_holder(pid, since) VALUES (?, ?)').run(process.pid, Date.now());+ return true;+ });+}++/**+ * Take this test-file process's token and hold it until the process exits; then name this process as the lender+ * of anything it starts.*- * @param {NodeJS.ProcessEnv} env - Changed in place: descendants see this file's loan.+ * @param {NodeJS.ProcessEnv} env - Changed in place: descendants see this file as their lender.* @returns {Promise<void>}*/export async function takeTestFileToken(env = process.env) {const pool = testFilePool(env);if (!pool) return;- const { takeResource } = await import('./resources.js');- // The test file runs with a private machine home; the pool lives in the real one, named by the run.- const ownHome = env.PULLBOARD_MACHINE_HOME;- env.PULLBOARD_MACHINE_HOME = pool.home;- let lease;- try {- lease = await takeResource(pool.loan- ? { name: pool.loan, capacity: 1, kind: 'test-file' }- : { name: POOL_NAME, capacity: pool.cores, kind: 'test-file' });- } finally {- if (ownHome === undefined) delete env.PULLBOARD_MACHINE_HOME;- else env.PULLBOARD_MACHINE_HOME = ownHome;- }- process.once('exit', () => lease.release());- env[TEST_FILE_POOL_ENV] = JSON.stringify({ home: pool.home, cores: pool.cores, loan: `test-file-loan:${randomUUID()}` });+ const db = await openPool(pool.home);+ immediate(db, () => db.prepare('INSERT OR IGNORE INTO file_waiter(pid) VALUES (?)').run(process.pid));+ while (!admit(db, pool.cores)) await new Promise(done => setTimeout(done, POLL_MS));+ process.once('exit', () => {+ try { db.prepare('DELETE FROM file_holder WHERE pid = ?').run(process.pid); db.close(); }+ catch { /* A row left behind is pruned once this process is gone. */ }+ });+ env[TEST_FILE_POOL_ENV] = JSON.stringify({ home: pool.home, cores: pool.cores, lender: process.pid });+}++/**+ * Mark this run as nested in a test file, so that file counts as waiting while the run lives. A run started+ * outside a test file lends nothing.+ *+ * @param {NodeJS.ProcessEnv} env+ * @returns {Promise<void>}+ */+export async function lendParentToken(env = process.env) {+ const pool = testFilePool(env);+ if (!pool?.lender) return;+ const db = await openPool(pool.home);+ immediate(db, () => db.prepare('INSERT OR REPLACE INTO file_loan(runner, lender) VALUES (?, ?)').run(process.pid, pool.lender));+ process.once('exit', () => {+ try { db.prepare('DELETE FROM file_loan WHERE runner = ?').run(process.pid); db.close(); }+ catch { /* A row left behind is pruned once this run is gone. */ }+ });}/**diff --git a/test/gate-admission.test.js b/test/gate-admission.test.jsindex 5fa3a251..3e991261 100644--- a/test/gate-admission.test.js+++ b/test/gate-admission.test.js@@ -74,7 +74,7 @@ test('four runs never hold more test files than cores, and the longest-recorded['z-slow.test.js', 'c.test.js', 'a.test.js', 'b.test.js', 'd.test.js', 'e.test.js'], 'timed files longest first, then the rest in order');});-test('a file that waits on a nested run lends it its token, so saturated parents still run at most cores files [C7]', async (t) => {+test('a file waiting on a nested run lends its place, so saturated parents still run at most cores files [C7]', async (t) => {const { root, env, log } = fixture(t, 2);for (const file of ['inner-a.test.js', 'inner-b.test.js']) writeFileSync(join(root, file), timedFile(file, 300));// Each outer file runs both inner files through run-tests and waits; its own body does no work meanwhile.@@ -89,7 +89,4 @@ test('a file that waits on a nested run lends it its token, so saturated parentsconst events = log();assert.equal(events.filter(event => event.at === 'start').length, 4, 'both inner files ran under both outer files');assert.ok(peak(events) <= 2, `two saturated outer files with nested runs still run at most 2 files, saw ${peak(events)}`);- for (const run of new Set(events.map(event => event.run))) {- assert.equal(peak(events.filter(event => event.run === run)), 1, 'a nested run takes only the token its parent lent');- }});diff --git a/test/test-runner.test.js b/test/test-runner.test.jsindex 4842ec21..66e904fe 100644--- a/test/test-runner.test.js+++ b/test/test-runner.test.js@@ -104,7 +104,7 @@ test('default discovery runs one fixture and does not recurse into the runner [CcopyFileSync(runner, join(bin, 'run-tests.js'));const source = join(root, 'src');mkdirSync(source);- for (const name of ['child-journal-preload.js', 'config.js', 'person.js', 'refused.js', 'landing-worker-identity.js', 'test-action-cache.js', 'test-isolation.js']) {+ for (const name of ['child-journal-preload.js', 'config.js', 'person.js', 'refused.js', 'landing-worker-identity.js', 'test-action-cache.js', 'test-isolation.js', 'test-file-pool.js', 'machine-home.js']) {copyFileSync(fileURLToPath(new URL('../src/' + name, import.meta.url)), join(source, name));}const marker = join(root, 'ran-once');