Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
feat(test): pullboard test runs named files on the runner pool [C7]
Corey Olson87309e6b965e
diff --git a/src/commands/test.js b/src/commands/test.jsnew file mode 100644index 00000000..ac17e187--- /dev/null+++ b/src/commands/test.js@@ -0,0 +1,23 @@+/** The pullboard test command, in its own module so an item adding another command never edits these lines. */+import { runTestFiles } from '../test-run.js';+import {+ context,+ gateWaitReporter,+ resourceAgent,+} from '../cli.js';++/** Run `pullboard test <files>` and return its exit code: 0 when every named file passes. */+export async function run(io, { first, second, rest }) {+ const ctx = context(io);+ const files = [first, second, ...rest].filter((file) => file !== undefined && file !== '');+ const result = await runTestFiles(ctx.info.root, ctx.config, files, { agent: resourceAgent(ctx) ?? `pid-${process.pid}`, onWait: gateWaitReporter(io) });+ io.result?.(result);+ if (!io.result && result.output) io.say(result.output.trimEnd());+ io.err(`pullboard test: ${result.green ? 'green' : 'red'} on ${result.runner} in ${result.seconds}s`);+ for (const entry of result.known) {+ io.err(entry.result === 'red'+ ? `known main failure: ${entry.file} fails on main ${String(result.main).slice(0, 12)} too`+ : `new failure: ${entry.file} passes on main ${String(result.main).slice(0, 12)}`);+ }+ return result.green ? 0 : 1;+}diff --git a/src/main-red.js b/src/main-red.jsindex 5d3f7ebd..0ee8b1a5 100644--- a/src/main-red.js+++ b/src/main-red.js@@ -132,8 +132,8 @@ function gitCall(cwd, args, env) {});}-/** Build one clean temporary checkout at the exact main commit and always remove it. */-async function withMainCheckout(root, commit, run) {+/** Build one clean temporary checkout at an exact commit (main's, or a pullboard test snapshot) and always remove it. */+export async function withMainCheckout(root, commit, run) {const directory = join(tmpdir(), `pullboard-main-red-${randomUUID()}`);mkdirSync(directory, { mode: 0o700 });const checkout = join(directory, 'checkout');diff --git a/src/resources.js b/src/resources.jsindex cf10982e..5e7b15d0 100644--- a/src/resources.js+++ b/src/resources.js@@ -475,7 +475,12 @@ function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirsbrowsers: remoteHolders.filter(holder => holder.browser === 1).length,}];}));- const unavailable = new Set(db.prepare('SELECT host FROM runner_unavailable UNION SELECT host FROM runner_draining').all().map(row => row.host));+ // A runner whose transport dropped rejoins the pool after RUNNER_RETRY_MS, so one network blip cannot send every+ // later run to this machine; another drop quarantines it again. Unverified cleanup stays out until re-registered.+ const unavailable = new Set([+ ...db.prepare("SELECT host, since, detail FROM runner_unavailable").all()+ .filter(row => row.detail !== 'TRANSPORT_LOST' || !(now - Date.parse(row.since) >= RUNNER_RETRY_MS)).map(row => row.host),+ ...db.prepare('SELECT host FROM runner_draining').all().map(row => row.host)]);/** Keep an already holding process from taking another slot ahead of a different landing. */const blockedByLanding = waiter => holders.some(holder => holder.pid === waiter.pid)&& waiters.some(other => other.landing === 1 && other.pid !== waiter.pid);@@ -619,6 +624,8 @@ export function updateRunnerSettings(host, persist, { drain = false } = {}) {}const MEMORY_CACHE_MS = 2_000;+/** How long a runner whose transport dropped sits out before the pool tries it again. */+const RUNNER_RETRY_MS = 5 * 60_000;const MEMORY_COMMAND_TIMEOUT_MS = 2_000;const LINUX_CRITICAL_FULL_AVG10 = 10;const LINUX_WARN_SOME_AVG10 = 10;diff --git a/src/test-run.js b/src/test-run.jsnew file mode 100644index 00000000..aefce9ee--- /dev/null+++ b/src/test-run.js@@ -0,0 +1,111 @@+/** pullboard test: named test files on the runner pool, with the focused runner's per-file timeout [C7]. */+import { spawnSync } from 'node:child_process';+import {+ existsSync,+ mkdtempSync,+ rmSync,+} from 'node:fs';+import { tmpdir } from 'node:os';+import {+ isAbsolute,+ join,+ relative,+ resolve,+ sep,+} from 'node:path';+import { testsNeedBrowser } from './browser-admission.js';+import {+ runProfiledShellAsync,+ withGateSlot,+} from './gate.js';+import {+ cleanGitEnvironment,+ headCommit,+} from './git.js';+import { testsNeedGui } from './gui-admission.js';+import {+ compareMainFailures,+ withMainCheckout,+} from './main-red.js';+import { Refused } from './refused.js';+import { runnerSource } from './runners.js';+import { browserFreeChecksEnabled } from './settings.js';++/** Quote one argument for the configured focused test command. */+function shellWord(value) {+ return `'${String(value).replaceAll("'", "'\\''")}'`;+}++/** Keep each named file a test file inside this checkout, as the runner's checkout names it. */+export function testFiles(root, files) {+ if (!files.length) throw new Refused('USAGE', 'name the test files to run: pullboard test test/one.test.js [more files]');+ return files.map((file) => {+ const path = relative(root, resolve(root, file));+ if (!path || isAbsolute(path) || path === '..' || path.startsWith('..' + sep) || !existsSync(join(root, path))) {+ throw new Refused('NO_TEST_FILE', `${file} is not a file in this checkout; name test files by their path from the checkout root`);+ }+ return path.split(sep).join('/');+ });+}++/** Run Git in this checkout with inherited selectors cleared and a private index. */+function git(root, args, env) {+ const run = spawnSync('git', args, { cwd: root, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });+ if (run.status !== 0) throw new Refused('TEST_SNAPSHOT', `could not snapshot this checkout for the runners (git ${args[0]}): ${run.stderr.trim().split('\n')[0]}`);+ return run.stdout.trim();+}++/**+ * The commit a runner checks out: HEAD when the tree is clean, else a snapshot commit of the working tree, made+ * with a private index so the checkout's own index and branches stay as they are. Ignored files stay out.+ */+function snapshotCommit(root) {+ if (runnerSource(root)) return { commit: headCommit(root), clean: true };+ const folder = mkdtempSync(join(tmpdir(), 'pullboard-test-index-'));+ try {+ const env = { ...cleanGitEnvironment(), GIT_INDEX_FILE: join(folder, 'index') };+ const head = headCommit(root);+ if (head) git(root, ['read-tree', head], env);+ git(root, ['add', '-A', '--', '.'], env);+ const tree = git(root, ['write-tree'], env);+ const commit = git(root, ['-c', 'user.name=pullboard', '-c', '[email protected]', 'commit-tree', tree, ...(head ? ['-p', head] : []), '-m', 'pullboard test snapshot'], env);+ return { commit, clean: false };+ } finally { rmSync(folder, { recursive: true, force: true }); }+}++/** Run the files where the work is: this clean checkout, or a clean temporary checkout of its snapshot. */+async function inCleanCheckout(root, work) {+ const { commit, clean } = snapshotCommit(root);+ if (clean) return work(root);+ const result = await withMainCheckout(root, commit, work);+ if (result?.unavailable) throw new Refused('TEST_SNAPSHOT', `${result.unavailable}; commit the work and run pullboard test again`);+ return result;+}++/**+ * Run named test files through a gate slot, which deals them to the registered runners first (#1054); the focused+ * runner (pullboard.json affectedTests) gives each file its timeout. A red file is compared with main's recorded+ * result for that file, so a failure main already has reads as known.+ *+ * @returns {Promise<{ green: boolean, output: string, runner: string, seconds: number, files: string[], known: Array<{file: string, result: string}>, main: string|null }>}+ */+export async function runTestFiles(root, config, files, { agent, onWait } = {}) {+ if (typeof config.affectedTests !== 'string' || !config.affectedTests.trim()) {+ throw new Refused('NO_TEST_RUNNER', 'set "affectedTests" in pullboard.json to the command that runs named test files, like "node bin/run-tests.js"');+ }+ const paths = testFiles(root, files);+ const command = `${config.affectedTests.trimEnd()} ${paths.map((path) => shellWord('./' + path)).join(' ')}`;+ const started = Date.now();+ const run = await inCleanCheckout(root, async (checkout) => {+ const browser = !browserFreeChecksEnabled() || await testsNeedBrowser(checkout, paths);+ return withGateSlot(checkout, (lease) => runProfiledShellAsync(checkout, command, { gateLease: lease, waitMs: lease.waitMs }),+ { browser, gui: testsNeedGui(checkout, paths), shortEligible: true, agent, kind: 'test', onWait });+ });+ const result = { green: run.isGreen, output: run.output, runner: run.executionRunner || 'this machine', seconds: Math.round((Date.now() - started) / 1000), files: paths, known: [], main: null };+ if (!run.isGreen) {+ const compared = await compareMainFailures(root, run.profile, onWait, run.output);+ result.main = compared.main;+ result.known = compared.files;+ }+ return result;+}diff --git a/test/test-command.test.js b/test/test-command.test.jsnew file mode 100644index 00000000..104ab3be--- /dev/null+++ b/test/test-command.test.js@@ -0,0 +1,73 @@+/** pullboard test runs named test files on the runner pool, and names a failure main already has [C7]. */+import assert from 'node:assert/strict';+import {+ mkdirSync,+ readFileSync,+ writeFileSync,+} from 'node:fs';+import { join } from 'node:path';+import {+ after,+ test,+} from 'node:test';+import { createE2eHelpers } from './e2e-helpers.js';+import {+ HOST,+ installSshStandIn,+} from './runner-ssh-fixture.js';++const e2e = createE2eHelpers();+after(e2e.cleanup);++const PASSES = "import { test } from 'node:test';\ntest('passes', () => {});\n";+const HANGS = "import { test } from 'node:test';\ntest('hangs', () => new Promise(() => {}));\n";+const FAILS = "import assert from 'node:assert/strict';\nimport { test } from 'node:test';\ntest('fails here', () => assert.equal(1, 2));\n";++test('pullboard test runs files on the runner pool with a per-file timeout [C7]', { timeout: 240_000 }, async () => { // budget: hang ceiling for the stand-in runner's three worker runs; not a speed target.+ const box = e2e.project('true');+ // One stand-in runner, preferred, and no local slot at all: a run that stayed on this machine could not start.+ const bin = join(box.dir, 'runner-bin');+ const worker = join(box.dir, 'runner-worker');+ const home = join(box.dir, 'runner-home');+ const requests = join(box.dir, 'runner-requests.jsonl');+ for (const folder of [bin, worker, home, join(home, '.pullboard')]) mkdirSync(folder, { recursive: true });+ box.env.PATH = `${bin}:${box.env.PATH}`;+ installSshStandIn({ bin, worker, home, machineHome: join(home, '.pullboard'), requests });+ assert.equal(box.run(box.repo, 'runner', 'add', HOST, '--slots', '2', '--browser', '1').code, 0);+ for (const [key, value] of [['gate.runnersFirst', 'on'], ['browserSlots', '0'], ['gateSlots', '0']]) {+ const set = box.run(box.repo, 'settings', key, value);+ assert.equal(set.code, 0, `${key} ${value}: ${set.err}`);+ }+ // Main holds the focused runner, with its per-test timeout, and a test that hangs on main as well.+ const config = JSON.parse(readFileSync(join(box.repo, 'pullboard.json'), 'utf8'));+ writeFileSync(join(box.repo, 'pullboard.json'), JSON.stringify({ ...config, affectedTests: 'node --test --test-timeout=3000' }, null, 2));+ mkdirSync(join(box.repo, 'test'));+ for (const [name, text] of [['ok', PASSES], ['hang', HANGS], ['changed', PASSES]]) writeFileSync(join(box.repo, 'test', `${name}.test.js`), text);+ box.git(box.repo, 'add', '-A');+ box.git(box.repo, 'commit', '-q', '-m', 'test: focused runner and three test files');+ box.git(box.web, 'merge', '-q', '--ff-only', 'main');+ // The agent's change is not committed: the runner still runs exactly what is in its worktree.+ writeFileSync(join(box.web, 'test', 'changed.test.js'), FAILS);++ const started = Date.now();+ const ran = box.run(box.web, 'test', 'test/ok.test.js', 'test/hang.test.js', 'test/changed.test.js', '--json');+ const seconds = (Date.now() - started) / 1000;+ assert.equal(ran.code, 1, `a red file makes the run red: ${ran.err}`);+ const result = JSON.parse(ran.out);+ assert.equal(result.runner, HOST, 'the files ran on the registered runner');+ assert.match(result.output, /^ok \d+ - .*passes/mu, 'the TAP of a passing file comes back as the runner printed it');+ assert.match(result.output, /fails here/u, 'the uncommitted change ran on the runner');+ assert.match(result.output, /timed? ?out|cancelled/iu, 'the hanging test stops at its timeout');+ assert.ok(seconds < 200, `the hung file never holds the run past its timeout (${seconds}s)`);+ const known = Object.fromEntries(result.known.map((entry) => [entry.file, entry.result]));+ assert.equal(known['test/hang.test.js'], 'red', 'the hang fails on main too, so it reads as a known main failure');+ assert.equal(known['test/changed.test.js'], 'green', 'the changed file passes on main, so its failure is new');+ assert.equal(known['test/ok.test.js'], undefined, 'a passing file needs no main comparison');+ const calls = readFileSync(requests, 'utf8').trim().split('\n').map((line) => JSON.parse(line));+ assert.ok(calls.length >= 2 && calls.every((call) => call.host === HOST), 'the run and its main comparisons all went to the runner');++ const text = box.run(box.web, 'test', 'test/hang.test.js');+ assert.equal(text.code, 1);+ assert.match(text.err, new RegExp(`red on ${HOST}`, 'u'), 'the text form names where it ran');+ assert.match(text.err, /known main failure: test\/hang\.test\.js fails on main/u, 'and names the failure main already has');+});