Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
feat(runner): remote reds get flake pairs with host load and main [C7]
Corey Olson9515552bb304
diff --git a/src/gate.js b/src/gate.jsindex 4fa2451e..bd93dace 100644--- a/src/gate.js+++ b/src/gate.js@@ -531,6 +531,11 @@ export async function withGateSlot(root, action, { landing = false, itemCheck =}}+/** Carry a failure observer's pinned main on the lease, so a remote runner's receipt names the main this run used (#925). */+export function pinRunMain(lease, observer) {+ if (typeof observer?.main === 'string') lease.runMetadata = { ...(lease.runMetadata ?? {}), main: observer.main };+}+/*** Run the configured gate in the repo, unless this exact tree already passed. Its output, both* streams in order, is kept whole in the git dir and returned for a digest.@@ -550,6 +555,7 @@ export async function runGate(root, config, { trustStamp = true, onWait, landing}if (trustStamp && isStampedGreen(root)) return { isGreen: true, isCached: true, output: '', seconds: 0, log: '' };return await withGateSlot(root, async (lease) => {+ pinRunMain(lease, onFailure);const run = fullGateRun(root, config, trustStamp, { waitMs: lease.waitMs, gateLease: lease, landing, stages, waivers });if (!run.isGreen && onFailure) await onFailure(run, { root, command: config.gate, lease });return run;@@ -569,7 +575,9 @@ function fullGateRun(root, config, trustStamp = false, { waitMs = 0, log = gitPaif (!run.sandboxBlocked) recordGateRun(root, gateRunRecord(root, config, isGreen, gateLease));refuseSandboxBlocked(run);if (isGreen && before !== null && committedTree(root) === before) writeFileSync(gitPath(root, STAMP), `${before}\n`);- return { isGreen, isCached: false, output, seconds, log, profile, profilePath, ...(run.executionRoot ? { executionRoot: run.executionRoot } : {}), ...(stages ? { stages: run.stages } : {}) };+ return { isGreen, isCached: false, output, seconds, log, profile, profilePath, ...(run.executionRoot ? { executionRoot: run.executionRoot } : {}),+ // A remote run's host load and pinned main reach the flake diagnostics with it (#925).+ ...(run.executionRunner ? { executionRunner: run.executionRunner, executionMain: run.executionMain ?? null, executionLoad: run.executionLoad ?? null } : {}), ...(stages ? { stages: run.stages } : {}) };}/**@@ -646,6 +654,7 @@ export async function runSubmitGate(root, config, { base, trunk, changed, check,: (selection.full || !allowBrowserFreeChecks || await testsNeedBrowser(root, proofFiles, { onFallback: onBrowserFallback }));const browser = checkUsesBrowser || proofUsesBrowser;return await withGateSlot(root, async (lease) => {+ pinRunMain(lease, onFailure);const startedAt = Date.now();const log = gitPath(root, 'pullboard-submit.log');const checkLog = `${log}.check.log`;diff --git a/src/land.js b/src/land.jsindex c9ae4551..12aebd3f 100644--- a/src/land.js+++ b/src/land.js@@ -254,12 +254,12 @@ async function gateGroup(state, label) {await save(state, { state: 'gating', tip: headCommit(state.batch.root), logPath });const pin = flakeSource(state.batch.root, { commit, main: state.batch.base });const gate = await runGate(state.batch.root, state.config, { trustStamp: false, landing: true, agent: 'coordinator', onWait: state.onWait, stages: true, waivers: liveWaiverProjection(state.board),- onFailure: (run, { root, command, lease }) => observeNativeFailure({ root, config: state.config, pin, run, command, lease, kind: 'land',+ onFailure: Object.assign((run, { root, command, lease }) => observeNativeFailure({ root, config: state.config, pin, run, command, lease, kind: 'land',record: observation => {const owner = observation.file ? laneOf(state.config, observation.file) : 'coordinator';const lane = owner === 'coordinator' ? store.getItem(state.board, state.batch.items[0].id).item_lane : owner;return state.write('recordFlakeObservation', [{ agentId: 'coordinator', observation, lane }]);- } }),+ } }), { main: pin.main }),});copyFileSync(gate.log, logPath);if (gate.profilePath) copyFileSync(gate.profilePath, `${logPath}.profile.json`);diff --git a/src/native-flake-cli.js b/src/native-flake-cli.jsindex cff1366e..f2ad2d46 100644--- a/src/native-flake-cli.js+++ b/src/native-flake-cli.js@@ -7,6 +7,7 @@ import {mainPolicy,policyAt,} from './trusted-policy.js';+import { pinRunMain } from './gate.js';import { runProfiledShell } from './gate.js';import {diagnosticUnavailable,@@ -27,7 +28,8 @@ export function observer(ctx, { kind, board, main, commit, privateCheck = false,// Diagnostics cannot block an existing check when its coordinator is detached or its observer cannot initialize.return async run => { if (!run.isGreen) diagnosticUnavailable({ run }, error); };}- return async (run, { root = ctx.info.root, command, lease }) => {+ // The pinned main rides on the observer, so a remote runner's receipt can name the main this run used (#925).+ return Object.assign(async (run, { root = ctx.info.root, command, lease }) => {await observeNativeFailure({ root, config: acceptedConfig, pin, run, command, lease, kind,record: observation => {const ownerLane = observation.file ? laneOf(acceptedConfig, observation.file) : COORDINATOR;@@ -35,13 +37,14 @@ export function observer(ctx, { kind, board, main, commit, privateCheck = false,const write = target => ordered(ctx, target, 'recordFlakeObservation', [{ agentId, observation, lane }]);return board ? write(board) : withBoard(ctx, write);} });- };+ }, { main: pin.main });}/** Prepare an ordinary check callback before it queues; the callback observes red under that same lease. */export function check(ctx, item, command, services) {const observe = observer(ctx, { kind: 'check' }, services);return async lease => {+ pinRunMain(lease, observe);const run = runProfiledShell(ctx.info.root, command, { waitMs: lease.waitMs, artifactPrefix: `pullboard-check-${item.item_id}`, persistLog: true, gateLease: lease });if (!run.isGreen) await observe(run, { command, lease });return run;@@ -52,6 +55,7 @@ export function check(ctx, item, command, services) {export function verify(ctx, item, command, options, services) {const observe = observer(ctx, { ...options, kind: 'verify', privateCheck: true }, services);return async lease => {+ pinRunMain(lease, observe);const result = checkAtCommit(ctx.info.root, item, { waitMs: lease.waitMs, gateLease: lease });if (result.state === 'red') await observe({ isGreen: false, output: result.outputPath ? readFileSync(result.outputPath, 'utf8') : result.output,log: result.outputPath, profilePath: result.profilePath, executionRoot: result.executionRoot, executionSourceUnchanged: result.executionSourceUnchanged }, { root: ctx.info.root, command, lease });diff --git a/src/native-flakes.js b/src/native-flakes.jsindex 35f3e985..cf142d40 100644--- a/src/native-flakes.js+++ b/src/native-flakes.js@@ -170,6 +170,17 @@ function privateProbe(root, commit, command, lease, policyCommit) {executionRoot: check.executionRoot, executionSourceUnchanged: check.executionSourceUnchanged, profilePath: check.profilePath, profile: check.profile };}+/**+ * Why a remote admission cannot be judged, or null when its receipt names both facts a local run measures: the+ * execution host's load, and the pinned main this run used (#925). A missing fact is never guessed or replaced by a+ * local measurement.+ */+function remoteGap(run, pin) {+ if (!run.executionLoad) return 'The remote receipt does not carry the execution host\'s load; the red remains unclassified, never guessed.';+ if (!pin.main || run.executionMain !== pin.main) return 'The remote receipt does not name the pinned main this run used; the red remains unclassified, never guessed.';+ return null;+}+/** Report only a bounded code when diagnostic work fails; never replace the original gate's red result. */export async function observeNativeFailure(options) {const started = Date.now();@@ -218,7 +229,7 @@ async function observeFailure({ root, config, pin, run, command, lease, kind, reinvalidateGitFacts();const changed = run.executionSourceUnchanged === false || !pin.privateCheck && (headCommit(root) !== pin.head || headTree(root) !== pin.tree || !isClean(root) || untracked(root).length > 0);if (!pin.clean || changed) reason = 'The source is dirty or changed; no unchanged-head diagnostic is claimed.';- else if (lease?.runner) reason = 'This remote admission has no supported pinned-main comparison; no local substitute is run.';+ else if (lease?.runner && remoteGap(run, pin)) reason = remoteGap(run, pin);else if (config.flakes?.rerun === true && prefix && named && failure.indent === 0) {const pattern = '^' + failure.name.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&') + '$';const selected = prefix + ' --test-name-pattern ' + word(pattern) + ' ' + word(failure.file);@@ -243,7 +254,8 @@ async function observeFailure({ root, config, pin, run, command, lease, kind, re}const observation = { id, file: named ? failure.file : null, name: named ? failure.name : null, head: pin.head, tree: pin.tree, main: pin.main,kind, classification, reason, commandSha256: digest(command), at: observedAt, diagnosticMs: Date.now() - targetStarted,- load, original, paired, mainReceipt };+ // A remote run records its own host's load; without one, the observer's sample stands and the reason says so.+ load: lease?.runner && run.executionLoad ? run.executionLoad : load, original, paired, mainReceipt };writeFileSync(join(directory, 'observation.json'), JSON.stringify(observation, null, 2) + '\n', { flag: 'wx', mode: 0o600 });const manifest = Object.fromEntries(readdirSync(directory).map(file => [file, fileDigest(join(directory, file))]));writeFileSync(join(directory, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n', { flag: 'wx', mode: 0o600 });diff --git a/src/runner-worker.js b/src/runner-worker.jsindex 7db359ac..083898fc 100644--- a/src/runner-worker.js+++ b/src/runner-worker.js@@ -22,6 +22,9 @@ import {statSync,writeFileSync,} from 'node:fs';+import { availableParallelism } from 'node:os';+import { freemem } from 'node:os';+import { loadavg } from 'node:os';import { userInfo } from 'node:os';import {delimiter,@@ -1203,6 +1206,8 @@ async function receipt(spec, fields, sink) {await flushStdout();const body = { id: spec.id, commit: spec.commit, tree: spec.tree, code: fields.code, signal: fields.signal,runtimeSha256: spec.runtime.sha256, toolingSha256: spec.tooling.sha256, root: fields.root ?? '', outputSha256: sink.hash.digest('hex'), wallMs: Math.max(0, Date.now() - sink.wallStart),+ // The execution host's load and the pinned main this run used, so a paired rerun here can be judged (#925).+ main: typeof spec.main === 'string' ? spec.main : null, load: { average: loadavg(), cpus: availableParallelism(), freeMB: Math.floor(freemem() / 1048576) },...(spec.stageMarker ? { pipeStatuses: fields.pipeStatuses ?? null } : {}), ...(fields.refused ? { refused: fields.refused } : {}), ...(fields.denial ? { denial: fields.denial } : {}) };const encoded = Buffer.from(JSON.stringify(body)).toString('base64');await new Promise((resolveWrite, rejectWrite) => process.stderr.write(`${RECEIPT} ${spec.id} ${encoded}\n`, error => error ? rejectWrite(error) : resolveWrite()));diff --git a/src/runners.js b/src/runners.jsindex 74e5be77..1dc26df6 100644--- a/src/runners.js+++ b/src/runners.js@@ -445,7 +445,8 @@ function requestFor(root, command, { runner, source, browser = false, pipefail =browser: runner.browser, browserRequired: browser, landing: metadata.landing === true,...(stages ? { stageMarker: `PULLBOARD_STAGE_${randomUUID().replaceAll('-', '')}` } : {}), waivers: projection,itemCheck: metadata.itemCheck === true, diagnostic: metadata.diagnostic === true,- item: String(metadata.item ?? ''), kind: String(metadata.kind ?? ''), ...(runner.root ? { root: runner.root } : {}) };+ item: String(metadata.item ?? ''), kind: String(metadata.kind ?? ''), ...(runner.root ? { root: runner.root } : {}),+ ...(typeof metadata.main === 'string' && /^[0-9a-f]{40,64}$/u.test(metadata.main) ? { main: metadata.main } : {}) };const encodedSpec = Buffer.from(JSON.stringify(spec));if (encodedSpec.length > 1024 ** 2) throw new Refused('RUNNER_REQUEST_LIMIT', 'runner tooling request exceeds its safe transfer limit; keep this run local');const remote = remoteWorkerCommand(runner, worker, tooling, runtime);@@ -501,7 +502,15 @@ function resultFrom(request, result) {...(request.spec.stageMarker ? { stages: staged?.stages ?? null } : {}),seconds: Math.round(receipt.wallMs / 1000), exitCode: receipt.code,executionRunner: request.runner.host, executionRoot: receipt.root, remoteWallMs: receipt.wallMs,- executionSource: { commit: receipt.commit, tree: receipt.tree, tooling: request.spec.tooling } };+ executionSource: { commit: receipt.commit, tree: receipt.tree, tooling: request.spec.tooling },+ executionMain: request.spec.main && receipt.main === request.spec.main ? receipt.main : null, executionLoad: remoteLoad(receipt.load) };+}++/** The execution host's load from a receipt, or null when any part of it is missing or malformed (#925). */+function remoteLoad(load) {+ const average = Array.isArray(load?.average) && load.average.length === 3 && load.average.every((value) => Number.isFinite(value) && value >= 0) ? load.average : null;+ return average && Number.isInteger(load.cpus) && load.cpus > 0 && Number.isInteger(load.freeMB) && load.freeMB >= 0+ ? { average, cpus: load.cpus, freeMB: load.freeMB } : null;}/** Keep synchronous gate callers on the same negotiated transport as asynchronous comparisons. */diff --git a/test/runners.test.js b/test/runners.test.jsindex a4c854d6..c290b47c 100644--- a/test/runners.test.js+++ b/test/runners.test.js@@ -1914,3 +1914,58 @@ test('a refused runner checkout names its failed Git step and stderr [C6,C7]', aawait t.test(mode, child => sourceSetupFailureFixture(child, { cli, git, runThroughGate, gateResult }, mode));}});++test('a red on a registered runner gets its paired rerun and ledger entry, or stays unclassified without the host load [C7]', { timeout: 300_000 }, async (t) => {+ for (const mode of ['remote-load', 'no-load']) await t.test(mode, async () => {+ const helper = createE2eHelpers();+ t.after(helper.cleanup);+ const box = helper.project('true');+ const repo = box.repo;+ const [fixtureBin, remoteHome, worker] = ['runner-bin', 'runner-home', 'second-machine'].map((name) => join(box.dir, name));+ const remoteMachineHome = join(remoteHome, '.pullboard');+ for (const folder of [fixtureBin, remoteMachineHome, worker, join(repo, 'test')]) mkdirSync(folder, { recursive: true });+ const env = { ...box.env, PATH: fixtureBin + ':' + box.env.PATH };+ writeFileSync(join(repo, 'package.json'), JSON.stringify({ private: true, type: 'module' }) + '\n');+ for (const part of ['src', 'bin']) cpSync(resolve(import.meta.dirname, '..', part), join(repo, part), { recursive: true });+ // Red on its first run, green on its paired rerun: a flake the ledger should record.+ const runs = join(box.dir, 'flaky-runs');+ writeFileSync(join(repo, 'test/flaky.test.js'), `import { test } from 'node:test'; import { appendFileSync, existsSync, readFileSync } from 'node:fs';+test('remote flaky case', () => { const n = existsSync(${JSON.stringify(runs)}) ? readFileSync(${JSON.stringify(runs)}, 'utf8').length : 0; appendFileSync(${JSON.stringify(runs)}, 'x'); if (n === 0) throw new Error('first run fails'); });\n`);+ const config = JSON.parse(readFileSync(join(repo, 'pullboard.json'), 'utf8'));+ writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ ...config, gate: 'node bin/run-tests.js test/flaky.test.js', flakes: { rerun: true } }) + '\n');+ box.git(repo, 'add', '-A');+ box.git(repo, 'commit', '-q', '-m', 'test: a remote flaky fixture');+ // The worker either reports a known load, or (as an older worker would) a receipt without one.+ const preload = join(box.dir, 'worker-preload.mjs');+ writeFileSync(preload, mode === 'remote-load'+ ? "import { createRequire } from 'node:module'; const require = createRequire(import.meta.url); const os = require('node:os'); os.loadavg = () => [7.25, 7.25, 7.25]; require('node:module').syncBuiltinESMExports();\n"+ : "const write = process.stderr.write.bind(process.stderr); process.stderr.write = (chunk, ...rest) => { const text = String(chunk); const match = /^(PULLBOARD_RUNNER_RESULT \\S+ )(\\S+)\\n$/u.exec(text); if (!match) return write(chunk, ...rest); const body = JSON.parse(Buffer.from(match[2], 'base64').toString('utf8')); delete body.load; return write(match[1] + Buffer.from(JSON.stringify(body)).toString('base64') + '\\n', ...rest); };\n");+ installSshStandIn({ bin: fixtureBin, worker, home: remoteHome, machineHome: remoteMachineHome, requests: join(box.dir, 'requests.jsonl'), nodeOptions: '--import=' + pathToFileURL(preload).href });+ assert.equal(cli(repo, env, 'settings', 'gateSlots', '1', '--json').status, 0);+ const registered = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');+ assert.equal(registered.status, 0, fixtureResultContext(registered));+ const prior = { home: process.env.PULLBOARD_HOME, machine: process.env.PULLBOARD_MACHINE_HOME };+ Object.assign(process.env, { PULLBOARD_HOME: env.PULLBOARD_HOME, PULLBOARD_MACHINE_HOME: env.PULLBOARD_MACHINE_HOME });+ const held = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo, agent: 'flake-local-holder', item: 'flake-local-holder', kind: 'force-registered-runner' });+ try {+ const run = observe(startToolingCli(repo, env, ['gate', '--json'], { detached: true }));+ const finished = await awaitRoundTrip(run.child, run.closed, join(box.dir, 'requests.jsonl.phases'), { dispatched: 0, returned: 0 });+ assert.equal(finished.status, 1, 'the original red stays the gate result');+ } finally {+ held.release();+ for (const [key, value] of [['PULLBOARD_HOME', prior.home], ['PULLBOARD_MACHINE_HOME', prior.machine]]) if (value === undefined) delete process.env[key]; else process.env[key] = value;+ }+ const evidence = join(repo, '.git', 'pullboard', 'flake-evidence');+ const [observation] = readdirSync(evidence).map((id) => JSON.parse(readFileSync(join(evidence, id, 'observation.json'), 'utf8')));+ assert.equal(observation.main, box.git(repo, 'rev-parse', 'HEAD'), 'the pinned main is recorded');+ if (mode === 'remote-load') {+ assert.equal(observation.classification, 'flake', observation.reason);+ assert.deepEqual(observation.load.average, [7.25, 7.25, 7.25], 'the execution host\'s load is recorded, not the controller\'s');+ assert.equal(observation.paired?.green, true, 'the one paired rerun ran and passed');+ } else {+ assert.equal(observation.classification, 'unclassified');+ assert.match(observation.reason, /does not carry the execution host's load/u);+ assert.equal(observation.paired, null, 'no rerun is guessed without the host load');+ }+ });+});