Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(runners): retain bounded setup diagnostics [C7]
Corey Olsone10c0eb1a23d
diff --git a/src/runner-worker.js b/src/runner-worker.jsindex 3c19d524..7a367627 100644--- a/src/runner-worker.js+++ b/src/runner-worker.js@@ -834,16 +834,49 @@ async function acquireResourceApi(api, options, machineHomePath) {}}-/** Invoke one bounded Git command in an owned group and return its exact stdout. */+/** Bound Git diagnostics and hide runner-private paths, URL credentials and credential assignments. */+function safeGitDiagnostic(value, root, env) {+ let text = String(value).replace(/\u001b\[[0-?]*[ -/]*[@-~]/gu, '').replace(/\r\n?/gu, '\n').replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu, '');+ for (const [path, label] of [[root, '<source>'], [env.HOME, '<job-home>'], [process.env.HOME, '<runner-home>'], [env.TMPDIR, '<job-tmp>']]) {+ if (path && path.length > 1) text = text.split(path).join(label);+ }+ text = text.replace(/https?:\/\/[^\s"'<>]+/giu, '<url>')+ .replace(/\b(authorization|cookie)\s*[:=][ \t]*[^\n]+/giu, '$1=<redacted>')+ .replace(/\b(authorization|cookie|token|secret|password|api[_-]?key|access[_-]?key)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s;]+)/giu, '$1=<redacted>')+ .replace(/\b(?:gh[pousr]_|github_pat_|ps_|pa_|pm_|pg_|sk-)[A-Za-z0-9_-]{8,}/gu, '<redacted>')+ .replace(/\b(?:10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b/gu, '<private-address>').trim();+ const bytes = Buffer.from(text);+ return bytes.length > 2_048 ? `${bytes.subarray(0, 768).toString('utf8')}\n...[truncated]...\n${bytes.subarray(-1_024).toString('utf8')}` : text;+}++/** Keep the failed Git operation and bounded cause without exposing command arguments or raw errors. */+function gitSetupFailure(root, args, env, result, stderr, overflow, inputError) {+ const commands = new Set(['init', 'fetch', 'checkout', 'index-pack', 'cat-file', 'rev-parse', 'rev-list', 'for-each-ref', 'update-ref', 'merge-base', 'verify-pack', 'check-ref-format']);+ const operation = args.find(arg => commands.has(arg)) ?? 'command';+ const role = root.endsWith('.git') || args.includes('--bare') ? 'source cache' : 'source checkout';+ const code = result.error?.code ?? inputError?.code;+ const status = overflow ? 'stdout limit exceeded' : result.error && /^[A-Z][A-Z0-9_]{0,63}$/u.test(code ?? '') ? code+ : Number.isInteger(result.code) && result.code !== 0 ? `exit ${result.code}`+ : result.signal ? `signal ${result.signal}` : /^[A-Z][A-Z0-9_]{0,63}$/u.test(code ?? '') ? code : 'input failed';+ const error = new Error('source checkout could not be verified');+ error.runnerSetup = { step: `${role}: git ${operation}`, status, stderr: safeGitDiagnostic(stderr, root, env) };+ return error;+}++/** Invoke bounded Git in an owned group, retaining stderr if it fails instead of erasing the cause. */async function git(root, args, env, input) {- const owned = spawnOwned('git', args, { cwd: root, env, stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'ignore'] });+ const owned = spawnOwned('git', args, { cwd: root, env, stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'] });const output = [];+ let stderr = Buffer.alloc(0);+ owned.child.stderr.on('data', chunk => { stderr = Buffer.concat([stderr, chunk]).subarray(-8_192); });let bytes = 0;let overflow = false;let inputError;+ let inputDone;if (input !== undefined) {owned.child.stdin.on('error', error => { inputError ??= error; });- owned.child.stdin.end(input);+ if (typeof input.pipe === 'function') inputDone = pipeline(input, owned.child.stdin).catch(error => { inputError ??= error; });+ else owned.child.stdin.end(input);}owned.child.stdout.on('data', chunk => {bytes += chunk.length;@@ -852,8 +885,9 @@ async function git(root, args, env, input) {} else if (!overflow) output.push(chunk);});const result = await owned.closed;+ await inputDone;await drainOwned(owned);- if (overflow || inputError || result.error || result.code !== 0 || result.signal) throw new Error('source checkout could not be verified');+ if (overflow || inputError || result.error || result.code !== 0 || result.signal) throw gitSetupFailure(root, args, env, result, stderr.toString('utf8'), overflow, inputError);return Buffer.concat(output).toString('utf8').trim();}@@ -872,7 +906,10 @@ async function checkoutHistory(api, runnerRoot, root, packFile, spec, env) {name: `repo-${spec.repoId}`, capacity: 1, scope: 'machine', root: runnerRoot,repo: runnerRoot, agent: `pullboard-ssh-cache-${spec.id}`, kind: 'source-cache',onWait: () => assertTransportLive(),- }, join(runnerRoot, 'cache', 'resource-pool'));+ }, join(runnerRoot, 'cache', 'resource-pool')).catch(error => {+ if (error && typeof error === 'object') error.runnerSetupStep = 'source cache admission';+ throw error;+ });const deliveryRefs = [];try {if (!existsSync(cache)) {@@ -880,24 +917,14 @@ async function checkoutHistory(api, runnerRoot, root, packFile, spec, env) {const staged = join(build, 'cache.git');try {const initialize = async () => {- const initialized = spawnOwned('git', ['init', '--bare', `--object-format=${spec.commit.length === 64 ? 'sha256' : 'sha1'}`, staged], {- cwd: build, env, stdio: ['ignore', 'ignore', 'ignore'],- });- const result = await initialized.closed;- await drainOwned(initialized);- if (result.error || result.code !== 0 || result.signal) throw new Error('source history cache could not be initialized');+ await git(build, ['init', '--bare', `--object-format=${spec.commit.length === 64 ? 'sha256' : 'sha1'}`, staged], env);};await initialize();if (spec.origin && spec.originBase) {let seeded = false;try {- const fetch = spawnOwned('git', ['-c', 'protocol.version=2', 'fetch', '--no-tags', '--no-recurse-submodules', spec.origin, spec.originBase], {- cwd: staged, env, stdio: ['ignore', 'ignore', 'ignore'],- });- const result = await fetch.closed;- await drainOwned(fetch);- seeded = !result.error && result.code === 0 && !result.signal- && await git(staged, ['cat-file', '-t', spec.originBase], env) === 'commit'+ await git(staged, ['-c', 'protocol.version=2', 'fetch', '--no-tags', '--no-recurse-submodules', spec.origin, spec.originBase], env);+ seeded = await git(staged, ['cat-file', '-t', spec.originBase], env) === 'commit'&& Number(await git(staged, ['rev-list', '--count', spec.originBase], env)) > 0&& !existsSync(join(staged, 'shallow'));if (seeded) await git(staged, ['update-ref', `refs/pullboard/published/${spec.originBase}`, spec.originBase], env);@@ -953,15 +980,7 @@ async function checkoutHistory(api, runnerRoot, root, packFile, spec, env) {const priorIndexes = new Set(readdirSync(join(cache, 'objects', 'pack')).filter(name => name.endsWith('.idx')));if (statSync(packFile).size < 12 + spec.commit.length / 2) throw new Error('source history pack is incomplete');- {- const index = spawnOwned('git', ['index-pack', '--stdin'], { cwd: cache, env, stdio: ['pipe', 'ignore', 'ignore'] });- try {- await pipeline(createReadStream(packFile), index.child.stdin);- const result = await index.closed;- await drainOwned(index);- if (result.error || result.code !== 0 || result.signal) throw new Error('source history pack could not be indexed');- } catch (error) { await drainOwned(index, true); throw error; }- }+ await git(cache, ['index-pack', '--stdin'], env, createReadStream(packFile));const objectType = await git(cache, ['cat-file', '-t', spec.commit], env);if (objectType !== 'commit' || await git(cache, ['rev-parse', `${spec.commit}^{tree}`], env) !== spec.tree) {throw new Error('source history does not match the requested commit and tree');@@ -1201,12 +1220,18 @@ function stopReading() {process.stdin.destroy();}-/** Turn an internal setup failure into a fixed safe line rather than disclosing paths or runner state. */-function safeSetupMessage(error) {+/** Name a setup phase and bounded cause while hiding private paths and credentials. */+function safeSetupMessage(error, step) {+ if (error?.runnerSetup) {+ const detail = error.runnerSetup;+ return `runner setup refused: ${detail.step} failed (${detail.status})\ngit stderr: ${detail.stderr || '(none)'}`;+ }const message = String(error?.message ?? '');- if (message.includes('npm') || message.includes('dependency')) return 'runner setup refused: committed dependencies are unavailable';- if (message.includes('capacity') || message.includes('queue')) return 'runner setup refused: remote capacity is unavailable';- return 'runner setup refused: source checkout could not be prepared';+ const category = message.includes('npm') || message.includes('dependency') ? 'committed dependencies are unavailable'+ : message.includes('capacity') || message.includes('queue') ? 'remote capacity is unavailable' : 'source checkout could not be prepared';+ const code = /^[A-Z][A-Z0-9_]{0,63}$/u.test(error?.code ?? '') ? ` (${error.code})` : '';+ const cause = safeGitDiagnostic(message, '', { HOME: process.env.HOME, TMPDIR: process.env.TMPDIR });+ return `runner setup refused: ${category}; step: ${error?.runnerSetupStep ?? step}${code}\ncause: ${cause || '(unavailable)'}`;}/** Execute one remote job end to end; stdout is reserved for raw command bytes and stderr for one receipt. */@@ -1220,6 +1245,7 @@ async function main() {let lease = null;let tracker = null;let runDirectory;+ let setupStep = 'runner directories';let final = { code: 1, signal: null, root: '' };try {const runnerHome = isAbsolute(process.env.HOME || '') ? process.env.HOME : userInfo().homedir;@@ -1229,6 +1255,7 @@ async function main() {const configuredMachineHome = process.env.PULLBOARD_MACHINE_HOME;if (configuredMachineHome !== undefined && !isAbsolute(configuredMachineHome)) throw new Error('machine home is invalid');const machineHomePath = configuredMachineHome ?? join(runnerHome, '.pullboard');+ setupStep = 'trusted resource API';api = loadResourceApi(spec.runtime);const previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;let resources;@@ -1243,6 +1270,7 @@ async function main() {// pullboard settings browserSlots writes this row and settings.json under one lock, so the row carries// this machine's setting without reading settings.json; a machine that set nothing keeps one browser slot.const browserCapacity = resources.find(resource => resource.name === 'browser')?.capacity ?? 1;+ setupStep = 'job directories';const base = join(runnerRoot, 'jobs');mkdirSync(base, { recursive: true, mode: 0o700 });chmodSync(base, 0o700);@@ -1255,6 +1283,7 @@ async function main() {writeFileSync(join(privateHome, 'npm-userconfig'), '', { mode: 0o600 });writeFileSync(join(privateHome, 'npm-globalconfig'), '', { mode: 0o600 });const packFile = join(runDirectory, 'source.pack');+ setupStep = 'runner gate admission';lease = await acquireResourceApi(api, {name: 'gate', capacity: gateCapacity, scope: 'machine', root: runnerRoot,repo: runnerRoot, agent: `pullboard-ssh-${spec.id}`,@@ -1269,12 +1298,15 @@ async function main() {cleanupDetails = tracker.cleanupState;activeDescendantTracker = tracker;const env = scopedEnvironment(privateHome, temporary, canonicalMachineHome, lease, api, tracker.nodeOptions);+ setupStep = 'source history checkout';await checkoutHistory(api, runnerRoot, root, packFile, spec, env);+ setupStep = 'committed dependencies';await installDependencies(root, runnerRoot, env, tracker);+ setupStep = 'command start';final = { ...(await runCommand(spec, root, { ...env, PULLBOARD_LANDING_RUN: spec.landing === true ? '1' : '0', PULLBOARD_LANDING_WAIVERS: JSON.stringify(spec.landing === true ? spec.waivers ?? [] : []) }, sink, tracker)), root };} catch (error) {const refused = setupRefusalCode(error);- const message = refused ? `${safeSetupMessage(error)} [${refused}]` : safeSetupMessage(error);+ const message = refused ? `${safeSetupMessage(error, setupStep)} [${refused}]` : safeSetupMessage(error, setupStep);if (!transportLost) await sink.write(`${message}\n`);final = { code: 1, signal: transportLost ? 'SIGHUP' : null, error: message, root: '', ...(refused ? { refused } : {}) };} finally {diff --git a/test/runner-source-setup-fixture.js b/test/runner-source-setup-fixture.jsnew file mode 100644index 00000000..ea8a2437--- /dev/null+++ b/test/runner-source-setup-fixture.js@@ -0,0 +1,93 @@+/** A real checkout lock proves runner setup diagnostics without changing an owner repository [C6,C7]. */+import assert from 'node:assert/strict';+import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { pathToFileURL } from 'node:url';+import { takeResource } from '../src/resources.js';+import { HOST, SECRET, installSshStandIn, shellQuote } from './runner-ssh-fixture.js';++/** Force Git's own checkout lock failure and require an actionable, path-free setup result. */+export async function sourceSetupFailureFixture(t, { cli, git, runThroughGate, gateResult }, mode = 'lock') {+ const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-setup-cause-')));+ let completed = false;+ t.after(() => {+ if (completed) rmSync(dir, { recursive: true, force: true });+ else t.diagnostic(`retained failed source-setup fixture at ${dir}`);+ });+ const home = join(dir, 'home'), machineHome = join(home, '.pullboard');+ const remoteHome = join(dir, 'worker-home'), remoteMachineHome = join(remoteHome, '.pullboard');+ const repo = join(dir, 'repo'), worker = join(dir, 'worker'), bin = join(dir, 'bin');+ const requests = join(dir, 'requests.jsonl'), injected = join(dir, 'injected');+ const executed = join(dir, 'command-executed'), preload = join(dir, 'checkout-lock.mjs');+ for (const path of [home, machineHome, remoteHome, remoteMachineHome, repo, worker, bin]) mkdirSync(path, { recursive: true });+ writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));+ const env = { ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,+ PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',+ GIT_AUTHOR_NAME: 'Setup Fixture', GIT_AUTHOR_EMAIL: '[email protected]',+ GIT_COMMITTER_NAME: 'Setup Fixture', GIT_COMMITTER_EMAIL: '[email protected]' };+ git(repo, env, 'init', '-q', '-b', 'main');+ writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'true' }));+ writeFileSync(join(repo, 'tracked.txt'), 'source\n');+ git(repo, env, 'add', '.'); git(repo, env, 'commit', '-qm', 'test: source setup fixture');+ if (mode === 'filesystem') {+ const state = join(remoteHome, '.pullboard-runner');+ mkdirSync(state, { recursive: true });+ writeFileSync(join(state, 'repos'), 'a file prevents creating the cache directory\n');+ writeFileSync(injected, 'cache directory obstacle injected\n');+ }+ writeFileSync(preload, `import childProcess from 'node:child_process';+import { syncBuiltinESMExports } from 'node:module';+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+const spawn = childProcess.spawn;+childProcess.spawn = (command, args, options) => {+ const operation = ${JSON.stringify(mode)} === 'index' ? 'index-pack' : 'checkout';+ const scope = ${JSON.stringify(mode)} === 'index' ? ${JSON.stringify(join(remoteHome, '.pullboard-runner', 'repos') + '/')} : ${JSON.stringify(join(remoteHome, '.pullboard-runner', 'jobs') + '/')};+ if (command === 'git' && args[0] === operation && options.cwd.startsWith(scope)) {+ writeFileSync(${JSON.stringify(injected)}, 'checkout lock injected\\n');+ if (${JSON.stringify(mode)} === 'index') return spawn(command, [...args, '--fixture-invalid-option'], options);+ if (${JSON.stringify(mode)} === 'missing') return spawn(${JSON.stringify(join(dir, 'missing-git'))}, args, options);+ if (${JSON.stringify(mode)} === 'large') return spawn(process.execPath, ['-e', ${JSON.stringify(`process.stderr.write('x'.repeat(20_000) + ${JSON.stringify(`\nfatal: fixture tail token=${SECRET} https://user:[email protected]/repo?token=${SECRET} ${remoteHome}/private.txt 192.168.1.99\nAuthorization: Bearer fixture-bearer-secret\nCookie: session=fixture-cookie-secret\n`)}); process.exit(23);`)}], options);+ writeFileSync(join(options.cwd, '.git', 'index.lock'), 'held by this fixture\\n');+ }+ return spawn(command, args, options);+};+syncBuiltinESMExports();\n`);+ installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests,+ nodeOptions: `--import=${pathToFileURL(preload).href}` });+ const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');+ assert.equal(added.status, 0, added.context);+ const previous = [process.env.PULLBOARD_HOME, process.env.PULLBOARD_MACHINE_HOME];+ process.env.PULLBOARD_HOME = home; process.env.PULLBOARD_MACHINE_HOME = machineHome;+ let held;+ try {+ held = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo, agent: 'setup-holder' });+ const command = `node -e ${shellQuote(`require('node:fs').writeFileSync(${JSON.stringify(executed)}, 'ran')`)}`;+ const result = await runThroughGate(repo, env, command, { requests });+ assert.equal(gateResult(result.stdout).isGreen, false, 'the failed setup is not a successful run');+ assert.equal(existsSync(injected), true, 'the real checkout Git operation encountered the lock');+ assert.equal(existsSync(executed), false, 'the requested command never ran or fell back locally');+ if (mode === 'filesystem') {+ assert.match(result.stdout, /step: source history checkout.*EEXIST[\s\S]*cause:/u, 'filesystem setup keeps its phase, code and cause');+ } else if (mode === 'index') {+ assert.match(result.stdout, /source cache: git index-pack.*exit \d+[\s\S]*git stderr:.*(?:usage|error)/iu, 'a broken pack command retains Git stderr even when its input pipe fails');+ } else assert.match(result.stdout, /git checkout/iu, 'the refusal names the failed Git step');+ if (mode === 'lock') assert.match(result.stdout, /index\.lock.*File exists/isu, 'the refusal retains Git’s concrete lock cause');+ if (mode === 'missing') assert.match(result.stdout, /ENOENT/u, 'a spawn failure keeps its code instead of an undefined exit');+ if (mode === 'large') {+ assert.match(result.stdout, /exit 23[\s\S]*fixture tail/u, 'bounded stderr retains the concrete failure tail');+ assert.ok(!result.stdout.includes(SECRET) && !result.stdout.includes('fake-password'), 'credential assignments and URL credentials stay private');+ assert.ok(!result.stdout.includes('fixture-bearer-secret') && !result.stdout.includes('fixture-cookie-secret'), 'spaced bearer and cookie credentials stay private');+ assert.ok(!result.stdout.includes('192.168.1.99'), 'private network addresses stay private');+ assert.ok(Buffer.byteLength(result.stdout.split('PULLBOARD_RUN_RESULT ')[0]) < 2_400, 'the diagnostic is bounded in bytes');+ }+ assert.ok(!result.stdout.includes(remoteHome), 'runner-private paths never reach the controller');+ completed = true;+ } finally {+ held?.release();+ for (const [index, key] of ['PULLBOARD_HOME', 'PULLBOARD_MACHINE_HOME'].entries()) {+ if (previous[index] === undefined) delete process.env[key]; else process.env[key] = previous[index];+ }+ }+}diff --git a/test/runner-ssh-fixture.js b/test/runner-ssh-fixture.jsnew file mode 100644index 00000000..64a9abe1--- /dev/null+++ b/test/runner-ssh-fixture.js@@ -0,0 +1,124 @@+/** A private SSH stand-in executes the pinned runner capsule without contacting a host [C7,O5]. */+import { chmodSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';++export const SECRET = ['fixture', 'runner', 'secret', 'must', 'stay', 'local'].join('-');+export const HOST = 'fixture-host';++/** Quote one literal value for the fixture's POSIX shell command. */+export function shellQuote(value) {+ return `'${String(value).replaceAll("'", "'\\''")}'`;+}++/**+ * Make a local ssh stand-in that executes the actual worker command in a second private machine.+ * It appends "dispatched" and "returned" to <requests>.phases so a round trip can be timed by phase,+ * and `hold` ('dispatch', 'run' or 'return') keeps it alive at that phase, as a stuck runner would.+ * `stderrLog` also copies the worker's stderr, so a test can read its result receipt, and `nodeOptions` reaches the+ * worker's own Node process, so a test can force an ordering inside it.+ */+export function installSshStandIn({ bin, worker, home, machineHome, requests, starts, nodeCommand = 'node', dropFirst = false, dropAfterMarker, hold = '', stderrLog, nodeOptions = '' }) {+ const ssh = join(bin, 'ssh');+ const source = `#!/usr/bin/env node+import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs';+import { spawn } from 'node:child_process';+import { createHash } from 'node:crypto';+const argv = process.argv.slice(2);+const divider = argv.indexOf('--');+const host = divider >= 0 ? argv[divider + 1] : '';+const remoteCommand = divider >= 0 ? argv[divider + 2] : '';+const requests = ${JSON.stringify(requests)};+const starts = ${JSON.stringify(starts ?? '')};+const worker = ${JSON.stringify(worker)};+const home = ${JSON.stringify(home)};+const machineHome = ${JSON.stringify(machineHome)};+const configuredNode = ${JSON.stringify(`${shellQuote(nodeCommand)} --input-type=module -e `)};+const dropAfterMarker = ${JSON.stringify(dropAfterMarker ?? '')};+const secret = ${JSON.stringify(SECRET)};+const call = existsSync(requests) ? readFileSync(requests, 'utf8').split('\\n').filter(Boolean).length + 1 : 1;+const stdinHash = createHash('sha256');+let stdinBytes = 0;+process.stdin.on('data', part => { stdinBytes += part.length; stdinHash.update(part); });+const record = extra => appendFileSync(requests, JSON.stringify({ call, host,+ options: divider < 0 ? [] : argv.slice(0, divider), remoteHasSecret: remoteCommand.includes(secret),+ framed: divider >= 0 && argv[divider] === '--', remoteCommandPresent: Boolean(remoteCommand),+ remoteStartsWithConfiguredNode: remoteCommand.startsWith(configuredNode),+ remoteHasLocalPath: remoteCommand.includes(${JSON.stringify(worker)}),+ remoteHasSourceHome: remoteCommand.includes(${JSON.stringify(home)}),+ remoteHasSourceMachineHome: remoteCommand.includes(${JSON.stringify(machineHome)}), ...extra }) + '\\n');+/** Wait for the concrete setup marker and allow a finished worker to cancel its wait. */+const waitForMarker = path => {+ let cancel = () => {};+ const promise = new Promise((resolve, reject) => {+ if (existsSync(path)) return resolve(true);+ let poll, timer;+ let settled = false;+ const finish = (error, found = true) => {+ if (settled) return;+ settled = true;+ clearInterval(poll); clearTimeout(timer); error ? reject(error) : resolve(found);+ };+ cancel = () => finish(null, false);+ const check = () => { if (existsSync(path)) finish(); };+ poll = setInterval(check, 20); // budget: observe actual setup readiness without relying on notifications.+ timer = setTimeout(() => finish(new Error('setup marker hang ceiling exceeded')), 20_000); // budget: unchanged SSH-to-setup handshake ceiling.+ check();+ });+ return { promise, cancel: () => cancel() };+};+let cancelDropWait = () => {};+if (host !== ${JSON.stringify(HOST)} && host !== ${JSON.stringify(`${HOST}-drop`)} && host !== ${JSON.stringify(`${HOST}-setup`)}) {+ record({ refused: true, stdinBytes }); process.exit(255);+}+if (starts) appendFileSync(starts, JSON.stringify({ call, host }) + '\\n');+const phases = requests + '.phases';+const hold = ${JSON.stringify(hold)};+const holdHere = () => new Promise(() => { setInterval(() => {}, 60_000); });+if (hold === 'dispatch') await holdHere();+appendFileSync(phases, 'dispatched\\n');+if (hold === 'run') await holdHere();+${dropFirst ? `if (host.endsWith('-drop') && call === 1) {+ process.stdin.once('data', () => {+ record({ dropped: true, stdinBytes, stdinSha256: stdinHash.digest('hex') }); process.exit(255);+ });+ process.stdin.resume();+} else` : ''} {+ const env = { PATH: process.env.PATH, HOME: home, TMPDIR: process.env.TMPDIR,+ PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome, LANG: 'C', LC_ALL: 'C', TZ: 'UTC',+ ...(${JSON.stringify(nodeOptions)} ? { NODE_OPTIONS: ${JSON.stringify(nodeOptions)} } : {}) };+ // Replace the marker-driven shell so the injected HUP reaches the actual worker process.+ const child = spawn('/bin/sh', ['-c', remoteCommand], { cwd: worker, env,+ stdio: ['pipe', 'pipe', 'pipe'] });+ child.stdout.pipe(process.stdout);+ child.stderr.pipe(process.stderr);+${stderrLog ? ` child.stderr.on('data', part => appendFileSync(${JSON.stringify(stderrLog)}, part));\n` : ''} process.stdin.pipe(child.stdin);+ let droppedAfterSetup = false;+ if (dropAfterMarker) {+ const waiting = waitForMarker(dropAfterMarker);+ cancelDropWait = waiting.cancel;+ waiting.promise.then(found => {+ if (found === false) return;+ droppedAfterSetup = true;+ process.kill(child.pid, 'SIGHUP');+ }, error => { record({ setupMarkerError: error.message }); child.kill('SIGKILL'); });+ }+ child.once('error', error => { record({ spawnError: error.code ?? error.name, stdinBytes }); process.exitCode = 255; });+ child.once('close', (code, signal) => {+ appendFileSync(phases, 'returned\\n');+ if (hold === 'return') setInterval(() => {}, 60_000);+ cancelDropWait();+ record({ workerStarted: true, code, signal, stdinBytes, stdinSha256: stdinHash.digest('hex'),+ droppedAfterSetup,+ secretInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_RUNNER_FIXTURE_SECRET'),+ localPoolMarkerInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_GATE_SLOT_POOL'),+ localHeldMarkerInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_GATE_SLOT_HELD'),+ localTokenMarkerInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_GATE_SLOT_TOKEN'),+ localMachineHomeInSshEnv: Object.hasOwn(process.env, 'PULLBOARD_MACHINE_HOME') });+ process.exitCode = signal ? 128 : (code ?? 255);+ });+}+`;+ writeFileSync(ssh, source, { mode: 0o700 });+ chmodSync(ssh, 0o700);+ return ssh;+}diff --git a/test/runners.test.js b/test/runners.test.jsindex a55e2f06..45103ee9 100644--- a/test/runners.test.js+++ b/test/runners.test.js@@ -16,20 +16,15 @@ import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state.import { parentDeathDrainFixture } from './runner-parent-death-fixture.js';import { createE2eHelpers } from './e2e-helpers.js';import { refuseSandboxBlocked } from '../src/gate.js';+import { SECRET, HOST, shellQuote, installSshStandIn } from './runner-ssh-fixture.js';+import { sourceSetupFailureFixture } from './runner-source-setup-fixture.js';const BIN = resolve(import.meta.dirname, '../bin/pullboard.js');const TEMP = [];-const SECRET = ['fixture', 'runner', 'secret', 'must', 'stay', 'local'].join('-');-const HOST = 'fixture-host';const TOOLING_SNAPSHOTS = new Map();const e2e = createE2eHelpers();after(e2e.cleanup);-/** Quote one literal value for the fixture's POSIX shell command. */-function shellQuote(value) {- return `'${String(value).replaceAll("'", "'\\''")}'`;-}-/** Run Git in the private source repository and preserve useful failure context. */function git(repo, env, ...args) {const result = runFixtureChild('git', args, { cwd: repo, env, encoding: 'utf8' });@@ -84,119 +79,6 @@ function cli(repo, env, ...args) {return runFixtureChild(process.execPath, [BIN, ...args], { cwd: repo, env, encoding: 'utf8' });}-/**- * Make a local ssh stand-in that executes the actual worker command in a second private machine.- * It appends "dispatched" and "returned" to <requests>.phases so a round trip can be timed by phase,- * and `hold` ('dispatch', 'run' or 'return') keeps it alive at that phase, as a stuck runner would.- * `stderrLog` also copies the worker's stderr, so a test can read its result receipt, and `nodeOptions` reaches the- * worker's own Node process, so a test can force an ordering inside it.- */-function installSshStandIn({ bin, worker, home, machineHome, requests, starts, nodeCommand = 'node', dropFirst = false, dropAfterMarker, hold = '', stderrLog, nodeOptions = '' }) {- const ssh = join(bin, 'ssh');- const source = `#!/usr/bin/env node-import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs';-import { spawn } from 'node:child_process';-import { createHash } from 'node:crypto';-const argv = process.argv.slice(2);-const divider = argv.indexOf('--');-const host = divider >= 0 ? argv[divider + 1] : '';-const remoteCommand = divider >= 0 ? argv[divider + 2] : '';-const requests = ${JSON.stringify(requests)};-const starts = ${JSON.stringify(starts ?? '')};-const worker = ${JSON.stringify(worker)};-const home = ${JSON.stringify(home)};-const machineHome = ${JSON.stringify(machineHome)};-const configuredNode = ${JSON.stringify(`${shellQuote(nodeCommand)} --input-type=module -e `)};-const dropAfterMarker = ${JSON.stringify(dropAfterMarker ?? '')};-const secret = ${JSON.stringify(SECRET)};-const call = existsSync(requests) ? readFileSync(requests, 'utf8').split('\\n').filter(Boolean).length + 1 : 1;-const stdinHash = createHash('sha256');-let stdinBytes = 0;-process.stdin.on('data', part => { stdinBytes += part.length; stdinHash.update(part); });-const record = extra => appendFileSync(requests, JSON.stringify({ call, host,- options: divider < 0 ? [] : argv.slice(0, divider), remoteHasSecret: remoteCommand.includes(secret),- framed: divider >= 0 && argv[divider] === '--', remoteCommandPresent: Boolean(remoteCommand),- remoteStartsWithConfiguredNode: remoteCommand.startsWith(configuredNode),- remoteHasLocalPath: remoteCommand.includes(${JSON.stringify(worker)}),- remoteHasSourceHome: remoteCommand.includes(${JSON.stringify(home)}),- remoteHasSourceMachineHome: remoteCommand.includes(${JSON.stringify(machineHome)}), ...extra }) + '\\n');-/** Wait for the concrete setup marker and allow a finished worker to cancel its wait. */-const waitForMarker = path => {- let cancel = () => {};- const promise = new Promise((resolve, reject) => {- if (existsSync(path)) return resolve(true);- let poll, timer;- let settled = false;- const finish = (error, found = true) => {- if (settled) return;- settled = true;- clearInterval(poll); clearTimeout(timer); error ? reject(error) : resolve(found);- };- cancel = () => finish(null, false);- const check = () => { if (existsSync(path)) finish(); };- poll = setInterval(check, 20); // budget: observe actual setup readiness without relying on notifications.- timer = setTimeout(() => finish(new Error('setup marker hang ceiling exceeded')), 20_000); // budget: unchanged SSH-to-setup handshake ceiling.- check();- });- return { promise, cancel: () => cancel() };-};-let cancelDropWait = () => {};-if (host !== ${JSON.stringify(HOST)} && host !== ${JSON.stringify(`${HOST}-drop`)} && host !== ${JSON.stringify(`${HOST}-setup`)}) {- record({ refused: true, stdinBytes }); process.exit(255);-}-if (starts) appendFileSync(starts, JSON.stringify({ call, host }) + '\\n');-const phases = requests + '.phases';-const hold = ${JSON.stringify(hold)};-const holdHere = () => new Promise(() => { setInterval(() => {}, 60_000); });-if (hold === 'dispatch') await holdHere();-appendFileSync(phases, 'dispatched\\n');-if (hold === 'run') await holdHere();-${dropFirst ? `if (host.endsWith('-drop') && call === 1) {- process.stdin.once('data', () => {- record({ dropped: true, stdinBytes, stdinSha256: stdinHash.digest('hex') }); process.exit(255);- });- process.stdin.resume();-} else` : ''} {- const env = { PATH: process.env.PATH, HOME: home, TMPDIR: process.env.TMPDIR,- PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome, LANG: 'C', LC_ALL: 'C', TZ: 'UTC',- ...(${JSON.stringify(nodeOptions)} ? { NODE_OPTIONS: ${JSON.stringify(nodeOptions)} } : {}) };- // Replace the marker-driven shell so the injected HUP reaches the actual worker process.- const child = spawn('/bin/sh', ['-c', remoteCommand], { cwd: worker, env,- stdio: ['pipe', 'pipe', 'pipe'] });- child.stdout.pipe(process.stdout);- child.stderr.pipe(process.stderr);-${stderrLog ? ` child.stderr.on('data', part => appendFileSync(${JSON.stringify(stderrLog)}, part));\n` : ''} process.stdin.pipe(child.stdin);- let droppedAfterSetup = false;- if (dropAfterMarker) {- const waiting = waitForMarker(dropAfterMarker);- cancelDropWait = waiting.cancel;- waiting.promise.then(found => {- if (found === false) return;- droppedAfterSetup = true;- process.kill(child.pid, 'SIGHUP');- }, error => { record({ setupMarkerError: error.message }); child.kill('SIGKILL'); });- }- child.once('error', error => { record({ spawnError: error.code ?? error.name, stdinBytes }); process.exitCode = 255; });- child.once('close', (code, signal) => {- appendFileSync(phases, 'returned\\n');- if (hold === 'return') setInterval(() => {}, 60_000);- cancelDropWait();- record({ workerStarted: true, code, signal, stdinBytes, stdinSha256: stdinHash.digest('hex'),- droppedAfterSetup,- secretInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_RUNNER_FIXTURE_SECRET'),- localPoolMarkerInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_GATE_SLOT_POOL'),- localHeldMarkerInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_GATE_SLOT_HELD'),- localTokenMarkerInLocalSshEnv: Object.hasOwn(process.env, 'PULLBOARD_GATE_SLOT_TOKEN'),- localMachineHomeInSshEnv: Object.hasOwn(process.env, 'PULLBOARD_MACHINE_HOME') });- process.exitCode = signal ? 128 : (code ?? 255);- });-}-`;- writeFileSync(ssh, source, { mode: 0o700 });- chmodSync(ssh, 0o700);- return ssh;-}-/** Build one committed probe that observes the actual checkout and remote process environment. */function writeProbe(repo, { remoteStateRoot, remoteMachineHome, sourceHome, sourceMachineHome, callerPool, callerToken,historyBaseCommit, annotatedTagObject, lightweightTagObject }) {@@ -2027,3 +1909,8 @@ test('review655 a registered runner honors the same live person waiver at landinif (priorMachine === undefined) delete process.env.PULLBOARD_MACHINE_HOME; else process.env.PULLBOARD_MACHINE_HOME = priorMachine;}});+test('a refused runner checkout names its failed Git step and stderr [C6,C7]', async t => {+ for (const mode of ['lock', 'missing', 'large', 'filesystem', 'index']) {+ await t.test(mode, child => sourceSetupFailureFixture(child, { cli, git, runThroughGate, gateResult }, mode));+ }+});