Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(runners): batch bounded current-source scans before dispatch [C7]

Corey Olson3909965fe9d3
diff --git a/src/runners.js b/src/runners.js
index 1dc26df6..a86f6a21 100644
--- a/src/runners.js
+++ b/src/runners.js
@@ -115,20 +115,63 @@ function sourceObjects(root, source) {
const configRecord = records.find(record => record.endsWith('\tpullboard.json'));
const configObject = configRecord?.match(/^\d+ blob ([a-f0-9]{40,64})\t/u)?.[1];
const config = configObject ? configFromSource(sourceGit(root, ['cat-file', 'blob', configObject])) : defaults();
- for (const record of records) {
+ const blobs = records.map(record => {
const match = /^(\d+) (\w+) ([a-f0-9]{40,64})\t([\s\S]+)$/u.exec(record);
if (!match || match[2] !== 'blob') throw new Refused('RUNNER_SOURCE', 'runner source contains a submodule or unsupported object; keep this run local');
const [, , , object, path] = match;
if (blockedPaths([path], config.protect).length || /(?:^|\/)board\.sqlite(?:-(?:wal|shm))?$/u.test(path)) {
throw new Refused('RUNNER_SECRET', 'committed runner source includes a blocked private file; remove it before remote execution');
}
- const size = Number(sourceGit(root, ['cat-file', '-s', object]).trim());
- if (!Number.isSafeInteger(size) || size > MAX_BLOB_BYTES) {
- throw new Refused('RUNNER_SCAN_LIMIT', 'a committed blob exceeds the safe runner scan limit; keep the run local or reduce that file');
- }
- const text = sourceGit(root, ['cat-file', 'blob', object]);
- const findings = secretsIn(text.split(/\r?\n/u).map((line, index) => ({ path, line: index + 1, text: line })));
- if (findings.length) throw new Refused('RUNNER_SECRET', `source leak scan refused ${findings[0]}; remove the secret before remote execution`);
+ return { object, path };
+ });
+ // Hundreds of per-blob Git launches used almost the entire dispatch budget under load.
+ // Check every size first, then stream the same pinned bodies through one bounded record reader.
+ if (blobs.length) {
+ const input = blobs.map(({ object }) => object + '\n').join('');
+ const metadata = sourceGit(root, ['cat-file', '--batch-check'], { input }).trimEnd().split('\n');
+ if (metadata.length !== blobs.length) throw new Refused('RUNNER_SOURCE', 'incomplete current source inventory; retry from the committed repository');
+ const sizes = metadata.map((line, index) => {
+ const match = /^([a-f0-9]{40,64}) blob (\d+)$/u.exec(line);
+ if (!match || match[1] !== blobs[index].object) throw new Refused('RUNNER_SOURCE', 'invalid current source inventory; retry from the committed repository');
+ const size = Number(match[2]);
+ if (!Number.isSafeInteger(size) || size > MAX_BLOB_BYTES) {
+ throw new Refused('RUNNER_SCAN_LIMIT', 'a committed blob exceeds the safe runner scan limit; keep the run local or reduce that file');
+ }
+ return size;
+ });
+ const scratch = mkdtempSync(join(tmpdir(), 'pullboard-runner-source-'));
+ try {
+ const scan = join(scratch, 'current.scan');
+ for (let start = 0; start < blobs.length;) {
+ let end = start, bytes = 0;
+ // Bound scratch usage too: an otherwise valid large tree must not fill the temp volume.
+ do {
+ bytes += sizes[end] + blobs[end].object.length + 32;
+ end++;
+ } while (end < blobs.length && bytes + sizes[end] + blobs[end].object.length + 32 <= MAX_BLOB_BYTES);
+ const output = openSync(scan, 'wx', 0o600);
+ try { sourceGit(root, ['cat-file', '--batch'], {
+ input: blobs.slice(start, end).map(({ object }) => object + '\n').join(''), stdio: ['pipe', output, 'pipe'],
+ }); } finally { closeSync(output); }
+ const descriptor = openSync(scan, 'r');
+ try {
+ for (let index = start; index < end; index++) {
+ const { object, path } = blobs[index];
+ const record = objectRecord(descriptor);
+ if (record.object !== object || record.type !== 'blob' || record.data.length !== sizes[index]) {
+ throw new Refused('RUNNER_SOURCE', 'current source record differs from its inventory; retry from the committed repository');
+ }
+ const findings = secretsIn(record.data.toString('utf8').split(/\r?\n/u)
+ .map((text, line) => ({ path, line: line + 1, text })));
+ if (findings.length) throw new Refused('RUNNER_SECRET', `source leak scan refused ${findings[0]}; remove the secret before remote execution`);
+ }
+ if (readSync(descriptor, Buffer.alloc(1), 0, 1, null) !== 0) {
+ throw new Refused('RUNNER_SOURCE', 'unexpected current source records; retry from the committed repository');
+ }
+ } finally { closeSync(descriptor); rmSync(scan, { force: true }); }
+ start = end;
+ }
+ } finally { rmSync(scratch, { recursive: true, force: true }); }
}
const commit = sourceGit(root, ['cat-file', 'commit', source.commit]);
if (secretsIn(commit.split('\n').map((text, index) => ({ path: 'commit', line: index + 1, text }))).length) {
diff --git a/test/runner-source-refusal-fixture.js b/test/runner-source-refusal-fixture.js
new file mode 100644
index 00000000..3e5cf79b
--- /dev/null
+++ b/test/runner-source-refusal-fixture.js
@@ -0,0 +1,95 @@
+/** Exercise source privacy and shape refusals before a runner transport can start [C7,S1]. */
+import assert from 'node:assert/strict';
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { runFixtureChild } from './fixture-child.js';
+import { shellQuote } from './runner-ssh-fixture.js';
+
+/** Run isolated Git commands and keep fixture setup failures actionable. */
+function git(root, env, ...args) {
+ const result = runFixtureChild('git', args, { cwd: root, env, encoding: 'utf8' });
+ assert.equal(result.status, 0, result.context);
+ return result.stdout.trim();
+}
+
+/** Create one committed source tree with the requested private or unsupported entry. */
+function sourceCase(root, env, kind) {
+ git(root, env, 'init', '-q', '-b', 'main');
+ writeFileSync(join(root, 'a-safe.txt'), 'ordinary committed source\n');
+ git(root, env, 'add', 'a-safe.txt');
+ git(root, env, 'commit', '-q', '-m', 'test: safe source base');
+ if (kind === 'later-secret') {
+ writeFileSync(join(root, 'b-empty.bin'), Buffer.alloc(0));
+ writeFileSync(join(root, 'c-binary.bin'), Buffer.from([0, 255, 10, 128]));
+ writeFileSync(join(root, 'm-at-limit.bin'), Buffer.alloc(8 * 1024 * 1024));
+ writeFileSync(join(root, 'z-private.txt'), 'PULLBOARD_FIXTURE_TOKEN=aaaaaaaaaaaaaaaaaaaaaaaa\n');
+ } else if (kind === 'blocked-env') {
+ writeFileSync(join(root, '.env'), 'ordinary fixture value\n');
+ git(root, env, 'add', '-f', '.env');
+ } else if (kind === 'oversized-blob') {
+ writeFileSync(join(root, 'large.txt'), Buffer.alloc(8 * 1024 * 1024 + 1, 0x61));
+ } else if (kind === 'gitlink') {
+ const moduleRoot = join(root, '..', 'gitlink-source');
+ mkdirSync(moduleRoot, { recursive: true });
+ git(moduleRoot, env, 'init', '-q', '-b', 'main');
+ writeFileSync(join(moduleRoot, 'module.txt'), 'committed submodule fixture\n');
+ git(moduleRoot, env, 'add', 'module.txt');
+ git(moduleRoot, env, 'commit', '-q', '-m', 'test: gitlink fixture source');
+ git(root, env, '-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', moduleRoot, 'nested-module');
+ }
+ git(root, env, 'add', '-A');
+ git(root, env, 'commit', '-q', '-m', 'test: committed runner source refusal fixture');
+}
+
+/** Ask the production runner API to prepare the committed source in a subprocess. */
+function invokeRunner(root, env, runnerModule) {
+ const moduleUrl = pathToFileURL(runnerModule).href;
+ const script = `import { runOnRunnerAsync } from ${JSON.stringify(moduleUrl)};
+try {
+ await runOnRunnerAsync(process.argv[1], 'true', { runner: { host: 'fixture.invalid', slots: 1, browser: 0 } });
+ process.stdout.write(JSON.stringify({ code: 'NO_REFUSAL' }));
+} catch (error) {
+ process.stdout.write(JSON.stringify({ code: error.code ?? error.name }));
+}
+`;
+ const result = runFixtureChild(process.execPath, ['--input-type=module', '-e', script, root], {
+ cwd: root, env, encoding: 'utf8', timeout: 30_000, // Budget: scan the 8 MiB boundary fixture within a bounded child lifetime.
+ });
+ assert.equal(result.status, 0, result.context);
+ return JSON.parse(result.stdout);
+}
+
+/** Prove four committed-source refusals happen before SSH, using a fresh real Git repository each. */
+export async function runnerSourceRefusalCases(t, runnerModule = resolve(import.meta.dirname, '../src/runners.js')) {
+ const scratch = mkdtempSync(join(tmpdir(), 'pullboard-runner-source-refusal-'));
+ t.after(() => rmSync(scratch, { recursive: true, force: true }));
+ const cases = [
+ ['later-secret', 'RUNNER_SECRET'],
+ ['blocked-env', 'RUNNER_SECRET'],
+ ['oversized-blob', 'RUNNER_SCAN_LIMIT'],
+ ['gitlink', 'RUNNER_SOURCE'],
+ ];
+ for (const [kind, expected] of cases) {
+ const root = join(scratch, kind), bin = join(scratch, `${kind}-bin`), marker = join(scratch, `${kind}-ssh-invoked`);
+ mkdirSync(root, { recursive: true }); mkdirSync(bin, { recursive: true });
+ const ssh = join(bin, 'ssh');
+ writeFileSync(ssh, `#!/bin/sh\nprintf invoked > ${shellQuote(marker)}\nexit 97\n`);
+ chmodSync(ssh, 0o755);
+ const env = {
+ ...process.env,
+ PATH: `${bin}:${process.env.PATH ?? ''}`,
+ GIT_CONFIG_GLOBAL: '/dev/null',
+ GIT_CONFIG_NOSYSTEM: '1',
+ GIT_AUTHOR_NAME: 'Runner Privacy Fixture',
+ GIT_AUTHOR_EMAIL: '[email protected]',
+ GIT_COMMITTER_NAME: 'Runner Privacy Fixture',
+ GIT_COMMITTER_EMAIL: '[email protected]',
+ };
+ for (const name of ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR', 'GIT_CONFIG_PARAMETERS']) delete env[name];
+ sourceCase(root, env, kind);
+ assert.deepEqual(invokeRunner(root, env, runnerModule), { code: expected }, `${kind} refuses before SSH`);
+ assert.equal(existsSync(marker), false, `${kind} never invokes the SSH stand-in`);
+ }
+}
diff --git a/test/runners.test.js b/test/runners.test.js
index a06bb6af..5f6f5e6e 100644
--- a/test/runners.test.js
+++ b/test/runners.test.js
@@ -20,6 +20,7 @@ import { createE2eHelpers } from './e2e-helpers.js';
import { refuseSandboxBlocked } from '../src/gate.js';
import { SECRET, HOST, shellQuote, installSshStandIn } from './runner-ssh-fixture.js';
import { sourceSetupFailureFixture } from './runner-source-setup-fixture.js';
+import { runnerSourceRefusalCases } from './runner-source-refusal-fixture.js';
/** Per-phase deadlines for one runner round trip; together they stay near the 60-second hang ceiling. */
const ROUND_TRIP_DEADLINES = { dispatch: 20_000, 'remote run': 30_000, 'result return': 10_000 }; // budget: a stuck runner fails by name in about a minute, far under the 660-second per-test limit.
@@ -92,11 +93,13 @@ function writeProbe(repo, { remoteStateRoot, remoteMachineHome, sourceHome, sour
historyBaseCommit, annotatedTagObject, lightweightTagObject }) {
const probe = `import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
-import { existsSync } from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
import { isAbsolute, join, relative, sep } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import { test } from 'node:test';
test('runner executed the committed source in its private checkout', () => {
+ assert.equal(readFileSync('empty.bin').length, 0, 'an empty blob preserves the next object boundary');
+ assert.deepEqual(readFileSync('binary.bin'), Buffer.from([0, 10, 255, 13, 0, 128]), 'binary framing preserves exact bytes');
const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
const tree = execFileSync('git', ['rev-parse', 'HEAD^{tree}'], { encoding: 'utf8' }).trim();
const shallow = execFileSync('git', ['rev-parse', '--is-shallow-repository'], { encoding: 'utf8' }).trim();
@@ -593,6 +596,8 @@ test('a private check never refuses its own remote lease [C7,V4]', async (t) =>
}
});
+test('current source refuses private, oversized and unsupported blobs before SSH [C7,S1]', runnerSourceRefusalCases);
+
test('a queued run executes on a registered runner and its result comes back [C7]', async (t) => {
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-')));
TEMP.push(dir);
@@ -631,9 +636,11 @@ test('a queued run executes on a registered runner and its result comes back [C7
assert.equal(initialized.status, 0, initialized.context);
writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'true' }));
writeFileSync(join(repo, 'tracked.txt'), 'committed runner source\n');
+ writeFileSync(join(repo, 'empty.bin'), Buffer.alloc(0));
+ writeFileSync(join(repo, 'binary.bin'), Buffer.from([0, 10, 255, 13, 0, 128]));
writeFileSync(join(repo, 'history-original.txt'), 'committed history witness\n');
writeFileSync(join(repo, '.gitignore'), '.env\n');
- git(repo, env, 'add', '.gitignore', 'pullboard.json', 'tracked.txt', 'history-original.txt');
+ git(repo, env, 'add', '.gitignore', 'pullboard.json', 'tracked.txt', 'history-original.txt', 'empty.bin', 'binary.bin');
git(repo, env, 'commit', '-q', '-m', 'test: prepare registered runner fixture');
const historyBaseCommit = git(repo, env, 'rev-parse', 'HEAD');
git(repo, env, 'tag', '-a', 'runner-release', '-m', 'runner release annotation witness', historyBaseCommit);