Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(resources): gate waiters retry a locked slot database [C7]

Corey Olsona4879084c042
diff --git a/bin/run-tests.js b/bin/run-tests.js
index 98b9d5bd..d6a1281a 100644
--- a/bin/run-tests.js
+++ b/bin/run-tests.js
@@ -1521,7 +1521,7 @@ async function main(args) {
}
} catch (error) {
process.stderr.write(`run-tests: ${error.message}\n`);
- process.exitCode = 1;
+ process.exitCode = error.neverRanExitCode ?? 1; // A run stopped before it started exits distinctly, never as a red.
return;
}
if (signal) process.kill(process.pid, signal);
diff --git a/src/resources.js b/src/resources.js
index 75f54ac2..ff8ce39b 100644
--- a/src/resources.js
+++ b/src/resources.js
@@ -1,5 +1,6 @@
/** Machine and repository queues persist leases; landings first, then arrival order, or by class (landing, review, submit, other) aging up over time where the machine opts in [Q1,Q2,Q3]. */
import {
+ existsSync,
mkdirSync,
realpathSync,
} from 'node:fs';
@@ -41,6 +42,13 @@ const HEARTBEAT_MS = 3_000;
const POLL_MS = 40;
const PROCESS_IDENTITY_CACHE_MS = 3_000;
const processIdentities = new Map();
+const SLOT_DB_BUSY_MS = 10_000;
+/** How long a queued run keeps retrying a locked slot database before it stops, never having run. */
+export const SLOT_DB_LOCK_DEADLINE_MS = 300_000;
+/** Overrides that deadline in milliseconds; tests use it to reach the deadline quickly. */
+export const SLOT_DB_LOCK_ENV = 'PULLBOARD_SLOT_DB_LOCK_MS';
+/** The exit status of a run that never started because the slot database stayed locked (EX_TEMPFAIL). */
+export const NEVER_RAN_EXIT_CODE = 75;
/** Mark child commands whose parent already owns a machine gate slot. */
export const GATE_SLOT_HELD_ENV = 'PULLBOARD_GATE_SLOT_HELD';
@@ -104,7 +112,7 @@ function open(file, machine = false) {
function openQueue(file) {
mkdirSync(dirname(file), { recursive: true });
const db = new DatabaseSync(file);
- db.exec('PRAGMA busy_timeout = 10000');
+ db.exec(`PRAGMA busy_timeout = ${Math.min(SLOT_DB_BUSY_MS, slotLockDeadlineMs())}`);
let transaction = false;
try {
db.exec('BEGIN IMMEDIATE');
@@ -158,6 +166,56 @@ function openQueue(file) {
return db;
}
+/** The configured slot-database lock deadline, or the default when unset or invalid. */
+function slotLockDeadlineMs() {
+ const value = Number(process.env[SLOT_DB_LOCK_ENV]);
+ return Number.isSafeInteger(value) && value > 0 ? value : SLOT_DB_LOCK_DEADLINE_MS;
+}
+
+/** Recognize SQLite's busy or locked failure, which a later attempt can pass. */
+function slotDatabaseLocked(error) {
+ const code = Number.isInteger(error?.errcode) ? error.errcode & 0xff : null;
+ return code === 5 || code === 6 || /database is locked/iu.test(error?.message ?? '');
+}
+
+/** Name other processes with the database open; SQLite does not record which one holds its lock. */
+function slotDatabaseOpeners(file) {
+ const lsof = ['/usr/sbin/lsof', '/usr/bin/lsof'].find(path => existsSync(path));
+ if (!lsof) return [];
+ let output = '';
+ try { output = execFileSync(lsof, ['-F', 'pc', '--', file], { encoding: 'utf8', timeout: 2_000, stdio: ['ignore', 'pipe', 'ignore'] }); }
+ catch (error) { output = typeof error.stdout === 'string' ? error.stdout : ''; }
+ const openers = [];
+ for (const line of output.split('\n')) {
+ if (line.startsWith('p')) openers.push({ pid: Number(line.slice(1)), command: '' });
+ else if (line.startsWith('c') && openers.length) openers.at(-1).command = line.slice(1);
+ }
+ return openers.filter(({ pid }) => Number.isSafeInteger(pid) && pid !== process.pid);
+}
+
+/** Retry a slot-database step while another process holds the lock, with a bounded backoff, until the deadline. */
+async function retryLockedSlotDatabase(file, work) {
+ const deadline = slotLockDeadlineMs();
+ const startedAt = Date.now();
+ let backoff = POLL_MS;
+ while (true) {
+ try { return work(); }
+ catch (error) {
+ if (!slotDatabaseLocked(error)) throw error;
+ const elapsed = Date.now() - startedAt;
+ if (elapsed >= deadline) {
+ const openers = slotDatabaseOpeners(file);
+ const holder = openers.length ? `held by one of: ${openers.map(({ pid, command }) => `pid ${pid}${command ? ` (${command})` : ''}`).join(', ')}` : 'lock holder unknown';
+ const stopped = new Refused('SLOT_DB_LOCKED', `never ran: the gate slot database ${file} stayed locked for ${elapsed}ms, past the ${deadline}ms deadline (${holder}). The run did not start, so this is not a test failure; rerun it.`);
+ stopped.neverRanExitCode = NEVER_RAN_EXIT_CODE;
+ throw stopped;
+ }
+ await pause(Math.min(backoff, deadline - elapsed));
+ backoff = Math.min(backoff * 2, 2_000);
+ }
+ }
+}
+
/** Check whether the operating system still recognizes this process ID. */
function alive(pid) {
try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; }
@@ -630,7 +688,8 @@ export async function takeResource(options) {
if (shortEligible && (name !== 'gate' || scope !== 'machine' || landing || diagnostic)) throw new Refused('BAD_SHORT_SLOT_REQUEST', 'short gate slots are only available to named non-landing, non-diagnostic machine gate runs');
if (diagnostic && (landing || itemCheck)) throw new Refused('BAD_RESOURCE_PRIORITY', 'a diagnostic slot cannot also be a landing or an item check');
const file = databaseFile(scope, root);
- const db = open(file, scope === 'machine');
+ // A queued run waits out a locked database in place; only a lock past the deadline stops it, as never ran.
+ const db = await retryLockedSlotDatabase(file, () => open(file, scope === 'machine'));
let pool;
try { pool = canonicalResourceFile(file); }
catch (error) { db.close(); throw error; }
@@ -653,14 +712,14 @@ export async function takeResource(options) {
throw new Refused('BAD_RESOURCE_METADATA', 'the waiting item renewal must return a valid leaseUntil timestamp');
}
leaseUntil = result.leaseUntil;
- const changed = update(db, () => db.prepare(`UPDATE ${table} SET lease_until = ? WHERE token = ? AND pid = ? AND started = ?`)
- .run(leaseUntil, token, pid, started));
+ const changed = await retryLockedSlotDatabase(file, () => update(db, () => db.prepare(`UPDATE ${table} SET lease_until = ? WHERE token = ? AND pid = ? AND started = ?`)
+ .run(leaseUntil, token, pid, started)));
if (changed.changes !== 1) throw new Refused('RESOURCE_LEASE_LOST', 'the queued resource lease disappeared during item renewal; retry the command');
}
};
try {
let initialTicket = resume?.ticket;
- update(db, () => {
+ await retryLockedSlotDatabase(file, () => update(db, () => {
const currentCapacity = options.capacityProvider?.() ?? capacity;
const currentShortCapacity = options.shortCapacityProvider?.() ?? 0;
if (!Number.isSafeInteger(currentCapacity) || currentCapacity < 1) throw new Refused('BAD_RESOURCE_CAPACITY', 'resource capacity must be a positive integer');
@@ -696,7 +755,7 @@ export async function takeResource(options) {
const inserted = db.prepare('INSERT INTO waiter(token, name, pid, started, agent, repo, since, heartbeat, landing, item_check, diagnostic, browser, item, kind, remote, classed, release_aware, board_key, release_rank, queue_ticket, short_eligible) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').run(token, name, pid, started, agent, repo, queuedAt, Date.now(), Number(landing), Number(itemCheck || diagnostic), Number(diagnostic), Number(browser), item, kind, Number(remote), classedMarker(queueOrder(scope)), 1, boardKey, releaseRank, resume?.queueTicket ?? null, Number(shortEligible));
initialTicket = Number(inserted.lastInsertRowid);
}
- });
+ }));
let memoryWarned = false;
while (true) {
const memory = options.memory ? await options.memory() : null;
@@ -706,7 +765,7 @@ export async function takeResource(options) {
await keepItemLease({ holders: [], line: [], position: 0, memory }, 'waiter');
}
const order = queueOrder(scope);
- const acquired = update(db, () => {
+ const acquired = await retryLockedSlotDatabase(file, () => update(db, () => {
const now = Date.now();
// The machine-settings provider performs a bounded local JSON read. Read it
// after taking the same SQLite write lock used by updateRunnerSettings so an
@@ -742,7 +801,7 @@ export async function takeResource(options) {
return { acquired: true, state: { ...spot, position: 0 }, ticket: waiter.ticket, queueTicket, runner: executor.host ? executor : undefined, shortSlot: executor.shortSlot === true };
}
return { acquired: false, state: { ...spot, position: reportPosition } };
- });
+ }));
if (acquired.acquired) {
// Only item-lease callbacks opt into admission. Await them so no renewal remains in
// flight during the run, and restart the ordinary item expiry from this boundary.
diff --git a/test/resources.test.js b/test/resources.test.js
index 744d6c08..9f58e507 100644
--- a/test/resources.test.js
+++ b/test/resources.test.js
@@ -1774,6 +1774,54 @@ test('[Q4] a red gate releases its lease before the same process starts another
});
+/** A write lock on the slot database delays a queued run; only a lock past the deadline stops it, as never ran [C7]. */
+test('a gate-slot waiter retries a locked slot database [C7]', async () => {
+ const box = fixture();
+ mkdirSync(box.home, { recursive: true });
+ const released = join(box.dir, 'lock-released');
+ // The waiter's test passes only if the lock was already released when it ran.
+ writeFileSync(join(box.first, 'ran.test.js'), `import assert from 'node:assert/strict';\nimport { existsSync } from 'node:fs';\nimport { test } from 'node:test';\ntest('the waiter ran', () => assert.ok(existsSync(${JSON.stringify(released)})));\n`);
+ const env = { ...box.env, PULLBOARD_SLOT_DB_LOCK_MS: '6000' };
+ for (const key of ['PULLBOARD_GATE_SLOT_HELD', 'PULLBOARD_GATE_SLOT_POOL', 'PULLBOARD_GATE_SLOT_TOKEN', 'PULLBOARD_BROWSER_SLOT_HELD']) delete env[key];
+ /** Hold a write lock on the machine slot database from another process for `ms`; wrapped so awaiting it waits only for the lock. */
+ const holdLock = async (ms) => {
+ const marker = join(box.dir, `locked-${ms}`);
+ const locker = runFixtureChildAsync(process.execPath, ['--input-type=module', '-e', `
+ import { DatabaseSync } from 'node:sqlite';
+ import { writeFileSync } from 'node:fs';
+ const [file, marker, ms, released] = process.argv.slice(1);
+ const db = new DatabaseSync(file);
+ db.exec('BEGIN IMMEDIATE');
+ db.exec('CREATE TABLE IF NOT EXISTS lock_probe (x)');
+ writeFileSync(marker, 'locked');
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Number(ms));
+ db.exec('COMMIT');
+ writeFileSync(released, 'released');`, join(box.home, 'resources.sqlite'), marker, String(ms), released], { env });
+ await waitForMarker(marker);
+ return { done: locker, pid: locker.child.pid, stop: () => locker.child.kill('SIGKILL') };
+ };
+ /** Run one named test file through the real runner, which waits for a machine gate slot. */
+ const runWaiter = () => runFixtureChildAsync(process.execPath, [TEST_RUNNER, 'ran.test.js'], { cwd: box.first, env });
+
+ const shortLock = await holdLock(3_000);
+ const kept = await runWaiter();
+ assert.equal((await shortLock.done).status, 0, 'the 3 s lock was taken and released');
+ assert.equal(kept.status, 0, `${kept.context}\n${kept.stdout}`);
+ assert.match(kept.stdout, /ok \d+ - the waiter ran/u);
+ assert.match(kept.stdout, /^# fail 0$/mu);
+
+ rmSync(released);
+ const longLock = await holdLock(60_000);
+ try {
+ const stopped = await runWaiter();
+ assert.equal(stopped.status, 75, stopped.context);
+ assert.match(stopped.stderr, /never ran/u);
+ assert.match(stopped.stderr, /not a test failure/u);
+ assert.match(stopped.stderr, new RegExp(`pid ${longLock.pid}\\b`, 'u'), 'names the process holding the lock');
+ assert.doesNotMatch(stopped.stdout, /the waiter ran/u);
+ } finally { longLock.stop(); await longLock.done; }
+});
+
const capacityCases = capacityChangeCases({ fixture, worker, event, acquired, release, privateList, poolWorker, POOL_MODULE_WORKER, configureBrowserSlots });
/** Existing queued processes observe a raised limit without restarting their holders [Q1,Q2,Q4]. */