Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

perf(gate): waiting files read, and time files from admission [C7]

Corey Olsondb13d691a537
diff --git a/bin/run-tests.js b/bin/run-tests.js
index 4ddf1aac..f3e76f3a 100644
--- a/bin/run-tests.js
+++ b/bin/run-tests.js
@@ -1575,13 +1575,15 @@ if (process.argv[1] && resolve(process.argv[1]) === runner) {
process.env.PULLBOARD_TEST_FILE = process.argv[1];
(await import('../src/test-action-cache.js')).recordTestAction(process.argv[1]); // #1045: reuse off below; record this file.
const fileTimes = process.env[LANDING_FILE_TIMES_ENV];
+ let tokenWaitMs = 0;
// Only this test file reports; anything it starts is not one of the shard's files.
delete process.env[LANDING_FILE_TIMES_ENV];
if (fileTimes && process.env.NODE_TEST_CONTEXT === 'child-v8') {
// One line per test file process: its wall time from start to exit, for the next shard plan.
const timedFile = relative(process.cwd(), resolve(process.argv[1]));
process.once('exit', () => {
- try { appendFileSync(fileTimes, `${JSON.stringify({ file: timedFile, ms: Math.round(process.uptime() * 1000) })}\n`); }
+ // The file's own time, not its wait for a token: the next shard plan weighs work, not this run's queue (#1046).
+ try { appendFileSync(fileTimes, `${JSON.stringify({ file: timedFile, ms: Math.round(process.uptime() * 1000 - tokenWaitMs) })}\n`); }
catch { /* A missing time only weakens the next shard plan. */ }
});
}
@@ -1636,6 +1638,6 @@ if (process.argv[1] && resolve(process.argv[1]) === runner) {
}
}
// After any browser wait, so a file never holds a core while it queues for a browser (#1046).
- if (process.env.NODE_TEST_CONTEXT === 'child-v8') await takeTestFileToken();
+ if (process.env.NODE_TEST_CONTEXT === 'child-v8') tokenWaitMs = await takeTestFileToken();
if (!landingWorker) installTestWatchdog();
}
diff --git a/src/test-file-pool.js b/src/test-file-pool.js
index 865e782e..56242fba 100644
--- a/src/test-file-pool.js
+++ b/src/test-file-pool.js
@@ -19,8 +19,10 @@ import { machineHome } from './machine-home.js';
export const TEST_FILE_POOL_ENV = 'PULLBOARD_TEST_FILE_POOL';
/** Overrides the pool size; a test models a smaller machine with it. */
export const TEST_FILE_CORES_ENV = 'PULLBOARD_TEST_FILE_CORES';
-// Budget: a waiting file looks again this often; admission is one short transaction.
-const POLL_MS = 20;
+// Budget: a waiting file looks again this often with one read; it takes the write lock only when a place looks
+// free, or once a second to prune rows of processes that have gone.
+const POLL_MS = 25;
+const PRUNE_MS = 1_000;
/**
* Read the pool a test-file process belongs to, or null outside a run-tests run.
@@ -73,6 +75,8 @@ async function openPool(home) {
const DatabaseSync = await databaseSync();
const db = new DatabaseSync(join(home, 'test-file-pool.sqlite'));
db.exec(`PRAGMA busy_timeout = 10000;
+ PRAGMA journal_mode = WAL;
+ PRAGMA synchronous = NORMAL;
CREATE TABLE IF NOT EXISTS file_holder(pid INTEGER PRIMARY KEY, since INTEGER NOT NULL);
CREATE TABLE IF NOT EXISTS file_waiter(ticket INTEGER PRIMARY KEY AUTOINCREMENT, pid INTEGER NOT NULL UNIQUE);
CREATE TABLE IF NOT EXISTS file_loan(runner INTEGER PRIMARY KEY, lender INTEGER NOT NULL);`);
@@ -111,6 +115,15 @@ function prune(db) {
}
}
+/** How many files are running and how many waiters are ahead of this one, read without a write lock. */
+function standing(db) {
+ const holders = db.prepare('SELECT count(*) AS n FROM file_holder').get().n;
+ const lending = db.prepare('SELECT count(DISTINCT lender) AS n FROM file_loan WHERE lender IN (SELECT pid FROM file_holder)').get().n;
+ const mine = db.prepare('SELECT ticket FROM file_waiter WHERE pid = ?').get(process.pid)?.ticket;
+ const ahead = db.prepare('SELECT count(*) AS n FROM file_waiter WHERE ticket < ?').get(mine).n;
+ return { running: holders - lending, ahead };
+}
+
/**
* Admit this process when a core is free and every earlier waiter fits first. Running files are holders less the
* holders lending to a live nested run.
@@ -118,12 +131,8 @@ function prune(db) {
function admit(db, cores) {
return immediate(db, () => {
prune(db);
- const holders = db.prepare('SELECT count(*) AS n FROM file_holder').get().n;
- const lending = db.prepare('SELECT count(DISTINCT lender) AS n FROM file_loan WHERE lender IN (SELECT pid FROM file_holder)').get().n;
- const free = cores - (holders - lending);
- const mine = db.prepare('SELECT ticket FROM file_waiter WHERE pid = ?').get(process.pid)?.ticket;
- const ahead = db.prepare('SELECT count(*) AS n FROM file_waiter WHERE ticket < ?').get(mine).n;
- if (ahead >= free) return false;
+ const { running, ahead } = standing(db);
+ if (ahead >= cores - running) return false;
db.prepare('DELETE FROM file_waiter WHERE pid = ?').run(process.pid);
db.prepare('INSERT OR REPLACE INTO file_holder(pid, since) VALUES (?, ?)').run(process.pid, Date.now());
return true;
@@ -135,19 +144,30 @@ function admit(db, cores) {
* of anything it starts.
*
* @param {NodeJS.ProcessEnv} env - Changed in place: descendants see this file as their lender.
- * @returns {Promise<void>}
+ * @returns {Promise<number>} How long this file waited for its token, in ms.
*/
export async function takeTestFileToken(env = process.env) {
const pool = testFilePool(env);
- if (!pool) return;
+ if (!pool) return 0;
+ const asked = Date.now();
const db = await openPool(pool.home);
immediate(db, () => db.prepare('INSERT OR IGNORE INTO file_waiter(pid) VALUES (?)').run(process.pid));
- while (!admit(db, pool.cores)) await new Promise(done => setTimeout(done, POLL_MS));
+ let pruned = 0;
+ while (true) {
+ const { running, ahead } = standing(db);
+ // A dead holder only goes in the write path, so look there at least once a second even when nothing looks free.
+ if (ahead < pool.cores - running || Date.now() - pruned >= PRUNE_MS) {
+ pruned = Date.now();
+ if (admit(db, pool.cores)) break;
+ }
+ await new Promise(done => setTimeout(done, POLL_MS));
+ }
process.once('exit', () => {
try { db.prepare('DELETE FROM file_holder WHERE pid = ?').run(process.pid); db.close(); }
catch { /* A row left behind is pruned once this process is gone. */ }
});
env[TEST_FILE_POOL_ENV] = JSON.stringify({ home: pool.home, cores: pool.cores, lender: process.pid });
+ return Date.now() - asked;
}
/**