Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(board): hot board queries use indexes, never a full scan [C7]

Corey Olsond12a73752c6a
diff --git a/src/board-hot.js b/src/board-hot.js
new file mode 100644
index 00000000..090dbec4
--- /dev/null
+++ b/src/board-hot.js
@@ -0,0 +1,86 @@
+/**
+ * The indexes and bounded reads behind the board's hot queries [C7]. Status, list, show, next, resume,
+ * health and the view read events and shouts by kind, sender, answer and recency; on a board of tens of
+ * thousands of rows each of those was a full scan (openDecisions 796 ms, health's agent activity 11 s).
+ */
+
+/** Each index a hot query needs; IF NOT EXISTS lets an existing board gain them on its next open. */
+const HOT_INDEXES = [
+ 'CREATE INDEX IF NOT EXISTS shout_answers ON shout (shout_answers)',
+ 'CREATE INDEX IF NOT EXISTS shout_open_decision ON shout (shout_decision, shout_to) WHERE shout_decision = 1',
+ 'CREATE INDEX IF NOT EXISTS shout_open_request ON shout (shout_request) WHERE shout_request = 1',
+ 'CREATE INDEX IF NOT EXISTS shout_evidence_item ON shout (shout_evidence_item) WHERE shout_evidence_item IS NOT NULL',
+ 'CREATE INDEX IF NOT EXISTS shout_from_at ON shout (shout_from, shout_at)',
+ 'CREATE INDEX IF NOT EXISTS event_by_at ON event (event_by, event_at)',
+ 'CREATE INDEX IF NOT EXISTS event_kind_id ON event (event_kind, event_id)',
+];
+
+/**
+ * Create the hot-query indexes. They index columns that migrate() adds to older boards, so this runs
+ * after it, never inside the table schema.
+ *
+ * @param {import('node:sqlite').DatabaseSync} db
+ */
+export function ensureHotIndexes(db) {
+ for (const sql of HOT_INDEXES) db.exec(sql);
+}
+
+/**
+ * Events of the given kinds, oldest first, read through the kind index instead of the whole log.
+ *
+ * @param {any} board
+ * @param {string[]} kinds
+ * @returns {any[]}
+ */
+export function eventsOfKinds(board, kinds) {
+ if (!kinds.length) return [];
+ return board.db.prepare(`SELECT * FROM event WHERE event_kind IN (${kinds.map(() => '?').join(', ')}) ORDER BY event_id`).all(...kinds);
+}
+
+/**
+ * The newest events, newest first. The id bound states the range read, so the plan seeks from the
+ * end of the log rather than scanning it.
+ *
+ * @param {any} board
+ * @param {number} limit
+ * @returns {any[]}
+ */
+export function recentEvents(board, limit) {
+ return board.db.prepare('SELECT * FROM event WHERE event_id > 0 ORDER BY event_id DESC LIMIT ?').all(limit);
+}
+
+/**
+ * The newest shouts on the board, newest first, read without marking anything read (N26), seeking
+ * from the end of the table the same way as recentEvents.
+ *
+ * @param {any} board
+ * @param {number} [limit]
+ * @returns {any[]}
+ */
+export function recentShouts(board, limit = 40) {
+ return board.db.prepare('SELECT * FROM shout WHERE shout_id > 0 ORDER BY shout_id DESC LIMIT ?').all(limit);
+}
+
+/**
+ * When each agent last moved on the board, one index seek per agent.
+ *
+ * @param {any} board
+ * @returns {Map<string, string>}
+ */
+export function lastMoves(board) {
+ const rows = board.db.prepare('SELECT agent_id, (SELECT MAX(event_at) FROM event WHERE event_by = agent.agent_id) AS at FROM agent').all();
+ return new Map(rows.filter((row) => row.at).map((row) => [row.agent_id, row.at]));
+}
+
+/**
+ * The shouts the coordinator's resume reads: every decision ask and every integration evidence shout,
+ * oldest first, through their partial indexes rather than the whole shout table. A UNION, because
+ * SQLite uses no partial index for one arm of an OR, and the order outside it, or SQLite scans for it.
+ *
+ * @param {any} board
+ * @returns {any[]}
+ */
+export function decisionAndEvidenceShouts(board) {
+ return board.db.prepare(`SELECT * FROM (SELECT * FROM shout WHERE shout_decision = 1
+ UNION SELECT * FROM shout WHERE shout_evidence_item > 0) ORDER BY shout_id`).all();
+}
diff --git a/src/board.js b/src/board.js
index 0b6717bc..d9ff0f97 100644
--- a/src/board.js
+++ b/src/board.js
@@ -24,6 +24,7 @@ import {
import { dirname } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import { boardNotWritable } from './board-access.js';
+import { ensureHotIndexes } from './board-hot.js';
import {
briefFiles,
briefSections,
@@ -315,6 +316,8 @@ export function openBoard(file, clock = systemClock, { isMain } = {}) {
if (file !== ':memory:') db.exec('PRAGMA journal_mode = WAL');
migrateEventLogVersion(db);
migrate(db);
+ // Only trunk code adds indexes to a shared board; a worktree reads it as the trunk left it (#1044).
+ if (isMain !== false) ensureHotIndexes(db);
db.prepare('INSERT OR IGNORE INTO board_meta (meta_key, meta_value) VALUES (?, ?)').run('board_id', randomBytes(16).toString('hex'));
const board = { db, clock };
guardStore(board);
@@ -2662,16 +2665,7 @@ export function peekShouts(board, agentId, limit = 2) {
.reverse();
}
-/**
- * The newest shouts on the board, newest first, read without marking anything read (N26).
- *
- * @param {any} board
- * @param {number} [limit]
- * @returns {any[]}
- */
-export function recentShouts(board, limit = 40) {
- return board.db.prepare('SELECT * FROM shout ORDER BY shout_id DESC LIMIT ?').all(limit);
-}
+export { recentShouts, eventsOfKinds, recentEvents, lastMoves, decisionAndEvidenceShouts } from './board-hot.js';
export { stats } from './board-stats.js';
diff --git a/src/cli.js b/src/cli.js
index 7a902224..b1ba2c8a 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -1452,7 +1452,7 @@ export function verifiedMergeGroups(root, items, shouts, now) {
/** Capture coordinator-only decision sections with stable ages and answers from the board. */
export function coordinatorDecisionSections(board, ctx) {
- const shouts = store.allShouts(board);
+ const shouts = store.decisionAndEvidenceShouts(board);
const openDecisions = store.openDecisions(board, COORDINATOR).map((ask) => ({
id: ask.shout_id, asker: ask.shout_from, question: ask.shout_text, at: ask.shout_at, age: span(ctx, ask.shout_at),
}));
diff --git a/src/commands/status.js b/src/commands/status.js
index bcafd7f2..cbc93481 100644
--- a/src/commands/status.js
+++ b/src/commands/status.js
@@ -35,7 +35,7 @@ export function run(io, { values }) {
const mine = store.listItems(board).filter((item) => item.item_status === 'claimed' && item.item_owner === me.id);
const approvalState = personNeedsSummary(ctx.info.root, { board, config: ctx.config });
return { me, mine, needs: approvalState.needs, specApprovalBaseline: approvalState.specApprovalBaseline, specApprovalEvidenceErrors: approvalState.specApprovalEvidenceErrors, stats: store.stats(board), reviewQueue: { ...store.reviewQueue(board), ...store.reviewQueueBreakdown(board) }, unread: store.unreadCount(board, me.id), relay: relayStatus(ctx.info.root),
- submitInvariantProposals: submitInvariantProposals(store.events(board), ctx.config) };
+ submitInvariantProposals: submitInvariantProposals(store.eventsOfKinds(board, ['submit', 'landing_batch']), ctx.config) };
});
if (values.json) {
io.result(summary);
diff --git a/src/roadmap.js b/src/roadmap.js
index 4e046fcf..179e373c 100644
--- a/src/roadmap.js
+++ b/src/roadmap.js
@@ -7,7 +7,7 @@ import {
import {
closeBoard,
getItem,
- events,
+ eventsOfKinds,
laneHolds,
listItems,
milestones,
@@ -126,8 +126,7 @@ export function milestoneRoadmap(root, board, options = {}) {
/** Count distinct currently landed local items at their first recording of the current merge. */
function landingPace(board, items, now) {
const first = new Map();
- for (const event of events(board)) {
- if (event.event_kind !== 'merged') continue;
+ for (const event of eventsOfKinds(board, ['merged'])) {
const item = items.get(event.item_id);
if (item?.item_status !== 'verified' || !item.item_merged_commit || first.has(event.item_id)) continue;
if (JSON.parse(event.event_detail).commit !== item.item_merged_commit) continue;
diff --git a/test/query-plans.test.js b/test/query-plans.test.js
new file mode 100644
index 00000000..37898ed9
--- /dev/null
+++ b/test/query-plans.test.js
@@ -0,0 +1,205 @@
+/** @pullboard-test-input source-tree */
+/**
+ * Every query the hot commands run reaches events and shouts through an index, never a full scan, on a
+ * seeded board of 1,000 items, 45,000 events and 23,000 shouts [C7]. The commands run as the real CLI
+ * with a preload that records each statement they prepare; each is then planned with EXPLAIN QUERY PLAN.
+ */
+import assert from 'node:assert/strict';
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { DatabaseSync } from 'node:sqlite';
+import { after, test } from 'node:test';
+import * as store from '../src/board.js';
+import { runFixtureChild, runFixtureGit } from './fixture-child.js';
+
+const BIN = resolve(import.meta.dirname, '../bin/pullboard.js');
+const SERVE = pathToFileURL(resolve(import.meta.dirname, '../src/serve.js')).href;
+const ITEMS = 1_000;
+const EVENTS = 45_000;
+const SHOUTS = 23_000;
+const SEATS = 800;
+const sandboxes = [];
+// Reads that fold the whole history by design, so no index can serve them, each with the item that
+// rewrites it: health's itemHistory (#1053), coordinator resume's undrafted Rigor in personNeedsState
+// (#1052), resume's proofStats (#1051) and the view state's projectState (#1055, which uses
+// lastMoves and recentEvents). The test fails if one stops scanning while still listed, so the list
+// can only shrink.
+const WHOLE_HISTORY = ['itemHistory', 'personNeedsState', 'projectState', 'proofStats'];
+
+after(() => {
+ for (const dir of sandboxes) rmSync(dir, { recursive: true, force: true });
+});
+
+/** Make a real repo with a coordinator and one joined builder, and return how to run the CLI there with a statement log. */
+function fixture() {
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-query-plans-')));
+ sandboxes.push(dir);
+ const log = join(dir, 'statements.log');
+ const preload = join(dir, 'record.mjs');
+ // Record every statement any connection prepares, so the test plans exactly what the command ran.
+ writeFileSync(preload, `import { DatabaseSync } from 'node:sqlite';
+import { appendFileSync } from 'node:fs';
+const prepare = DatabaseSync.prototype.prepare;
+DatabaseSync.prototype.prepare = function (sql, ...rest) {
+ // The function that asked: past the events() helper and serve's withProject wrapper.
+ const caller = new Error().stack.split('\\n').slice(2).map((line) => /at (?:Module\\.)?([\\w$.]+) \\(/u.exec(line)?.[1])
+ .find((name) => name && name !== 'events' && name !== 'withProject');
+ appendFileSync(process.env.PULLBOARD_QUERY_LOG, JSON.stringify({ sql, caller }) + '\\n');
+ return prepare.call(this, sql, ...rest);
+};
+`);
+ const shims = join(dir, 'bin');
+ mkdirSync(shims);
+ writeFileSync(join(shims, 'pullboard'), `#!/bin/sh\nexec "${process.execPath}" "${BIN}" "$@"\n`, { mode: 0o755 });
+ chmodSync(join(shims, 'pullboard'), 0o755);
+ const env = {
+ ...process.env,
+ PATH: `${shims}:${process.env.PATH}`,
+ GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
+ GIT_AUTHOR_NAME: 'Test Agent', GIT_AUTHOR_EMAIL: '[email protected]',
+ GIT_COMMITTER_NAME: 'Test Agent', GIT_COMMITTER_EMAIL: '[email protected]',
+ PULLBOARD_HOME: join(dir, 'home'), PULLBOARD_MACHINE_HOME: join(dir, 'machine-home'),
+ PULLBOARD_MODEL: 'Test Model',
+ };
+ delete env.PULLBOARD_RELAY_TOKEN;
+ const repo = join(dir, 'repo');
+ mkdirSync(repo);
+ const git = (...args) => runFixtureGit(args, { cwd: repo, env });
+ /** Run the CLI, or a node script, in a checkout; `record` adds the statement-recording preload. */
+ const run = (cwd, args, { record = false, node = false } = {}) => {
+ const argv = [...(record ? ['--import', pathToFileURL(preload).href] : []), ...(node ? [] : [BIN]), ...args];
+ const result = runFixtureChild(process.execPath, argv, { cwd, env: { ...env, PULLBOARD_QUERY_LOG: log }, encoding: 'utf8' });
+ return { code: result.status, out: result.stdout ?? '', err: result.failure ?? result.stderr ?? '' };
+ };
+ git('init', '-q', '-b', 'main');
+ assert.equal(run(repo, ['init']).code, 0);
+ writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({
+ gate: 'true', spec: 'SPEC.md', verify: 'any', lease: '2h',
+ lanes: { web: { owns: ['web/'], specs: ['G1'] } },
+ }, null, 2));
+ writeFileSync(join(repo, 'SPEC.md'), '# Plans fixture\n\n## G · Goals\n- G1 [approved, must] The feature works. | gate: test\n');
+ mkdirSync(join(repo, 'web'));
+ writeFileSync(join(repo, 'web', 'a.txt'), 'base\n');
+ git('add', '-A');
+ git('commit', '-q', '-m', 'chore: create plans fixture');
+ const made = run(repo, ['worktree', 'web', '--route', 'strong', '--json']);
+ assert.equal(made.code, 0, `${made.err}\n${made.out}`);
+ const board = join(git('rev-parse', '--path-format=absolute', '--git-common-dir').trim(), 'pullboard', 'board.sqlite');
+ return { dir, repo, web: JSON.parse(made.out).path, board, log, run };
+}
+
+/** Fill the board to the measured scale: seats, items, an event log, shouts, and decisions half answered. */
+function seed(file) {
+ const db = new DatabaseSync(file);
+ const at = (n) => new Date(Date.UTC(2026, 9, 1) + n * 1000).toISOString();
+ db.exec('BEGIN');
+ const seat = db.prepare('INSERT INTO agent (agent_id, agent_lane, agent_path, agent_created_at) VALUES (?, ?, ?, ?)');
+ for (let n = 0; n < SEATS; n += 1) seat.run(`web-${n + 100}`, 'web', `/gone/web-${n + 100}`, at(n));
+ const item = db.prepare(`INSERT INTO item (item_lane, item_title, item_criterion, item_created_by, item_created_at, item_updated_at)
+ VALUES ('web', ?, 'The feature works.', 'coordinator', ?, ?)`);
+ for (let n = 0; n < ITEMS; n += 1) item.run(`Item ${n}`, at(n), at(n));
+ const event = db.prepare('INSERT INTO event (event_at, event_by, event_kind, item_id, event_detail) VALUES (?, ?, ?, ?, ?)');
+ const kinds = ['add', 'claim', 'renew', 'shout', 'edit'];
+ for (let n = 0; n < EVENTS; n += 1) event.run(at(n), `web-${100 + (n % SEATS)}`, kinds[n % kinds.length], n % 10 ? 1 + (n % ITEMS) : null, '{}');
+ const shout = db.prepare('INSERT INTO shout (shout_from, shout_to, shout_text, shout_at, shout_decision, shout_answers) VALUES (?, ?, ?, ?, ?, ?)');
+ for (let n = 0; n < SHOUTS; n += 1) {
+ const asks = n % 10 === 0;
+ const to = asks ? (n % 20 === 0 ? 'coordinator' : 'person') : 'all';
+ // Every other coordinator ask gets its answer two shouts later (shout ids start at 1).
+ const answers = n % 40 === 2 ? n - 1 : null;
+ shout.run(`web-${100 + (n % SEATS)}`, to, `shout ${n}`, at(n), asks ? 1 : 0, answers);
+ }
+ db.exec('COMMIT');
+ db.close();
+}
+
+/** Add each recorded statement to `statements`, with the commands and the functions that ran it. */
+function record(statements, log, command) {
+ for (const line of readFileSync(log, 'utf8').split('\n').filter(Boolean)) {
+ const { sql, caller } = JSON.parse(line);
+ const seen = statements.get(sql) ?? { commands: new Set(), callers: new Set() };
+ seen.commands.add(command);
+ seen.callers.add(caller ?? 'unknown');
+ statements.set(sql, seen);
+ }
+}
+
+/** The aliases a statement gives the event and shout tables, besides their own names. */
+function scanNames(sql) {
+ const names = new Set(['event', 'shout', 'ask', 'reply']);
+ for (const match of sql.matchAll(/\b(?:event|shout)\s+(?:AS\s+)?([A-Za-z_]\w*)/giu)) {
+ if (!/^(?:WHERE|ORDER|GROUP|LIMIT|JOIN|LEFT|INNER|ON|USING|SET|VALUES|AND|OR|WHEN|INDEXED|NOT)$/iu.test(match[1])) names.add(match[1]);
+ }
+ return names;
+}
+
+test('hot board queries never scan events or shouts [C7]', () => {
+ const box = fixture();
+ seed(box.board);
+ // The commands run worktree code, which never changes a shared board's schema, so the board gets
+ // its indexes the way the trunk's next open gives them.
+ store.closeBoard(store.openBoard(box.board));
+ const commands = [
+ [box.repo, ['status']], [box.repo, ['list']], [box.repo, ['show', '1']], [box.repo, ['next']],
+ [box.repo, ['resume']], [box.repo, ['health']],
+ [box.web, ['status']], [box.web, ['list']], [box.web, ['show', '1']], [box.web, ['next']], [box.web, ['resume']],
+ ];
+ const statements = new Map();
+ const timings = [];
+ for (const [cwd, args] of commands) {
+ writeFileSync(box.log, '');
+ const started = performance.now();
+ const result = box.run(cwd, args, { record: true });
+ timings.push(`${cwd === box.repo ? 'coordinator' : 'builder'} ${args[0]} ${Math.round(performance.now() - started)} ms`);
+ assert.ok(result.code === 0 || result.code === 1, `${args.join(' ')} crashed: ${result.err}`);
+ record(statements, box.log, args.join(' '));
+ }
+ // The view's state, as the page polls it.
+ writeFileSync(box.log, '');
+ const view = join(box.dir, 'view-state.mjs');
+ writeFileSync(view, `const { projectState } = await import(${JSON.stringify(SERVE)});\nprojectState(process.argv[2]);\n`);
+ const state = box.run(box.repo, [view, box.repo], { record: true, node: true });
+ assert.equal(state.code, 0, state.err);
+ record(statements, box.log, 'view state');
+ assert.ok(statements.size > 20, `only ${statements.size} statements were recorded`);
+
+ const board = store.openBoard(box.board);
+ try {
+ const scans = [];
+ const wholeHistory = new Set();
+ for (const [sql, { commands, callers }] of statements) {
+ if (!/\b(?:event|shout)\b/iu.test(sql)) continue;
+ const plan = board.db.prepare(`EXPLAIN QUERY PLAN ${sql}`).all().map((row) => row.detail);
+ const names = scanNames(sql);
+ // A full scan, or an automatic index, which SQLite builds by scanning the table on every run.
+ const scanned = plan.filter((detail) => {
+ const match = /^(?:SCAN (\w+)|SEARCH (\w+) USING AUTOMATIC)/u.exec(detail);
+ return match && names.has(match[1] ?? match[2]);
+ });
+ if (!scanned.length) continue;
+ if ([...callers].every((caller) => WHOLE_HISTORY.includes(caller))) {
+ for (const caller of callers) wholeHistory.add(caller);
+ continue;
+ }
+ scans.push(`${[...commands].join(', ')} (${[...callers].join(', ')}): ${sql.replace(/\s+/gu, ' ').slice(0, 200)}\n ${plan.join('\n ')}`);
+ }
+ assert.deepEqual(scans, [], `${scans.length} hot statements scan events or shouts:\n${scans.join('\n')}`);
+ assert.deepEqual([...wholeHistory].sort(), WHOLE_HISTORY, 'a whole-history read stopped scanning: remove it from WHOLE_HISTORY');
+
+ store.openDecisions(board, 'coordinator');
+ const runs = [];
+ for (let n = 0; n < 5; n += 1) {
+ const started = performance.now();
+ const open = store.openDecisions(board, 'coordinator');
+ runs.push(performance.now() - started);
+ assert.equal(open.length, 575);
+ }
+ const median = runs.sort((a, b) => a - b)[2];
+ assert.ok(median < 5, `openDecisions took ${median.toFixed(2)} ms, over 5 ms`);
+ console.log(`# openDecisions median ${median.toFixed(2)} ms; ${timings.join('; ')}`);
+ } finally {
+ store.closeBoard(board);
+ }
+});