Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(pool): apply memory holds only to local executors [C7]
Corey Olson86558fb106b1
diff --git a/src/commands/resources.js b/src/commands/resources.jsindex a105f5b7..d0e4f39a 100644--- a/src/commands/resources.js+++ b/src/commands/resources.js@@ -43,7 +43,7 @@ export async function run(io) {for (const resource of resources) {io.say(`${resource.name} (${resource.scope}, capacity ${resource.capacity})${resource.order === 'arrival' ? '; an older pullboard is waiting, so this line is in arrival order, landings first' : ''}`);for (const holder of resource.holders) io.say(` held by ${agentDisplayName(agents, ctx.config, holder.agent)}${holder.repo ? ` in ${holder.repo}` : ''}${resourceRunLabel(holder, shortSlots)} since ${holder.since}`);- resource.line.forEach((waiter, index) => io.say(` ${index + 1}. waiting: ${agentDisplayName(agents, ctx.config, waiter.agent)}${waiter.repo ? ` in ${waiter.repo}` : ''}${resourceRunLabel(waiter, shortSlots)}; class ${waiter.queueClass}${waiter.agedFrom ? `, aged up from ${waiter.agedFrom}` : ''}`));+ resource.line.forEach((waiter, index) => io.say(` ${index + 1}. waiting: ${agentDisplayName(agents, ctx.config, waiter.agent)}${waiter.repo ? ` in ${waiter.repo}` : ''}${resourceRunLabel(waiter, shortSlots)}; class ${waiter.queueClass}${waiter.agedFrom ? `, aged up from ${waiter.agedFrom}` : ''}${waiter.memoryHeld ? '; held by local memory' : ''}`));}for (const runner of runners) io.say(`runner ${runner.host}: ${runner.inUse}/${runner.slots} slots, ${runner.browserInUse}/${runner.browser} browser, GUI ${runner.gui === true ? 'yes' : runner.gui === false ? 'no' : 'unknown'}${runner.unavailable ? ' (unavailable; re-register after checking the host)' : ''}`);return 0;diff --git a/src/commands/status.js b/src/commands/status.jsindex 9e5bb7e9..7a4e0db8 100644--- a/src/commands/status.js+++ b/src/commands/status.js@@ -6,6 +6,7 @@ import { identifyCaller } from '../cli-read-context.js';import { relayStateExists } from '../local-state.js';import { unlinkedRelayStatus } from '../relay-status-fields.js';import { ruleProofCounts } from '../rule-proof-counts.js';+import { memoryHeldRuns } from '../memory-held.js';import {reviewQueueLine,} from '../next-steps.js';@@ -41,6 +42,7 @@ export async function run(io, { values }) {return { me, mine, ruleCounts: ruleProofCounts(ctx.info.root, board, approvalState.spec), needs: approvalState.needs, specApprovalBaseline: approvalState.specApprovalBaseline, specApprovalEvidenceErrors: approvalState.specApprovalEvidenceErrors, stats: store.stats(board), reviewQueue: { ...store.reviewQueue(board), ...store.reviewQueueBreakdown(board) }, unread: store.unreadCount(board, me.id), relay: relay ? relay.relayStatus(ctx.info.root) : unlinkedRelayStatus(),submitInvariantProposals: submitInvariantProposals(store.submitInvariantEvents(board, ctx.config), ctx.config) };});+ summary.memoryHeld = memoryHeldRuns();if (values.json) {io.result(summary);return 0;@@ -53,6 +55,8 @@ export async function run(io, { values }) {if (summary.specApprovalBaseline?.pending) io.say('approval baseline pending: the next coordinator or person command in the main checkout records the existing approved rows; lane work may continue and none are flagged until then');if (summary.specApprovalBaseline?.invalid) io.say('approval baseline unreadable: existing rows need approval evidence; restore a known-good board export to recover the original baseline');io.say(reviewQueueLine(ctx, summary.reviewQueue, span));+ if (summary.memoryHeld === null) io.say('local memory hold status unavailable; pullboard resources shows the queue');+ else for (const run of summary.memoryHeld) io.say(`local memory holds ${run.agent}${run.item ? ` #${run.item}` : ''}${run.kind ? ` (${run.kind})` : ''}${run.repo ? ` in ${run.repo}` : ''}`);for (const proposal of summary.submitInvariantProposals) io.say(submitInvariantLine(proposal));for (const need of summary.needs.filter(need => need.kind === 'agent-approved')) io.say(`${need.text}; ${need.what}`);if (!summary.relay.linked && !originRepository(ctx.info.root, { optional: true })) {diff --git a/src/gate.js b/src/gate.jsindex dcfb9be3..095ae14a 100644--- a/src/gate.js+++ b/src/gate.js@@ -355,7 +355,7 @@ function runTiming(run, command) {}/** Run work under the machine queue: landings first, then every other eligible waiter in arrival order.- * The whole queue waits while the machine is out of memory, so a run never starts into swap.+ * Local executors wait while this machine is out of memory; remote runners keep admitting.** @param {string} root - Checkout used to locate the machine resource database.* shortEligible marks a run that names its test files: with gate.shortSlots above 0 it may take adiff --git a/src/memory-held.js b/src/memory-held.jsnew file mode 100644index 00000000..633ed1a9--- /dev/null+++ b/src/memory-held.js@@ -0,0 +1,24 @@+/** Read current local-memory queue holds without opening or migrating the resource database [C7]. */+import { existsSync } from 'node:fs';+import { join } from 'node:path';+import { DatabaseSync } from 'node:sqlite';+import { machineHome } from './machine-home.js';+import { readMachineSettingsFile } from './queue-settings.js';++/** An unset floor keeps status on its existing lightweight path; old pools have no per-run holds. */+export function memoryHeldRuns() {+ if (readMachineSettingsFile()?.gate?.memoryFloorMB === undefined) return [];+ const file = join(machineHome(), 'resources.sqlite');+ if (!existsSync(file)) return [];+ let db;+ try {+ db = new DatabaseSync(file, { readOnly: true });+ if (!db.prepare('PRAGMA table_info(waiter)').all().some(column => column.name === 'memory_held')) return [];+ return db.prepare("SELECT agent, repo, item, kind, since FROM waiter WHERE name='gate' AND memory_held=1 AND heartbeat >= ? ORDER BY ticket")+ .all(Date.now() - 20_000);+ } catch (error) {+ if (['EACCES', 'EPERM', 'SQLITE_BUSY', 'SQLITE_CANTOPEN'].includes(error.code)+ || error.code === 'ERR_SQLITE_ERROR' && [5, 14].includes(error.errcode & 0xff)) return null;+ throw error;+ } finally { db?.close(); }+}diff --git a/src/resources.js b/src/resources.jsindex cf10982e..cf8fb938 100644--- a/src/resources.js+++ b/src/resources.js@@ -146,6 +146,7 @@ function openQueue(file) {if (table === 'waiter' && !columns.includes('gui')) db.exec('ALTER TABLE waiter ADD COLUMN gui INTEGER NOT NULL DEFAULT 0');if (table === 'waiter' && !columns.includes('remote')) db.exec('ALTER TABLE waiter ADD COLUMN remote INTEGER NOT NULL DEFAULT 0');if (table === 'waiter' && !columns.includes('short_eligible')) db.exec('ALTER TABLE waiter ADD COLUMN short_eligible INTEGER NOT NULL DEFAULT 0');+ if (table === 'waiter' && !columns.includes('memory_held')) db.exec('ALTER TABLE waiter ADD COLUMN memory_held INTEGER NOT NULL DEFAULT 0');}// Older clients insert the default zero. A database trigger witnesses even short legacy// arrivals between the timing owner's polls; no old client has to understand the new flag.@@ -345,18 +346,18 @@ function queueOrder(scope) {/** Read the complete resource line with the metadata every admission view orders identically. */function queuedWaiters(db, name) {- return db.prepare('SELECT ticket, pid, agent, repo, since, landing, item_check AS itemCheck, diagnostic, browser, gui, remote, item, kind, lease_until AS leaseUntil, classed, release_aware, board_key, release_rank, queue_ticket, short_eligible AS shortEligible, timing FROM waiter WHERE name = ?').all(name);+ return db.prepare('SELECT ticket, pid, agent, repo, since, landing, item_check AS itemCheck, diagnostic, browser, gui, remote, item, kind, lease_until AS leaseUntil, classed, release_aware, board_key, release_rank, queue_ticket, short_eligible AS shortEligible, timing, memory_held AS memoryHeld FROM waiter WHERE name = ?').all(name);}/** Current holders and the waiting line in admission order, each waiter naming its class (aged only while class order is on). */function snapshot(db, name, order = ARRIVAL_ORDER, now = Date.now()) {/** Preserve older public entries while naming diagnostics as a distinct queue class. */- const project = ({ diagnostic, browser, gui, remote, runner, item, kind, leaseUntil, shortSlot, shortEligible, timing, ...entry }) => ({ ...entry, landing: entry.landing === 1,+ const project = ({ diagnostic, browser, gui, remote, runner, item, kind, leaseUntil, shortSlot, shortEligible, timing, memoryHeld, ...entry }) => ({ ...entry, landing: entry.landing === 1,itemCheck: entry.itemCheck === 1 && diagnostic !== 1, ...(diagnostic === 1 ? { diagnostic: true } : {}),...(['gate', 'browser'].includes(name) && shortSlot === 1 ? { slot: 'short' } : {}),...(['gate', 'browser'].includes(name) && shortSlot === 0 ? { slot: 'gate' } : {}),...(['gate', 'browser'].includes(name) && shortEligible === 1 ? { requestedSlot: 'short-eligible' } : {}),- ...(timing === 1 ? { timing: true } : {}), ...(browser === 1 ? { browser: true } : {}), ...(gui === 1 ? { gui: true } : {}), ...(remote === 1 ? { remote: true } : {}),+ ...(memoryHeld === 1 ? { memoryHeld: true } : {}), ...(timing === 1 ? { timing: true } : {}), ...(browser === 1 ? { browser: true } : {}), ...(gui === 1 ? { gui: true } : {}), ...(remote === 1 ? { remote: true } : {}),...(runner ? { runner } : {}), ...(item ? { item } : {}), ...(kind ? { kind } : {}), ...(leaseUntil ? { leaseUntil } : {}) });// Browser is an additional capability on a gate lease, not a second partially held row.const where = name === 'browser' ? 'name = ? AND browser = 1' : 'name = ?';@@ -458,7 +459,7 @@ function validatedRunners(value) {* @param {number} [now] - The clock reading to age against.* @returns {{host:string,slots:number,browser:number}|null}*/-function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirst = false, order = ARRIVAL_ORDER, now = Date.now()) {+function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirst = false, order = ARRIVAL_ORDER, now = Date.now(), memoryHeld = false) {const waiters = orderedWaiters(queuedWaiters(db, 'gate'), now, order);const holders = db.prepare("SELECT pid, landing, browser, short_slot AS shortSlot, timing FROM holder WHERE name = 'gate'").all();const localUsed = holders.filter(holder => holder.shortSlot !== 1).length;@@ -489,9 +490,7 @@ function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirsconst start = Math.max(0, runners.findIndex(runner => runner.host === cursor));const ordered = runnersFirst ? [...runners.slice(start), ...runners.slice(0, start)] : runners;const compatible = ordered.filter(runner => {- const observed = db.prepare('SELECT gui FROM runner_gui WHERE host = ?').get(runner.host);- const gui = observed ? observed.gui === 1 : runner.gui === true;- if (unavailable.has(runner.host) || (waiter.browser === 1 && runner.browser < 1) || (waiter.gui === 1 && !gui)) return false;+ if (!runnerCompatible(db, runner, waiter, unavailable)) return false;const used = usage.get(runner.host);return !used.timing && (waiter.timing !== 1 || used.slots === 0)&& used.slots < runner.slots && (waiter.browser !== 1 || used.browsers < runner.browser);@@ -505,6 +504,7 @@ function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirs}/** The local slot follows runner admission only while runner-first is enabled. */function localSlot() {+ if (memoryHeld) return null;if (!holders.some(holder => holder.timing === 1) && (waiter.timing !== 1 || holders.length === 0)&& localUsed < capacity && (waiter.browser !== 1 || localBrowsers < browserCapacity)) {return { host: '', slots: capacity, browser: waiter.browser === 1 ? 1 : 0, shortSlot: false };@@ -522,7 +522,7 @@ function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirsif (remote) return remote;}const earlierLanding = waiters.some(other => other.landing === 1 && other.ticket < waiter.ticket);- if (waiter.timing !== 1 && !holders.some(holder => holder.timing === 1)+ if (!memoryHeld && waiter.timing !== 1 && !holders.some(holder => holder.timing === 1)&& waiter.shortEligible === 1 && !earlierLanding && localShortUsed < shortCapacity&& (waiter.browser !== 1 || localBrowsers < browserCapacity)) {return { host: '', slots: shortCapacity, browser: waiter.browser === 1 ? 1 : 0, shortSlot: true };@@ -554,6 +554,13 @@ function gateAdmission(db, ticket, capacity, shortCapacity, runners, runnersFirsreturn null;}+/** Capability/availability permits a remote executor even when every compatible slot is currently busy. */+function runnerCompatible(db, runner, waiter, unavailable) {+ const observed = db.prepare('SELECT gui FROM runner_gui WHERE host = ?').get(runner.host);+ const gui = observed ? observed.gui === 1 : runner.gui === true;+ return !unavailable.has(runner.host) && (waiter.browser !== 1 || runner.browser >= 1) && (waiter.gui !== 1 || gui);+}+/** List configured runners with active slot usage from the same local admission database. */export function runnerResourceStatus(runners) {const configured = validatedRunners(runners);@@ -731,7 +738,7 @@ export function memoryVerdict(reading, floorBytes) {const pressure = reading.pressure === 'unknown' ? 'pressure not reported' : `pressure ${reading.pressure}`;const state = `${pressure}, ${memorySize(reading.availableBytes)} free of ${memorySize(reading.totalBytes)}${off ? '' : ` (floor ${memorySize(floorBytes)})`}`;const biggest = (reading.processes ?? []).map(({ pid, name, bytes, cwd, cwdFailure }) => `pid ${pid} ${name} ${memorySize(bytes)} in ${cwd || cwdFailure || 'an unreadable directory'}`).join('; ');- const verdict = off ? 'hold off (set gate.memoryFloorMB to turn it on)' : held ? `holding the gate queue because ${why}` : 'admitting';+ const verdict = off ? 'hold off (set gate.memoryFloorMB to turn it on)' : held ? `holding local gate runs because ${why}` : 'admitting';return { checked: true, held, line: `memory: ${verdict}: ${state}${biggest ? `; biggest: ${biggest}` : ''}` };}@@ -761,7 +768,7 @@ function pause(ms) { return new Promise((resolvePromise) => setTimeout(resolvePr* Join the queue in its admission class (landing, review, submit, other), aging up over time; FIFO by ticket within a class.** @param {{ name: string, capacity: number, capacityProvider?: () => number, shortCapacityProvider?: () => number, browserCapacityProvider?: () => number, scope?: 'machine'|'repo'|'board', root?: string, agent?: string, repo?: string, landing?: boolean, itemCheck?: boolean, shortEligible?: boolean, diagnostic?: boolean, browser?: boolean, remote?: boolean, timing?: boolean, runnerProvider?: () => Array<{host:string,slots:number,browser:number}>, item?: string, kind?: string, onWait?: (state: object) => any | Promise<any>, allowIdleCapacityUpdate?: boolean, memory?: () => Promise<{ checked: boolean, held: boolean, line: string, warning?: string }>, _resume?: object }} options - Resource identity, queue class, short-slot eligibility, optional remote executor pool and memory check. An awaited wait callback may return {leaseUntil} and then receives a final {admitted:true} callback before work starts.- * memory, when given, is read before each admission: a held verdict keeps the whole queue+ * memory, when given, is read before each admission: a held verdict blocks local executors* waiting in place, local, remote and short slots alike, and an unchecked one is passed to onWait once* so the caller can warn.* @returns {Promise<{ name: string, scope: string, token: string, pool: string, browser: boolean, slotClass: 'gate'|'short', runner?: {host:string,slots:number,browser:number}, waitMs: number, release: () => void, renew: () => void, requeueRunner?: (reason?: string, options?: { local?: boolean }) => Promise<object> }>}@@ -899,7 +906,7 @@ export async function takeResource(options) {// older pre-lock snapshot cannot overwrite a newer configured capacity.const runners = runnerProvider ? validatedRunners(runnerProvider()) : [];const runnersFirst = runnersFirstProvider?.() === true;- const waiter = db.prepare('SELECT ticket FROM waiter WHERE token = ?').get(token);+ const waiter = db.prepare('SELECT ticket, remote, browser, gui FROM waiter WHERE token = ?').get(token);if (!waiter) {if (resume && initialTicket) {db.prepare('INSERT OR IGNORE INTO waiter(ticket, token, name, pid, started, agent, repo, since, heartbeat, landing, item_check, diagnostic, browser, item, kind, remote, classed, release_aware, board_key, release_rank, queue_ticket, short_eligible, gui, timing, timing_aware) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').run(initialTicket, token, name, pid, started, agent, repo, queuedAt, Date.now(), Number(landing), Number(itemCheck || diagnostic), Number(diagnostic), Number(browser), item, kind, Number(remote), classedMarker(order), 1, boardKey, releaseRank, resume?.queueTicket ?? null, Number(shortEligible), Number(gui), Number(timing), 1);@@ -912,13 +919,17 @@ export async function takeResource(options) {}// The marker follows this poll's setting, so a change mid-wait reaches every process's ranking.db.prepare('UPDATE waiter SET heartbeat = ?, classed = ? WHERE token = ?').run(Date.now(), classedMarker(order), token);+ const unavailable = new Set(db.prepare('SELECT host FROM runner_unavailable UNION SELECT host FROM runner_draining').all().map(row => row.host));+ const memoryHeld = Boolean(memory?.held && (name !== 'gate' || waiter.remote !== 1+ || !runners.some(runner => runnerCompatible(db, runner, waiter, unavailable))));+ db.prepare('UPDATE waiter SET memory_held = ? WHERE token = ?').run(Number(memoryHeld), token);const spot = snapshot(db, name, order, now);const reportPosition = waiterPosition(db, name, waiter.ticket, { order, now, shortCapacity: options.shortCapacityProvider?.() ?? 0 });const limit = db.prepare('SELECT capacity FROM resource WHERE name = ?').get(name).capacity;- // A machine short of memory holds the whole line in place; recovery admits it in order.- if (memory?.held) return { acquired: false, state: { ...spot, position: reportPosition, memory } };+ // Local pressure blocks local executors, including short slots, but never a remote executor.+ if (name !== 'gate' && memoryHeld) return { acquired: false, state: { ...spot, position: reportPosition, memory } };const executor = name === 'gate'- ? gateAdmission(db, waiter.ticket, limit, options.shortCapacityProvider?.() ?? 0, runners, runnersFirst, order, now)+ ? gateAdmission(db, waiter.ticket, limit, options.shortCapacityProvider?.() ?? 0, runners, runnersFirst, order, now, memory?.held === true): (waiterPosition(db, name, waiter.ticket, { eligibleOnly: true, order, now }) === 1 && spot.holders.length < limit ? { host: '' } : null);if (executor) {const queueTicket = consumeOpportunity(db, name, waiter.ticket, order, now);@@ -928,7 +939,7 @@ export async function takeResource(options) {db.prepare('INSERT INTO holder(token, name, pid, started, agent, repo, since, heartbeat, landing, item_check, diagnostic, browser, item, kind, runner, short_slot, timing, timing_aware) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').run(token, holderName, pid, started, agent, repo, acquiredAt, Date.now(), Number(landing), Number(itemCheck || diagnostic), Number(diagnostic), Number(browser), item, kind, executor.host, Number(executor.shortSlot === true), Number(timing), 1);return { acquired: true, state: { ...spot, position: 0 }, ticket: waiter.ticket, queueTicket, runner: executor.host ? executor : undefined, shortSlot: executor.shortSlot === true, capacity: limit, admittedAt: acquiredAt };}- return { acquired: false, state: { ...spot, position: reportPosition } };+ return { acquired: false, state: { ...spot, position: reportPosition, ...(memoryHeld ? { memory } : {}) } };}));if (acquired.acquired) {// Only item-lease callbacks opt into admission. Await them so no renewal remains indiff --git a/test/resources.test.js b/test/resources.test.jsindex 38883fb1..5ea4242c 100644--- a/test/resources.test.js+++ b/test/resources.test.js@@ -1518,7 +1518,7 @@ test('the pool holds work while the machine is out of memory and names the biggeassert.equal(critical.waiting, 1, 'the held run keeps its place at the head of the line');assert.equal(critical.memory?.held, true, `the run is held for memory: ${JSON.stringify(critical)}`);assert.equal(critical.memory.line,- `memory: holding the gate queue because pressure is critical: pressure critical, 8.0 GB free of 36.0 GB (floor 512 MB); ${biggest}`,+ `memory: holding local gate runs because pressure is critical: pressure critical, 8.0 GB free of 36.0 GB (floor 512 MB); ${biggest}`,'one line says why and names the three biggest processes with pid, size and working directory');assert.doesNotMatch(critical.memory.line, /\n/u, 'the hold is one line');const gate = privateList(box).find(({ name }) => name === 'gate');@@ -1531,7 +1531,7 @@ test('the pool holds work while the machine is out of memory and names the biggeconst low = await event(run);assert.equal(low.memory?.held, true, `free memory under the settings floor still holds: ${JSON.stringify(low)}`);assert.equal(low.memory.line,- `memory: holding the gate queue because free memory is under the floor: pressure normal, 400 MB free of 36.0 GB (floor 512 MB); ${biggest}`,+ `memory: holding local gate runs because free memory is under the floor: pressure normal, 400 MB free of 36.0 GB (floor 512 MB); ${biggest}`,'the floor comes from gate.memoryFloorMB in pullboard settings');writeMemoryReading(reading, { availableMB: 8192 });