Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(relay): idle commands ask the publisher to check the page [H5,H16]
Corey Olsonf2ff0057596c
diff --git a/src/relay-checkpoint-publish.js b/src/relay-checkpoint-publish.jsindex db582725..e82ab94c 100644--- a/src/relay-checkpoint-publish.js+++ b/src/relay-checkpoint-publish.js@@ -57,10 +57,12 @@ export function checkpointPublisher({ request, saveLink, loadLink, linkFile, rea* Decide what checkpoint to send and save its exact sealed record first; null when nothing is to be sent (no machine* credential, an upload pause not yet over, or a definitive rejection of this same image).*/- async function stageCheckpoint(root, file, state, io, prepared) {+ async function stageCheckpoint(root, file, state, io, prepared, { probe = true } = {}) {if (!state.token?.startsWith('pm_') || process.env.PULLBOARD_RELAY_TOKEN) return null;const resumed = Boolean(state.checkpointRefusal);if (state.checkpointRefusal && uploadPauseType(state.checkpointRefusal)) {+ // The background publisher probes before taking the lock; a pause that appeared since waits for its next pass.+ if (!probe) return null;if (!await uploadRetryReady(state, state.checkpointRefusal, path => request(state, path, {}, io))) {saveLink(file, state); reportCheckpointRefusal(state, io); return null;}@@ -109,23 +111,36 @@ export function checkpointPublisher({ request, saveLink, loadLink, linkFile, rea* before the relay lock, the lock is held only to save the sealed record and later to record the reply, and the* upload itself holds no lock. A board whose link is gone or being removed publishes nothing.*+ * With `ifChanged`, a board whose page matches the last acknowledged one, and that has no unsent checkpoint,+ * uploads nothing: a command's check request must not re-send an unchanged 6 MB page.+ ** @param {string} root* @param {{ err: (line: string) => void, say: (line: string) => void }} io- * @returns {Promise<boolean>} Whether a linked board was published.+ * @param {{ ifChanged?: boolean }} [options]+ * @returns {Promise<boolean>} Whether a linked board was published or found already current.*/- async function publishLatestCheckpoint(root, io) {+ async function publishLatestCheckpoint(root, io, { ifChanged = false } = {}) {const file = linkFile(root);const linked = loadLink(file);if (!linked || linked.unlinking) return false;+ // Without a machine credential nothing is uploaded, so the board is not read (stageCheckpoint decides the same).+ if (!linked.token?.startsWith('pm_') || process.env.PULLBOARD_RELAY_TOKEN) return false;+ // A pending recovery or an unacknowledged send is settled by a command's own sync first, never published around.+ if (linked.recovery || linked.pending) return false;+ const paused = Boolean(linked.checkpointRefusal && uploadPauseType(linked.checkpointRefusal));+ if (!await probeUploadPause(file, linked, io)) return false;// Reading and sealing a large board takes about a second; it happens before the lock, so moves never wait on it.const prepared = await prepareCheckpoint(root, linked);const staged = await locked(file, async () => {const state = loadLink(file);- if (!state || state.unlinking || state.board !== linked.board) return null;- const result = await stageCheckpoint(root, file, state, io, prepared);- return result && { ...result, sent: state.checkpoint, copy: structuredClone(state) };+ if (!state || state.unlinking || state.recovery || state.pending || state.board !== linked.board) return null;+ if (ifChanged && !state.checkpoint && !state.checkpointRefusal && state.presentationDigest === prepared.digest) return { current: true };+ const result = await stageCheckpoint(root, file, state, io, prepared, { probe: false });+ // A pause the probe ended counts as resumed, so its notices clear once this upload is acknowledged.+ return result && { ...result, resumed: result.resumed || paused, sent: state.checkpoint, copy: structuredClone(state) };}, io);if (!staged) return false;+ if (staged.current) return true;// The upload holds no lock, so a move during it returns at once and the next checkpoint covers it. It runs on a// copy of the link: the copy saves nothing, and what the upload learns is carried back under the lock below.let outcome;@@ -141,5 +156,31 @@ export function checkpointPublisher({ request, saveLink, loadLink, linkFile, reareturn true;}+ /**+ * Ask the relay whether a quota or size pause is over, holding no lock while the request is out: a slow probe must+ * never keep a move waiting. The probe runs on a copy of the link; its answer is applied under the lock only while+ * the link still carries the same pause. Returns whether an upload may go ahead.+ *+ * @param {string} file+ * @param {any} linked - The link as read before the probe.+ * @param {{ err: (line: string) => void }} io+ * @returns {Promise<boolean>}+ */+ async function probeUploadPause(file, linked, io) {+ if (!linked.checkpointRefusal || !uploadPauseType(linked.checkpointRefusal)) return true;+ const copy = structuredClone(linked);+ const ready = await uploadRetryReady(copy, copy.checkpointRefusal, path => request(copy, path, {}, io));+ return locked(file, async () => {+ const state = loadLink(file);+ if (!state || state.board !== linked.board || JSON.stringify(state.checkpointRefusal) !== JSON.stringify(linked.checkpointRefusal)) return false;+ if (copy.snapshotApiVersion !== undefined) state.snapshotApiVersion = copy.snapshotApiVersion;+ if (copy.storage !== undefined) state.storage = copy.storage;+ if (ready) delete state.checkpointRefusal;+ saveLink(file, state);+ if (!ready) reportCheckpointRefusal(state, io);+ return ready;+ }, io);+ }+return { publishCheckpoint, publishLatestCheckpoint };}diff --git a/src/relay-checkpoint-worker.js b/src/relay-checkpoint-worker.jsindex 3a003c56..7679789f 100644--- a/src/relay-checkpoint-worker.js+++ b/src/relay-checkpoint-worker.js@@ -21,16 +21,30 @@ import {import { publishLatestCheckpoint } from './relay.js';const [root, file] = process.argv.slice(2);-const { due, done, pid, log } = checkpointFiles(file);+const { due, done, asked, checked, pid, log } = checkpointFiles(file);-/** Read the time of the last move, or 0 when none was noted. */-function lastMove() {- try { return Number(readFileSync(due, 'utf8')) || 0; } catch { return 0; }+/** Read a time one of the publisher's files holds, or 0 when none was noted. */+function noted(path) {+ try { return Number(readFileSync(path, 'utf8')) || 0; } catch { return 0; }}+/** Read the time of the last move, or 0 when none was noted. */+function lastMove() { return noted(due); }+/** Read the move time the last upload covered, or 0. */-function lastCovered() {- try { return Number(readFileSync(done, 'utf8')) || 0; } catch { return 0; }+function lastCovered() { return noted(done); }++/** Whether a command asked for a presentation check after the board was last read for a checkpoint. */+function checkWanted() { return noted(asked) > noted(checked); }++/**+ * Read and seal the board, noting when it was read: a check asked before then is answered by this read. A read+ * that fails or finds nothing to send answers it too; the next command asks again, so this publisher never spins.+ */+async function publishFrom(options) {+ const reading = Date.now();+ try { return await publishLatestCheckpoint(root, io, options); }+ finally { writeFileSync(checked, `${reading}\n`, { mode: 0o600 }); }}/**@@ -48,6 +62,13 @@ async function claim() {}, io);}+// Budget: refresh the claim well inside its ten-minute staleness, even during one long read or upload, so a live+// publisher is never taken for a dead one and a second publisher never uploads beside it.+const HEARTBEAT_MS = 60_000;+setInterval(() => {+ try { if (Number(readFileSync(pid, 'utf8')) === process.pid) utimesSync(pid, new Date(), new Date()); } catch { /* No claim to refresh. */ }+}, HEARTBEAT_MS).unref();+/** Give up the claim under the relay lock, and only while it is still this process's own. */async function release() {await locked(file, () => {@@ -62,21 +83,31 @@ const io = {err(line) { try { appendFileSync(log, `${new Date().toISOString()} ${String(line).trim()}\n`, { mode: 0o600 }); } catch { /* No log is no worse than none. */ } },};-/** Publish until a checkpoint covers the last move; returns the time of the move it covered. */+/**+ * Publish until a checkpoint covers the last move and answers the last check request; returns the time of the move+ * it covered. Moves go first; a check reads the board once commands go quiet and uploads only a changed page.+ */async function publishUntilQuiet() {const quiet = checkpointQuietMs();for (;;) {// Refresh the claim, so a scheduler never takes this live publisher for a dead one.try { utimesSync(pid, new Date(), new Date()); } catch { /* A removed claim is re-made by the next pass. */ }const moved = lastMove();- // An earlier publisher already covered this move: nothing to send.- if (lastCovered() >= moved) return moved;- const wait = quiet - (Date.now() - moved);+ const movePending = lastCovered() < moved;+ // An earlier publisher already covered this move and answered every check: nothing to send.+ if (!movePending && !checkWanted()) return moved;+ const since = movePending ? moved : noted(asked);+ const wait = quiet - (Date.now() - since);if (wait > 0) { await new Promise(next => setTimeout(next, wait)); continue; }- io.say(`publishing for the move at ${new Date(moved).toISOString()}`);- if (await publishLatestCheckpoint(root, io)) writeFileSync(done, `${moved}\n`, { mode: 0o600 });- io.say('published');- if (lastMove() === moved) return moved;+ if (movePending) {+ io.say(`publishing for the move at ${new Date(moved).toISOString()}`);+ if (await publishFrom()) writeFileSync(done, `${moved}\n`, { mode: 0o600 });+ io.say('published');+ if (lastMove() === moved && !checkWanted()) return moved;+ } else {+ io.say('checking the presentation');+ await publishFrom({ ifChanged: true });+ }}}@@ -86,5 +117,5 @@ for (let covered = -1; await claim();) {try { covered = await publishUntilQuiet(); }catch (error) { io.err(`checkpoint not published: ${error.message}`); covered = Infinity; }finally { await release(); }- if (lastMove() <= covered) break;+ if (lastMove() <= covered && !checkWanted()) break;}diff --git a/src/relay-checkpoint.js b/src/relay-checkpoint.jsindex 82bfa031..ab0ec143 100644--- a/src/relay-checkpoint.js+++ b/src/relay-checkpoint.js@@ -50,14 +50,16 @@ export function checkpointQuietMs(env = process.env) {/*** The files the publisher keeps beside the relay link: when the last move happened, which move the last upload- * covered, and which process publishes.+ * covered, when a command last asked for a presentation check, when the last checkpoint was read, and which+ * process publishes.** @param {string} file - The relay link file.- * @returns {{ due: string, done: string, pid: string, log: string }}+ * @returns {{ due: string, done: string, asked: string, checked: string, pid: string, log: string }}*/export function checkpointFiles(file) {const directory = dirname(file);return { due: join(directory, 'relay-checkpoint.due'), done: join(directory, 'relay-checkpoint.done'),+ asked: join(directory, 'relay-checkpoint.asked'), checked: join(directory, 'relay-checkpoint.checked'),pid: join(directory, 'relay-checkpoint.pid'), log: join(directory, 'relay-checkpoint.log') };}@@ -95,8 +97,26 @@ export function alive(pid) {* @param {string} file - The relay link file.*/export function scheduleCheckpoint(root, file) {- const { due, pid } = checkpointFiles(file);- writeFileSync(due, `${Date.now()}\n`, { mode: 0o600 });+ notePublisher(root, file, 'due');+}++/**+ * Ask the publisher to read the board once things are quiet and upload only if the presentation changed. A spec,+ * doctrine or config edit changes the phone's page without any move, and a command cannot afford to build the+ * presentation to find out, so every synced command asks and the publisher compares in the background.+ *+ * @param {string} root+ * @param {string} file - The relay link file.+ */+export function requestCheckpointCheck(root, file) {+ notePublisher(root, file, 'asked');+}++/** Note the time in one of the publisher's files and start a publisher unless a live one holds the board. */+function notePublisher(root, file, marker) {+ const files = checkpointFiles(file);+ const { pid } = files;+ writeFileSync(files[marker], `${Date.now()}\n`, { mode: 0o600 });if (publisherHolding(pid)) return;const child = spawn(process.execPath, [WORKER, root, file], { cwd: root, detached: true, stdio: 'ignore', env: process.env });child.unref();diff --git a/src/relay.js b/src/relay.jsindex 8f2560ee..3a1c5302 100644--- a/src/relay.js+++ b/src/relay.js@@ -94,6 +94,7 @@ import { locked } from './relay-lock.js';import { checkpointPublisher } from './relay-checkpoint-publish.js';import {checkpointInBackground,+ requestCheckpointCheck,scheduleCheckpoint,} from './relay-checkpoint.js';import {@@ -1249,9 +1250,10 @@ export async function syncRelay(root, io) {saveLink(file, state);io.err(`pullboard: the relay ${state.url} did not sequence the previous move; retry that move explicitly if still wanted`);}- // In the background, a command never builds the presentation to compare it: only a saved, unsent checkpoint- // asks for a publisher. Otherwise the command publishes a changed presentation itself, as before #1014.- if (checkpointInBackground()) { if (state.checkpoint) scheduleCheckpoint(root, file); }+ // In the background, a command never builds the presentation to compare it: a saved, unsent checkpoint asks+ // for an upload, and any other command asks the publisher to compare, so a spec edit alone still reaches the+ // phone. Otherwise the command publishes a changed presentation itself, as before #1014.+ if (checkpointInBackground()) { if (state.checkpoint) scheduleCheckpoint(root, file); else requestCheckpointCheck(root, file); }else if (state.checkpoint || state.presentationDigest !== presentationDigest(relayPresentation(root))) await publishCheckpoint(root, file, state, io);await deliverDeviceWraps(root, io);return summary(root, state);diff --git a/test/relay-checkpoint.test.js b/test/relay-checkpoint.test.jsindex 4647cc46..da6f685e 100644--- a/test/relay-checkpoint.test.js+++ b/test/relay-checkpoint.test.js@@ -122,3 +122,62 @@ test('publishers that find a dead holder\'s claim together upload one checkpointassert.equal(box.calls.slice(from).filter(checkpointUpload).length, 1, 'four publishers recovering one dead claim sent one checkpoint');assert.equal(existsSync(join(directory, 'relay-checkpoint.pid')), false, 'the claim is released');});++test('a spec edit with no move reaches the next checkpoint, and an idle command uploads none [H5,H16]', async (t) => {+ const inline = process.env[CHECKPOINT_BACKGROUND_ENV];+ delete process.env[CHECKPOINT_BACKGROUND_ENV];+ t.after(() => { if (inline === undefined) delete process.env[CHECKPOINT_BACKGROUND_ENV]; else process.env[CHECKPOINT_BACKGROUND_ENV] = inline; });+ const box = await relayClientFixture(t);+ await box.link();+ const warm = box.calls.length;+ assert.equal((await box.cli('shout', 'all', 'warm up')).code, 0);+ await publisherDone(box, warm, 1);++ // Nothing changed: the publisher reads the board, finds the page the relay already has, and uploads nothing.+ const idle = box.calls.length;+ assert.equal((await box.cli('status')).code, 0);+ await new Promise(next => setTimeout(next, 4_000)); // budget: declared quiet period plus a read, so an upload would show.+ assert.equal(box.calls.slice(idle).filter(checkpointUpload).length, 0, 'an idle command re-sent no unchanged page');++ // A draft spec row changes only the presentation; the next command's check sends it within a few seconds.+ writeFileSync(join(box.root, 'SPEC.md'), '# Checkpoint fixture spec\n\n## G · Goals\n\n- G1 [draft, must] SPEC_ONLY_CHECKPOINT_1014 | gate: review\n');+ const edited = box.calls.length;+ const started = Date.now();+ assert.equal((await box.cli('status')).code, 0);+ await publisherDone(box, edited, 1);+ const elapsed = Date.now() - started;+ assert.ok(elapsed < 5_000, `the spec edit reached a checkpoint in ${elapsed} ms`); // budget: ceiling from the review: a presentation-only change goes out within five seconds.+});++test('a move returns while the publisher asks whether an upload pause is over [H5,H17]', async (t) => {+ const inline = process.env[CHECKPOINT_BACKGROUND_ENV];+ delete process.env[CHECKPOINT_BACKGROUND_ENV];+ t.after(() => { if (inline === undefined) delete process.env[CHECKPOINT_BACKGROUND_ENV]; else process.env[CHECKPOINT_BACKGROUND_ENV] = inline; });+ const box = await relayClientFixture(t);+ await box.link();+ const warm = box.calls.length;+ assert.equal((await box.cli('shout', 'all', 'warm up')).code, 0);+ await publisherDone(box, warm, 1);++ // A size pause from an earlier refused upload; the publisher's next pass first asks the relay if it is over.+ const link = JSON.parse(readFileSync(box.linkFile, 'utf8'));+ link.checkpointRefusal = { code: 'BAD_REQUEST', message: 'the snapshot exceeds the size limit', status: 413, size: true };+ writeFileSync(box.linkFile, JSON.stringify(link));+ box.holdProbes(true);+ const release = setTimeout(() => box.holdProbes(false), 10_000); // budget: ceiling: a held probe ends well after the move's limit.+ t.after(() => { clearTimeout(release); box.holdProbes(false); });+ const from = box.calls.length;+ assert.equal((await box.cli('status')).code, 0);+ const deadline = Date.now() + 15_000;+ while (box.heldProbes() === 0 && Date.now() < deadline) await new Promise(next => setTimeout(next, 50));+ assert.equal(box.heldProbes(), 1, 'the publisher\'s pause probe is in flight and held');+ const started = performance.now();+ const moved = await box.cli('shout', 'all', 'moves during the held probe');+ const elapsed = performance.now() - started;+ assert.equal(moved.code, 0, JSON.stringify(moved.document));+ assert.equal(box.heldProbes(), 1, 'the move returned while the probe was still held');+ assert.ok(elapsed < 2000, `the move took ${Math.round(elapsed)} ms; it must not wait for the probe`); // budget: ceiling for one CLI move with headroom, far under the held probe.+ box.holdProbes(false);+ await publisherDone(box, from, 1);+ assert.equal(JSON.parse(readFileSync(box.linkFile, 'utf8')).checkpointRefusal, undefined, 'the ended pause is cleared');+});diff --git a/test/relay-client-fixture.js b/test/relay-client-fixture.jsindex 7a23cef0..5738ddc7 100644--- a/test/relay-client-fixture.js+++ b/test/relay-client-fixture.js@@ -240,6 +240,8 @@ globalThis.fetch = function tracedFetch(input, init) {let stallApi = false;let holdPublishes = false;const heldPublishes = [];+ let holdProbes = false;+ const heldProbes = [];let loseSnapshotReply = null;const lostSnapshotReplies = [];let rejectSnapshotWrites = null;@@ -389,6 +391,10 @@ globalThis.fetch = function tracedFetch(input, init) {if (holdPublishes && req.method === 'POST' && /^\/api\/v2\/boards\/[0-9a-f]{32}\/uploads\/[^/]+\/publish$/u.test(req.url ?? '')) {await new Promise(release => heldPublishes.push(release));}+ // A held upload-pause probe waits too, so a test can move while the publisher asks whether a pause is over (#1014).+ if (holdProbes && req.method === 'GET' && /^\/api\/v2(?:\/boards\/[0-9a-f]{32}\/storage)?$/u.test(req.url ?? '')) {+ await new Promise(release => heldProbes.push(release));+ }if (legacySnapshotApi && req.method === 'GET' && req.url === '/api/v2') {res.writeHead(legacySnapshotApi === 'bad-route' ? 400 : 200, { 'content-type': 'application/json' });res.end(JSON.stringify(legacySnapshotApi === 'bad-route'@@ -776,6 +782,13 @@ globalThis.fetch = function tracedFetch(input, init) {},/** How many API2 checkpoint publishes are being held. */heldPublishes() { return heldPublishes.length; },+ /** Hold every upload-pause probe (the API version and storage reads) until called with false. */+ holdProbes(value = true) {+ holdProbes = value;+ if (!value) for (const release of heldProbes.splice(0)) release();+ },+ /** How many upload-pause probes are being held. */+ heldProbes() { return heldProbes.length; },/*** Let the real relay accept the next API1 snapshot PUT, then drop its reply by closing the socket,* optionally answering later state reads with a chosen refusal, as the #919 lost-ACK review did.