Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
refactor(relay): checkpoint publishing gets its own module [H5]
Corey Olsona48c4ad1c2f6
diff --git a/src/relay-checkpoint-publish.js b/src/relay-checkpoint-publish.jsnew file mode 100644index 00000000..db582725--- /dev/null+++ b/src/relay-checkpoint-publish.js@@ -0,0 +1,145 @@+/**+ * Publishing the relay checkpoint (#1014): read and seal the board, save the exact sealed record, upload it, and+ * record the reply. Inside a command all of it runs under the relay lock, as before; the background publisher reads+ * and seals before the lock and uploads outside it, so moves never wait on either.+ */+import {+ clearUploadPauseNotices,+ permanentUploadRefusal,+ uploadPauseType,+ uploadRetryReady,+} from './relay-upload-pause.js';+import {+ presentationDigest,+ relaySnapshot,+} from './relay-presentation.js';+import { locked } from './relay-lock.js';+import { Refused } from './refused.js';++/**+ * Build the checkpoint publisher over relay.js's own link and transport helpers, which it cannot import without a cycle.+ *+ * @param {{ request: Function, saveLink: Function, loadLink: Function, linkFile: Function, readBoardKey: Function,+ * sealedRecord: Function, reportCheckpointRefusal: Function, safePauseCode: Function, safePauseMessage: Function }} relay+ * @returns {{ publishCheckpoint: Function, publishLatestCheckpoint: Function }}+ */+export function checkpointPublisher({ request, saveLink, loadLink, linkFile, readBoardKey, sealedRecord, reportCheckpointRefusal, safePauseCode, safePauseMessage }) {+ /**+ * Read and seal this board's checkpoint: the expensive part of publishing, which needs no relay lock.+ *+ * @param {string} root+ * @param {any} state - The relay link, for the board's identity and key.+ * @returns {Promise<{ sequence: number, digest: string, sealed: string }>}+ */+ async function prepareCheckpoint(root, state) {+ const document = relaySnapshot(root);+ const sequence = Number(document.tables.board_meta.find((row) => row.meta_key === 'relay_applied_sequence')?.meta_value ?? 0);+ return { sequence, digest: presentationDigest(document.presentation),+ sealed: await sealedRecord(readBoardKey(state.board), document, state, 'snapshot', sequence) };+ }++ /**+ * Persist the exact checkpoint before uploading, so a lost reply retries identical ciphertext. `prepared`, sealed+ * outside the lock by the background publisher, stands in for reading the board again; an older one never replaces+ * a newer saved checkpoint.+ */+ async function publishCheckpoint(root, file, state, io, prepared = null) {+ const staged = await stageCheckpoint(root, file, state, io, prepared);+ if (!staged) return;+ const sent = state.checkpoint;+ let outcome;+ try { outcome = { reply: await request(state, '/api/v1/boards/' + state.board + '/state', { method: 'PUT', body: sent }, io) }; }+ catch (error) { outcome = { error }; }+ recordCheckpointUpload(file, state, io, staged, sent, outcome);+ }++ /**+ * Decide what checkpoint to send and save its exact sealed record first; null when nothing is to be sent (no machine+ * credential, an upload pause not yet over, or a definitive rejection of this same image).+ */+ async function stageCheckpoint(root, file, state, io, prepared) {+ if (!state.token?.startsWith('pm_') || process.env.PULLBOARD_RELAY_TOKEN) return null;+ const resumed = Boolean(state.checkpointRefusal);+ if (state.checkpointRefusal && uploadPauseType(state.checkpointRefusal)) {+ if (!await uploadRetryReady(state, state.checkpointRefusal, path => request(state, path, {}, io))) {+ saveLink(file, state); reportCheckpointRefusal(state, io); return null;+ }+ delete state.checkpointRefusal;+ saveLink(file, state);+ } else {+ const fresh = prepared && !(state.checkpoint?.sequence > prepared.sequence) ? prepared : await prepareCheckpoint(root, state);+ if (!state.checkpoint || state.checkpoint.sequence !== fresh.sequence || state.checkpointPresentationDigest !== fresh.digest) {+ delete state.checkpointRefusal;+ state.checkpointPresentationDigest = fresh.digest;+ state.checkpoint = { sequence: fresh.sequence, sealed: fresh.sealed };+ saveLink(file, state);+ }+ // A definitive rejection retries only a fresh image after the board changes.+ if (state.checkpointRefusal) { reportCheckpointRefusal(state, io); return null; }+ }+ return { resumed };+ }++ /**+ * Record how the upload of `sent` ended. A newer checkpoint saved meanwhile is left for the next upload, so an older+ * acknowledgement never marks it sent.+ */+ function recordCheckpointUpload(file, state, io, staged, sent, { reply, error }) {+ if (error) {+ if (!(error instanceof Refused) || !permanentUploadRefusal(error)) throw error;+ if (state.checkpoint?.sealed === sent.sealed) {+ state.checkpointRefusal = { code: safePauseCode(error), message: safePauseMessage(error), status: error.httpStatus };+ saveLink(file, state);+ reportCheckpointRefusal(state, io);+ }+ return;+ }+ if (reply.state?.sequence !== sent.sequence || reply.state.sealed !== sent.sealed) throw new Refused('RELAY_RESPONSE', 'the relay did not acknowledge this checkpoint; retry the configured relay');+ if (state.checkpoint?.sealed === sent.sealed) {+ state.presentationDigest = state.checkpointPresentationDigest;+ delete state.checkpoint;+ delete state.checkpointPresentationDigest;+ }+ saveLink(file, state);+ if (staged.resumed) clearUploadPauseNotices(state);+ }++ /**+ * Publish this board's checkpoint once: the background publisher's one step (#1014). The board is read and sealed+ * before the relay lock, the lock is held only to save the sealed record and later to record the reply, and the+ * upload itself holds no lock. A board whose link is gone or being removed publishes nothing.+ *+ * @param {string} root+ * @param {{ err: (line: string) => void, say: (line: string) => void }} io+ * @returns {Promise<boolean>} Whether a linked board was published.+ */+ async function publishLatestCheckpoint(root, io) {+ const file = linkFile(root);+ const linked = loadLink(file);+ if (!linked || linked.unlinking) return false;+ // Reading and sealing a large board takes about a second; it happens before the lock, so moves never wait on it.+ const prepared = await prepareCheckpoint(root, linked);+ const staged = await locked(file, async () => {+ const state = loadLink(file);+ if (!state || state.unlinking || state.board !== linked.board) return null;+ const result = await stageCheckpoint(root, file, state, io, prepared);+ return result && { ...result, sent: state.checkpoint, copy: structuredClone(state) };+ }, io);+ if (!staged) return false;+ // The upload holds no lock, so a move during it returns at once and the next checkpoint covers it. It runs on a+ // copy of the link: the copy saves nothing, and what the upload learns is carried back under the lock below.+ let outcome;+ try { outcome = { reply: await request(staged.copy, '/api/v1/boards/' + staged.copy.board + '/state', { method: 'PUT', body: staged.sent }, io) }; }+ catch (error) { outcome = { error }; }+ await locked(file, async () => {+ const state = loadLink(file);+ if (!state || state.board !== linked.board) return;+ if (staged.copy.snapshotApiVersion !== undefined) state.snapshotApiVersion = staged.copy.snapshotApiVersion;+ if (staged.copy.storage !== undefined) state.storage = staged.copy.storage;+ recordCheckpointUpload(file, state, io, staged, staged.sent, outcome);+ }, io);+ return true;+ }++ return { publishCheckpoint, publishLatestCheckpoint };+}diff --git a/src/relay-checkpoint-worker.js b/src/relay-checkpoint-worker.jsindex 328aebba..3c74b884 100644--- a/src/relay-checkpoint-worker.js+++ b/src/relay-checkpoint-worker.js@@ -5,8 +5,18 @@** Usage (spawned by scheduleCheckpoint): node relay-checkpoint-worker.js <root> <relay link file>*/-import { appendFileSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'node:fs';-import { checkpointFiles, checkpointQuietMs, publisherHolding } from './relay-checkpoint.js';+import {+ appendFileSync,+ readFileSync,+ rmSync,+ utimesSync,+ writeFileSync,+} from 'node:fs';+import {+ checkpointFiles,+ checkpointQuietMs,+ publisherHolding,+} from './relay-checkpoint.js';import { publishLatestCheckpoint } from './relay.js';const [root, file] = process.argv.slice(2);diff --git a/src/relay-checkpoint.js b/src/relay-checkpoint.jsindex 1b26635d..6e7a301f 100644--- a/src/relay-checkpoint.js+++ b/src/relay-checkpoint.js@@ -8,8 +8,15 @@* keeps one upload in flight; moves during an upload are covered by the next one.*/import { spawn } from 'node:child_process';-import { readFileSync, statSync, writeFileSync } from 'node:fs';-import { dirname, join } from 'node:path';+import {+ readFileSync,+ statSync,+ writeFileSync,+} from 'node:fs';+import {+ dirname,+ join,+} from 'node:path';import { fileURLToPath } from 'node:url';/** Set to 0 to publish the checkpoint inside the command, as before #1014; the test runner sets it, like the view. */diff --git a/src/relay.js b/src/relay.jsindex aeeb7e13..8f2560ee 100644--- a/src/relay.js+++ b/src/relay.js@@ -91,6 +91,7 @@ import {import { terminalQr } from './qr.js';import { environmentLink } from './relay-environment.js';import { locked } from './relay-lock.js';+import { checkpointPublisher } from './relay-checkpoint-publish.js';import {checkpointInBackground,scheduleCheckpoint,@@ -864,122 +865,11 @@ async function replayRelayRecord(board, key, state, record) {}}-/**- * Read and seal this board's checkpoint: the expensive part of publishing, which needs no relay lock.- *- * @param {string} root- * @param {any} state - The relay link, for the board's identity and key.- * @returns {Promise<{ sequence: number, digest: string, sealed: string }>}- */-async function prepareCheckpoint(root, state) {- const document = relaySnapshot(root);- const sequence = Number(document.tables.board_meta.find((row) => row.meta_key === 'relay_applied_sequence')?.meta_value ?? 0);- return { sequence, digest: presentationDigest(document.presentation),- sealed: await sealedRecord(readBoardKey(state.board), document, state, 'snapshot', sequence) };-}--/**- * Persist the exact checkpoint before uploading, so a lost reply retries identical ciphertext. `prepared`, sealed- * outside the lock by the background publisher, stands in for reading the board again; an older one never replaces- * a newer saved checkpoint.- */-async function publishCheckpoint(root, file, state, io, prepared = null) {- const staged = await stageCheckpoint(root, file, state, io, prepared);- if (!staged) return;- const sent = state.checkpoint;- let outcome;- try { outcome = { reply: await request(state, '/api/v1/boards/' + state.board + '/state', { method: 'PUT', body: sent }, io) }; }- catch (error) { outcome = { error }; }- recordCheckpointUpload(file, state, io, staged, sent, outcome);-}--/**- * Decide what checkpoint to send and save its exact sealed record first; null when nothing is to be sent (no machine- * credential, an upload pause not yet over, or a definitive rejection of this same image).- */-async function stageCheckpoint(root, file, state, io, prepared) {- if (!state.token?.startsWith('pm_') || process.env.PULLBOARD_RELAY_TOKEN) return null;- const resumed = Boolean(state.checkpointRefusal);- if (state.checkpointRefusal && uploadPauseType(state.checkpointRefusal)) {- if (!await uploadRetryReady(state, state.checkpointRefusal, path => request(state, path, {}, io))) {- saveLink(file, state); reportCheckpointRefusal(state, io); return null;- }- delete state.checkpointRefusal;- saveLink(file, state);- } else {- const fresh = prepared && !(state.checkpoint?.sequence > prepared.sequence) ? prepared : await prepareCheckpoint(root, state);- if (!state.checkpoint || state.checkpoint.sequence !== fresh.sequence || state.checkpointPresentationDigest !== fresh.digest) {- delete state.checkpointRefusal;- state.checkpointPresentationDigest = fresh.digest;- state.checkpoint = { sequence: fresh.sequence, sealed: fresh.sealed };- saveLink(file, state);- }- // A definitive rejection retries only a fresh image after the board changes.- if (state.checkpointRefusal) { reportCheckpointRefusal(state, io); return null; }- }- return { resumed };-}--/**- * Record how the upload of `sent` ended. A newer checkpoint saved meanwhile is left for the next upload, so an older- * acknowledgement never marks it sent.- */-function recordCheckpointUpload(file, state, io, staged, sent, { reply, error }) {- if (error) {- if (!(error instanceof Refused) || !permanentUploadRefusal(error)) throw error;- if (state.checkpoint?.sealed === sent.sealed) {- state.checkpointRefusal = { code: safePauseCode(error), message: safePauseMessage(error), status: error.httpStatus };- saveLink(file, state);- reportCheckpointRefusal(state, io);- }- return;- }- if (reply.state?.sequence !== sent.sequence || reply.state.sealed !== sent.sealed) throw new Refused('RELAY_RESPONSE', 'the relay did not acknowledge this checkpoint; retry the configured relay');- if (state.checkpoint?.sealed === sent.sealed) {- state.presentationDigest = state.checkpointPresentationDigest;- delete state.checkpoint;- delete state.checkpointPresentationDigest;- }- saveLink(file, state);- if (staged.resumed) clearUploadPauseNotices(state);-}--/**- * Publish this board's checkpoint once: the background publisher's one step (#1014). The board is read and sealed- * before the relay lock, the lock is held only to save the sealed record and later to record the reply, and the- * upload itself holds no lock. A board whose link is gone or being removed publishes nothing.- *- * @param {string} root- * @param {{ err: (line: string) => void, say: (line: string) => void }} io- * @returns {Promise<boolean>} Whether a linked board was published.- */-export async function publishLatestCheckpoint(root, io) {- const file = linkFile(root);- const linked = loadLink(file);- if (!linked || linked.unlinking) return false;- // Reading and sealing a large board takes about a second; it happens before the lock, so moves never wait on it.- const prepared = await prepareCheckpoint(root, linked);- const staged = await locked(file, async () => {- const state = loadLink(file);- if (!state || state.unlinking || state.board !== linked.board) return null;- const result = await stageCheckpoint(root, file, state, io, prepared);- return result && { ...result, sent: state.checkpoint, copy: structuredClone(state) };- }, io);- if (!staged) return false;- // The upload holds no lock, so a move during it returns at once and the next checkpoint covers it. It runs on a- // copy of the link: the copy saves nothing, and what the upload learns is carried back under the lock below.- let outcome;- try { outcome = { reply: await request(staged.copy, '/api/v1/boards/' + staged.copy.board + '/state', { method: 'PUT', body: staged.sent }, io) }; }- catch (error) { outcome = { error }; }- await locked(file, async () => {- const state = loadLink(file);- if (!state || state.board !== linked.board) return;- if (staged.copy.snapshotApiVersion !== undefined) state.snapshotApiVersion = staged.copy.snapshotApiVersion;- if (staged.copy.storage !== undefined) state.storage = staged.copy.storage;- recordCheckpointUpload(file, state, io, staged, staged.sent, outcome);- }, io);- return true;-}+// Checkpoint publishing lives in its own module, over these link and transport helpers (#1014).+const checkpoints = checkpointPublisher({ request, saveLink, loadLink, linkFile, readBoardKey, sealedRecord, reportCheckpointRefusal, safePauseCode, safePauseMessage });+const publishCheckpoint = (...args) => checkpoints.publishCheckpoint(...args);+/** Publish this board's checkpoint once, as the background publisher's one step (#1014). */+export const publishLatestCheckpoint = (...args) => checkpoints.publishLatestCheckpoint(...args);/** Drain a legacy source once, refusing foreign history before it can be silently checkpointed. */async function ensureOrdered(root, file, state, io) {diff --git a/test/relay-checkpoint.test.js b/test/relay-checkpoint.test.jsindex 985adf39..b91156d6 100644--- a/test/relay-checkpoint.test.js+++ b/test/relay-checkpoint.test.js@@ -62,7 +62,7 @@ test('a move on a large board sends only its event and never the checkpoint [H5]assert.equal(sent.filter(checkpointUpload).length, 0, `move ${index} uploaded no checkpoint`);const bytes = sent.reduce((total, call) => total + (call.requestBytes ?? 0), 0);assert.ok(bytes < 64 * 1024, `move ${index} sent ${bytes} bytes, under 64 KB`);- assert.ok(elapsed < 1000, `move ${index} returned in ${Math.round(elapsed)} ms, under one second`);+ assert.ok(elapsed < 1000, `move ${index} returned in ${Math.round(elapsed)} ms, under one second`); // budget: declared by #1014's bar: a move returns in under one second.}await publisherDone(box, before, 1);await new Promise(next => setTimeout(next, 4_000)); // budget: a second quiet period, so a second upload would show.@@ -92,7 +92,7 @@ test('a move during a held checkpoint upload returns before the upload ends [H5]const elapsed = performance.now() - started;assert.equal(moved.code, 0, JSON.stringify(moved.document));assert.equal(box.heldPublishes(), 1, 'the move returned while the upload was still held');- assert.ok(elapsed < 2000, `the move took ${Math.round(elapsed)} ms; it must not wait for the upload`);+ assert.ok(elapsed < 2000, `the move took ${Math.round(elapsed)} ms; it must not wait for the upload`); // budget: ceiling for one CLI move with headroom, far under the held upload.box.holdPublishes(false);await publisherDone(box, from, 2);});