Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(relay): queued moves follow their item if it is renumbered [B26]

Corey Olsond28b4ff1f206
diff --git a/src/relay-move-fallback.js b/src/relay-move-fallback.js
index fd11daf6..8f43ed35 100644
--- a/src/relay-move-fallback.js
+++ b/src/relay-move-fallback.js
@@ -18,3 +18,14 @@ export function transientRelayFailure(error) {
if (['RELAY_UNAVAILABLE', 'RELAY_RESPONSE', 'RELAY_BUSY'].includes(error.code)) return true;
return Number.isInteger(error.httpStatus) && (error.httpStatus >= 500 || [408, 425, 429].includes(error.httpStatus));
}
+
+/**
+ * Name how a relay request failed before any response, from the error's own fields only.
+ * Why: a bare "did not answer" hid whether a request timed out, was refused or lost its socket,
+ * so load-dependent relay failures could not be attributed (#512).
+ */
+export function transportFailure(error) {
+ if (error?.name === 'TimeoutError' || error?.name === 'AbortError') return 'timed out';
+ const code = error?.cause?.code ?? error?.code;
+ return typeof code === 'string' && /^[A-Z][A-Z0-9_]{1,40}$/u.test(code) ? code : 'no connection';
+}
diff --git a/src/relay-outbox.js b/src/relay-outbox.js
index b5d61955..26e24eae 100644
--- a/src/relay-outbox.js
+++ b/src/relay-outbox.js
@@ -19,6 +19,28 @@ import {
gunzipSync,
gzipSync,
} from 'node:zlib';
+import * as store from './board.js';
+import {
+ engineReceipt,
+ executableArgs,
+ prepareEngineMove,
+} from './engine.js';
+import {
+ exportBoard,
+ restoreRelaySnapshot,
+} from './exchange.js';
+import { readBoardKey } from './relay-key.js';
+import { transientRelayFailure } from './relay-move-fallback.js';
+import {
+ actorTransport,
+ seatReceipt,
+ withSeatName,
+} from './relay-seats.js';
+import {
+ credentialActor,
+ operationIntent,
+ relayMoveActor,
+} from './relay-sender.js';
import {
seal,
unseal,
@@ -28,6 +50,9 @@ import { Refused } from './refused.js';
/** How many replay refusals the link keeps for pullboard status; older ones stay in the relay's own receipts. */
const KEPT_REFUSALS = 20;
+/** Commands that already said their move waits in the outbox, so a command says it once. */
+const OUTBOX_NOTICES = new WeakSet();
+
/** The sealed outbox beside a link file. */
export function outboxFile(linkFile) {
return join(dirname(linkFile), 'relay-outbox.sealed');
@@ -121,3 +146,240 @@ export function refusalLine(kept) {
: '; no other move reached the relay first';
return `[RELAY_OUTBOX_REFUSED] the queued move ${kept.move} (made ${kept.queuedAt}) was refused when it replayed: [${kept.code}] ${kept.message}${first}`;
}
+
+/** Moves whose first argument is an item number; a decision's answer names a shout instead. */
+const ITEM_MOVES = new Set(['claim', 'release', 'submit', 'verify', 'reopen', 'refreeze', 'merged', 'withdraw', 'editItem', 'holdItem', 'releaseItemHold', 'escalate', 'appendFact']);
+
+/**
+ * Rewrite the item numbers a queued move names through `remap` (first local number to the number it has now), and
+ * list any it names whose add was refused at replay.
+ * Why: another machine may add an item while this one is offline, so an item added here can get another number from
+ * the relay; a later queued move must follow it rather than act on the other machine's item (#922).
+ */
+export function remapMove(move, remap, lost) {
+ const next = structuredClone(move);
+ const touched = [];
+ const follow = (id) => {
+ if (!Number.isSafeInteger(id)) return id;
+ if (lost.includes(id)) touched.push(id);
+ return Object.hasOwn(remap, id) ? remap[id] : id;
+ };
+ if (ITEM_MOVES.has(next.operation)) next.args[0] = follow(next.args[0]);
+ const options = next.operation === 'addItem' ? next.args[0] : next.operation === 'editItem' ? next.args[1] : null;
+ if (options && typeof options === 'object') {
+ if (options.parentId !== undefined) options.parentId = follow(options.parentId);
+ if (Array.isArray(options.after)) options.after = options.after.map(follow);
+ }
+ return { move: next, lost: touched };
+}
+
+/** Record that the item first added here as `was` is now `is`; true when its number changed. */
+function followItem(remap, was, is) {
+ if (!Number.isSafeInteger(is) || is === was) { delete remap[was]; return false; }
+ remap[was] = is;
+ return true;
+}
+
+/** Drop the replayed move from the outbox and save what is left. */
+async function finishEntry(link, outbox, where, key) {
+ outbox.moves.shift();
+ if (outbox.moves.length) await writeOutbox(where, key, link.state.board, outbox);
+ else removeOutbox(where);
+ saveOutboxSummary(link, outbox);
+}
+
+/** The queued moves and replay refusals pullboard status shows, when there are any. */
+export function outboxFields(state) {
+ return { ...(state?.outbox?.moves?.length ? { outbox: structuredClone(state.outbox.moves) } : {}),
+ ...(state?.outboxRefusals?.length ? { outboxRefusals: structuredClone(state.outboxRefusals) } : {}),
+ ...(state?.outboxRenumbered?.length ? { outboxRenumbered: structuredClone(state.outboxRenumbered) } : {}) };
+}
+
+/** Whether this command may queue a move in the ordered outbox: the board's relay.outbox setting, off by default (#922). */
+function outboxEnabled(io) {
+ try { return io?.relayOutbox?.() === true; } catch { return false; }
+}
+
+/**
+ * Whether a failure on an accepted link may queue the move instead of refusing it (#922): the relay said nothing
+ * definite, and the move needs no relay answer to exist (a registration mints its token there; a person's request
+ * is the phone's to send).
+ */
+export function outboxQueueable(state, error, operation, io) {
+ return state.mode === 'ordered' && state.baselineAccepted === true && !state.baselineConflict && !io?.personRequest
+ && !['register', 'ensureCoordinator'].includes(operation) && transientRelayFailure(error) && outboxEnabled(io);
+}
+
+/** The board's relay-applied sequence, read from an exported image. */
+function exportedSequence(native) {
+ return Number(native.tables.board_meta.find((row) => row.meta_key === 'relay_applied_sequence')?.meta_value ?? 0);
+}
+
+/** Keep the key-free outbox summary on the link, or drop it once the outbox is empty. */
+function saveOutboxSummary({ file, state, saveLink }, outbox) {
+ if (outbox?.moves.length) state.outbox = { restored: outbox.restored, moves: outbox.moves.map(outboxEntrySummary) };
+ else delete state.outbox;
+ saveLink(file, state);
+}
+
+/**
+ * Queue a move the relay could not take on an accepted link, then make it on this machine (#922).
+ * The first queued move seals the board as the relay last ordered it; the move is written before it is applied,
+ * so a crash between the two leaves it queued and the replay starts from that image anyway. A move this board
+ * refuses is refused now and never queued.
+ */
+export async function queueMove(link, { operation, args, command, move, error, applyLocally }) {
+ const { root, file, state, io, localRecords, safePauseCode, safePauseMessage } = link;
+ const key = readBoardKey(state.board);
+ const where = outboxFile(file);
+ const outbox = await readOutbox(where, key, state.board) ?? (() => {
+ const base = localRecords(root, (board) => exportBoard(board));
+ return { version: 1, base, baseSequence: exportedSequence(base), restored: false, moves: [] };
+ })();
+ if (!move) {
+ const sent = withSeatName(root, operation, args);
+ move = localRecords(root, (board) => prepareEngineMove(board, operation, sent));
+ move.actor = relayMoveActor(move) ?? credentialActor(move);
+ }
+ // `after` is the relay sequence this machine had seen: a refusal at replay names the moves ordered since.
+ const entry = { move, intent: operationIntent(operation, args), queuedAt: new Date().toISOString(), after: state.sequence,
+ ...(command ? { command, commandIntent: operationIntent('cli:' + command.cliOperation, command) } : {}) };
+ outbox.moves.push(entry);
+ await writeOutbox(where, key, state.board, outbox);
+ if (state.pending?.move?.id === move.id) delete state.pending;
+ saveOutboxSummary(link, outbox);
+ let result;
+ try { result = await applyLocally(); }
+ catch (refusal) {
+ outbox.moves.pop();
+ if (outbox.moves.length) await writeOutbox(where, key, state.board, outbox);
+ else removeOutbox(where);
+ saveOutboxSummary(link, outbox);
+ throw refusal;
+ }
+ if (move.operation === 'addItem' && Number.isSafeInteger(result)) {
+ entry.localItem = result;
+ await writeOutbox(where, key, state.board, outbox);
+ }
+ if (!OUTBOX_NOTICES.has(io)) {
+ OUTBOX_NOTICES.add(io);
+ io.err(`pullboard: [${safePauseCode(error)}] ${error.transportFailure ? `the relay did not answer (${error.transportFailure})` : safePauseMessage(error)}; the move is saved on this machine and waits in the relay outbox (${outbox.moves.length} pending); it goes up in order when the relay answers, and pullboard status lists it`);
+ }
+ return result;
+}
+
+/**
+ * Put the queued moves back on top of the board after a replay stopped part way (#922): seal the board as the relay
+ * has now ordered it, then apply each remaining move here again, so this machine keeps showing them.
+ */
+async function rebaseOutbox(link, outbox, key) {
+ const { root, file, state, localRecords } = link;
+ outbox.base = localRecords(root, (board) => exportBoard(board));
+ outbox.baseSequence = exportedSequence(outbox.base);
+ outbox.restored = false;
+ await writeOutbox(outboxFile(file), key, state.board, outbox);
+ saveOutboxSummary(link, outbox);
+ const remap = outbox.remap ??= {};
+ for (const entry of outbox.moves) {
+ const { move } = remapMove(entry.original ?? entry.move, remap, outbox.lost ?? []);
+ const result = localRecords(root, (board) => {
+ try { return store.atomic(board, () => store[move.operation](board, ...executableArgs(move, board))); }
+ catch (error) { if (!(error instanceof Refused)) throw error; return null; }
+ });
+ // A re-applied add may get another number here; later queued moves name it by its first local number.
+ if (Number.isSafeInteger(entry.localItem) && Number.isSafeInteger(result)) followItem(remap, entry.localItem, result);
+ }
+ await writeOutbox(outboxFile(file), key, state.board, outbox);
+}
+
+/**
+ * Replay the ordered outbox before this machine applies or sends anything else (#922). The board goes back to the
+ * image the relay last ordered, catches up with every other machine's moves, then sends each queued move in order:
+ * a move the relay already holds from a lost reply is not sent twice, and a move refused at replay is reported with
+ * the moves ordered before it. A relay that fails again part way puts the rest back on top and throws its failure.
+ */
+export async function flushOutbox(link) {
+ const { root, file, state, io, request, catchUp, sendPending, sealedRecord, localRecords, saveLink } = link;
+ const where = outboxFile(file);
+ if (!state.outbox && !existsSync(where)) return;
+ const key = readBoardKey(state.board);
+ const outbox = await readOutbox(where, key, state.board);
+ if (!outbox?.moves.length) {
+ removeOutbox(where);
+ saveOutboxSummary(link, null);
+ return;
+ }
+ try {
+ if (!outbox.restored) {
+ // Probe first: a relay that is still down leaves this machine's board exactly as its person last saw it.
+ try { await request(state, '/api/v1/boards/' + state.board + '/events?after=' + outbox.baseSequence, {}, io); }
+ catch (error) { if (error.code !== 'SNAPSHOT_REQUIRED') throw error; }
+ localRecords(root, (board) => {
+ const cursors = board.db.prepare('SELECT agent_id, agent_last_shout_id FROM agent').all();
+ restoreRelaySnapshot(board, outbox.base, outbox.baseSequence);
+ // Reading shouts is this machine's own progress; the restore keeps the newer read position.
+ const keep = board.db.prepare('UPDATE agent SET agent_last_shout_id = ? WHERE agent_id = ? AND (agent_last_shout_id IS NULL OR agent_last_shout_id < ?)');
+ for (const row of cursors) if (Number.isSafeInteger(row.agent_last_shout_id)) keep.run(row.agent_last_shout_id, row.agent_id, row.agent_last_shout_id);
+ });
+ state.sequence = outbox.baseSequence;
+ outbox.restored = true;
+ await writeOutbox(where, key, state.board, outbox);
+ saveOutboxSummary(link, outbox);
+ }
+ await catchUp(root, file, state, io);
+ outbox.remap ??= {};
+ outbox.lost ??= [];
+ while (outbox.moves.length) {
+ const entry = outbox.moves[0];
+ const receiptOf = () => localRecords(root, (board) => engineReceipt(board, entry.move.id));
+ let receipt = receiptOf();
+ let sequence = receipt?.sequence;
+ if (!receipt) {
+ // Map from the move as it was made, never from an earlier mapped copy, so no number is mapped twice.
+ entry.original ??= entry.move;
+ const { move, lost } = remapMove(entry.original, outbox.remap, outbox.lost);
+ if (lost.length) {
+ io.err(`pullboard: ${keepRefusal(state, entry, { code: 'RELAY_OUTBOX_DEPENDS', message: `it acts on #${lost[0]}, which this machine added while the relay was down, and that add was refused` }, [])}`);
+ await finishEntry(link, outbox, where, key);
+ continue;
+ }
+ // The renumbered move is saved before it is sent, so a lost reply is matched by these exact contents.
+ entry.move = move;
+ await writeOutbox(where, key, state.board, outbox);
+ await actorTransport(root, file, state, credentialActor(entry.move), io);
+ sequence = state.sequence + 1;
+ const sealed = await sealedRecord(key, entry.move, state, 'move', sequence);
+ const registration = seatReceipt(entry.move, sequence, sealed);
+ state.pending = { move: entry.move, intent: entry.intent, ...(entry.command ? { command: entry.command, commandIntent: entry.commandIntent } : {}),
+ sequence, sealed, ...(registration ? { registration } : {}) };
+ saveLink(file, state);
+ let failure;
+ try { await sendPending(root, file, state, io); } catch (error) { failure = error; }
+ if (failure && !receiptOf()) {
+ if (!['RELAY_UNAVAILABLE', 'RELAY_RESPONSE'].includes(failure.code)) throw failure;
+ try { await catchUp(root, file, state, io); } catch { throw failure; }
+ }
+ receipt = receiptOf();
+ if (!receipt) throw failure ?? new Refused('RELAY_RESPONSE', 'the relay acknowledged a queued move without its replay receipt; run pullboard status to retry the outbox');
+ sequence = receipt.sequence;
+ }
+ if (state.pending?.move?.id === entry.move.id) delete state.pending;
+ if (state.recovered?.move?.id === entry.move.id) delete state.recovered;
+ if (receipt.outcome.error) {
+ const earlier = localRecords(root, (board) => earlierMoves(board, entry, sequence));
+ io.err(`pullboard: ${keepRefusal(state, entry, receipt.outcome.error, earlier)}`);
+ if (Number.isSafeInteger(entry.localItem)) outbox.lost.push(entry.localItem);
+ } else if (Number.isSafeInteger(entry.localItem) && followItem(outbox.remap, entry.localItem, receipt.outcome.result)) {
+ state.outboxRenumbered = [...(state.outboxRenumbered ?? []), { was: entry.localItem, is: receipt.outcome.result }].slice(-KEPT_REFUSALS);
+ io.err(`pullboard: the item this machine added as #${entry.localItem} while the relay was down is #${receipt.outcome.result} on the board; its queued moves follow it`);
+ }
+ await finishEntry(link, outbox, where, key);
+ }
+ } catch (error) {
+ if (outbox.restored && outbox.moves.length) {
+ if (state.pending && outbox.moves.some((entry) => entry.move.id === state.pending.move?.id)) delete state.pending;
+ await rebaseOutbox(link, outbox, key);
+ }
+ throw error;
+ }
+}
diff --git a/src/relay-output.js b/src/relay-output.js
index f029ac90..96e738a1 100644
--- a/src/relay-output.js
+++ b/src/relay-output.js
@@ -24,6 +24,7 @@ export function outboxLines(state) {
for (const move of state.outbox) lines.push(` ${describeMove({ operation: move.operation, args: [move.item], actor: move.actor })} (made ${move.queuedAt}): pending`);
}
for (const refused of state.outboxRefusals ?? []) lines.push(refusalLine(refused));
+ for (const { was, is } of state.outboxRenumbered ?? []) lines.push(`relay outbox: the item added here as #${was} while the relay was down is #${is} on the board`);
return lines;
}
diff --git a/src/relay-sender.js b/src/relay-sender.js
index 15015ae3..97c0c3ac 100644
--- a/src/relay-sender.js
+++ b/src/relay-sender.js
@@ -71,3 +71,30 @@ export function relaySenderProblem(move, sender, kind, { phoneReceipt = false }
if (actor !== sender.agent) return new Refused('RELAY_SENDER_MISMATCH', `agent ${sender.agent} cannot act as ${typeof actor === 'string' ? actor : '(no declared agent)'}; send this operation as the authenticated agent`);
return null;
}
+
+/** Choose credentials only; main's relay sender policy remains the sole replay authorization. */
+export function credentialActor(move) {
+ const args = move.args;
+ if (move.operation === 'register' && move.engine >= VERIFIER_STANDING_ENGINE) {
+ const creator = args?.[0]?.createdBy;
+ return typeof creator === 'string' && creator !== 'machine' ? creator : 'machine';
+ }
+ if (move.personRequest || ['register', 'ensureCoordinator'].includes(move.operation)) return 'machine';
+ if (['recordRowDecisions', 'recordLandingWaiver'].includes(move.operation)
+ || move.operation === 'answerDecision' && args[1]?.asPerson
+ || move.operation === 'shout' && args[0]?.request) return 'person';
+ return relayMoveActor(move);
+}
+
+/** Compare caller intent without executing frozen callbacks or allocating another operation id. */
+export function operationIntent(operation, args) {
+ const values = Array.isArray(args) ? args.map((value, index) => {
+ if (!['claim', 'refreeze'].includes(operation) || index !== 1 || !value || typeof value !== 'object') return value;
+ const options = { ...value };
+ delete options.freeze;
+ delete options.frozen;
+ delete options.freezeError;
+ return options;
+ }) : args;
+ return JSON.stringify({ operation, args: values }, (_key, value) => typeof value === 'function' ? undefined : value);
+}
diff --git a/src/relay.js b/src/relay.js
index da382c6a..c90dbd13 100644
--- a/src/relay.js
+++ b/src/relay.js
@@ -43,13 +43,17 @@ import {
applyRelayMove,
checkpointSequence,
engineReceipt,
- executableArgs,
prepareEngineMove,
refuseRelayMove,
requireSupportedEngine,
startRelayEpoch,
} from './engine.js';
-import { relayMoveActor } from './relay-sender.js';
+import {
+ credentialActor,
+ operationIntent,
+ relayMoveActor,
+ relaySenderProblem,
+} from './relay-sender.js';
import {
actorTransport,
cacheSeatToken,
@@ -93,20 +97,17 @@ import {
import {
baselineNotAccepted,
transientRelayFailure,
+ transportFailure,
} from './relay-move-fallback.js';
import {
- earlierMoves,
- keepRefusal,
+ flushOutbox,
outboxAhead,
- outboxEntrySummary,
- outboxFile,
- readOutbox,
- removeOutbox,
- writeOutbox,
+ outboxFields,
+ outboxQueueable,
+ queueMove,
} from './relay-outbox.js';
import { Refused } from './refused.js';
import { refusalFromReceipt } from './recorded-refusal.js';
-import { relaySenderProblem } from './relay-sender.js';
import { relayNotices } from './relay-notices.js';
import {
readRelayMachine,
@@ -141,7 +142,6 @@ export const DEFAULT_RELAY = 'https://app.pullboard.dev';
const LINK_FILES = new WeakMap();
const STATE_FILES = new WeakMap();
const MOVE_FALLBACK_NOTICES = new WeakSet();
-const OUTBOX_NOTICES = new WeakSet();
const KEY_WARNED_COMMANDS = new WeakSet();
const BASELINE_ATTEMPTS = new WeakMap();
const BASELINE_NOTICES = new WeakSet();
@@ -333,17 +333,6 @@ export function saveLink(file, state) {
} finally { rmSync(temporary, { force: true }); }
}
-/**
- * Name how a relay request failed before any response, from the error's own fields only.
- * Why: a bare "did not answer" hid whether a request timed out, was refused or lost its socket,
- * so load-dependent relay failures could not be attributed (#512).
- */
-function transportFailure(error) {
- if (error?.name === 'TimeoutError' || error?.name === 'AbortError') return 'timed out';
- const code = error?.cause?.code ?? error?.code;
- return typeof code === 'string' && /^[A-Z][A-Z0-9_]{1,40}$/u.test(code) ? code : 'no connection';
-}
-
/** Use the page adapter only for snapshot requests; the saved opt-in still owns every network call. */
export async function request(state, path, options = {}, io) {
return relaySnapshotRequest(state, path, options, io, {
@@ -453,12 +442,6 @@ function recoverySummary(state) {
return { pending: skip > 0, skip, next: skip > 0 ? `pullboard relay recover --skip ${skip}` : '' };
}
-/** The queued moves and replay refusals pullboard status shows, when there are any (#922). */
-function outboxFields(state) {
- return { ...(state?.outbox?.moves?.length ? { outbox: structuredClone(state.outbox.moves) } : {}),
- ...(state?.outboxRefusals?.length ? { outboxRefusals: structuredClone(state.outboxRefusals) } : {}) };
-}
-
/** Expose key-free link status; only an explicit link operation includes its pairing carrier. */
function summary(root, state, { pairing = false } = {}) {
if (!state) {
@@ -1115,20 +1098,6 @@ async function sendPending(root, file, state, io) {
if (state.pending) throw new Refused('RELAY_BUSY', 'the relay order stayed busy; retry this command after other clients finish');
}
-/** Choose credentials only; main's relay sender policy remains the sole replay authorization. */
-function credentialActor(move) {
- const args = move.args;
- if (move.operation === 'register' && move.engine >= VERIFIER_STANDING_ENGINE) {
- const creator = args?.[0]?.createdBy;
- return typeof creator === 'string' && creator !== 'machine' ? creator : 'machine';
- }
- if (move.personRequest || ['register', 'ensureCoordinator'].includes(move.operation)) return 'machine';
- if (['recordRowDecisions', 'recordLandingWaiver'].includes(move.operation)
- || move.operation === 'answerDecision' && args[1]?.asPerson
- || move.operation === 'shout' && args[0]?.request) return 'person';
- return relayMoveActor(move);
-}
-
/** Set up this machine's coordinator token inside a paired join's own person sign-in (#579). */
export async function relayPersonCoordinator(root, session, io) {
const file = linkFile(root);
@@ -1138,19 +1107,6 @@ export async function relayPersonCoordinator(root, session, io) {
});
}
-/** Compare caller intent without executing frozen callbacks or allocating another operation id. */
-function operationIntent(operation, args) {
- const values = Array.isArray(args) ? args.map((value, index) => {
- if (!['claim', 'refreeze'].includes(operation) || index !== 1 || !value || typeof value !== 'object') return value;
- const options = { ...value };
- delete options.freeze;
- delete options.frozen;
- delete options.freezeError;
- return options;
- }) : args;
- return JSON.stringify({ operation, args: values }, (_key, value) => typeof value === 'function' ? undefined : value);
-}
-
/** Keep a recovered outcome until its caller repeats the interrupted operation to receive it. */
function recoveredMove(root, state, operation, args, command) {
if (!state.recovered) return null;
@@ -1182,167 +1138,9 @@ function keepMoveForLaterBaseline(file, state, error, io) {
io.err(`pullboard: [${safePauseCode(error)}] the move is saved on this machine; ${error.transportFailure ? `the relay did not answer (${error.transportFailure})` : safePauseMessage(error)}; it goes up inside the board baseline on a later command once the relay answers`);
}
-/** Whether this command may queue a move in the ordered outbox: the board's relay.outbox setting, off by default (#922). */
-function outboxEnabled(io) {
- try { return io?.relayOutbox?.() === true; } catch { return false; }
-}
-
-/**
- * Whether a failure on an accepted link may queue the move instead of refusing it (#922): the relay said nothing
- * definite, and the move needs no relay answer to exist (a registration mints its token there; a person's request
- * is the phone's to send).
- */
-function outboxQueueable(state, error, operation, io) {
- return state.mode === 'ordered' && state.baselineAccepted === true && !state.baselineConflict && !io?.personRequest
- && !['register', 'ensureCoordinator'].includes(operation) && transientRelayFailure(error) && outboxEnabled(io);
-}
-
-/** The board's relay-applied sequence, read from an exported image. */
-function exportedSequence(native) {
- return Number(native.tables.board_meta.find((row) => row.meta_key === 'relay_applied_sequence')?.meta_value ?? 0);
-}
-
-/** Keep the key-free outbox summary on the link, or drop it once the outbox is empty. */
-function saveOutboxSummary(file, state, outbox) {
- if (outbox?.moves.length) state.outbox = { restored: outbox.restored, moves: outbox.moves.map(outboxEntrySummary) };
- else delete state.outbox;
- saveLink(file, state);
-}
-
-/**
- * Queue a move the relay could not take on an accepted link, then make it on this machine (#922).
- * The first queued move seals the board as the relay last ordered it; the move is written before it is applied,
- * so a crash between the two leaves it queued and the replay starts from that image anyway. A move this board
- * refuses is refused now and never queued.
- */
-async function queueMove(root, file, state, io, { operation, args, command, move, error, applyLocally }) {
- const key = readBoardKey(state.board);
- const where = outboxFile(file);
- const outbox = await readOutbox(where, key, state.board) ?? (() => {
- const base = localRecords(root, (board) => exportBoard(board));
- return { version: 1, base, baseSequence: exportedSequence(base), restored: false, moves: [] };
- })();
- if (!move) {
- const sent = withSeatName(root, operation, args);
- move = localRecords(root, (board) => prepareEngineMove(board, operation, sent));
- move.actor = relayMoveActor(move) ?? credentialActor(move);
- }
- // `after` is the relay sequence this machine had seen: a refusal at replay names the moves ordered since.
- const entry = { move, intent: operationIntent(operation, args), queuedAt: new Date().toISOString(), after: state.sequence,
- ...(command ? { command, commandIntent: operationIntent('cli:' + command.cliOperation, command) } : {}) };
- outbox.moves.push(entry);
- await writeOutbox(where, key, state.board, outbox);
- if (state.pending?.move?.id === move.id) delete state.pending;
- saveOutboxSummary(file, state, outbox);
- let result;
- try { result = await applyLocally(); }
- catch (refusal) {
- outbox.moves.pop();
- if (outbox.moves.length) await writeOutbox(where, key, state.board, outbox);
- else removeOutbox(where);
- saveOutboxSummary(file, state, outbox);
- throw refusal;
- }
- if (!OUTBOX_NOTICES.has(io)) {
- OUTBOX_NOTICES.add(io);
- io.err(`pullboard: [${safePauseCode(error)}] ${error.transportFailure ? `the relay did not answer (${error.transportFailure})` : safePauseMessage(error)}; the move is saved on this machine and waits in the relay outbox (${outbox.moves.length} pending); it goes up in order when the relay answers, and pullboard status lists it`);
- }
- return result;
-}
-
-/**
- * Put the queued moves back on top of the board after a replay stopped part way (#922): seal the board as the relay
- * has now ordered it, then apply each remaining move here again, so this machine keeps showing them.
- */
-async function rebaseOutbox(root, file, state, outbox, key) {
- outbox.base = localRecords(root, (board) => exportBoard(board));
- outbox.baseSequence = exportedSequence(outbox.base);
- outbox.restored = false;
- await writeOutbox(outboxFile(file), key, state.board, outbox);
- saveOutboxSummary(file, state, outbox);
- for (const entry of outbox.moves) {
- localRecords(root, (board) => {
- try { store.atomic(board, () => store[entry.move.operation](board, ...executableArgs(entry.move, board))); }
- catch (error) { if (!(error instanceof Refused)) throw error; }
- });
- }
-}
-
-/**
- * Replay the ordered outbox before this machine applies or sends anything else (#922). The board goes back to the
- * image the relay last ordered, catches up with every other machine's moves, then sends each queued move in order:
- * a move the relay already holds from a lost reply is not sent twice, and a move refused at replay is reported with
- * the moves ordered before it. A relay that fails again part way puts the rest back on top and throws its failure.
- */
-async function flushOutbox(root, file, state, io) {
- const where = outboxFile(file);
- if (!state.outbox && !existsSync(where)) return;
- const key = readBoardKey(state.board);
- const outbox = await readOutbox(where, key, state.board);
- if (!outbox?.moves.length) {
- removeOutbox(where);
- saveOutboxSummary(file, state, null);
- return;
- }
- try {
- if (!outbox.restored) {
- // Probe first: a relay that is still down leaves this machine's board exactly as its person last saw it.
- try { await request(state, '/api/v1/boards/' + state.board + '/events?after=' + outbox.baseSequence, {}, io); }
- catch (error) { if (error.code !== 'SNAPSHOT_REQUIRED') throw error; }
- localRecords(root, (board) => {
- const cursors = board.db.prepare('SELECT agent_id, agent_last_shout_id FROM agent').all();
- restoreRelaySnapshot(board, outbox.base, outbox.baseSequence);
- // Reading shouts is this machine's own progress; the restore keeps the newer read position.
- const keep = board.db.prepare('UPDATE agent SET agent_last_shout_id = ? WHERE agent_id = ? AND (agent_last_shout_id IS NULL OR agent_last_shout_id < ?)');
- for (const row of cursors) if (Number.isSafeInteger(row.agent_last_shout_id)) keep.run(row.agent_last_shout_id, row.agent_id, row.agent_last_shout_id);
- });
- state.sequence = outbox.baseSequence;
- outbox.restored = true;
- await writeOutbox(where, key, state.board, outbox);
- saveOutboxSummary(file, state, outbox);
- }
- await catchUp(root, file, state, io);
- while (outbox.moves.length) {
- const entry = outbox.moves[0];
- const receiptOf = () => localRecords(root, (board) => engineReceipt(board, entry.move.id));
- let receipt = receiptOf();
- let sequence = receipt?.sequence;
- if (!receipt) {
- await actorTransport(root, file, state, credentialActor(entry.move), io);
- sequence = state.sequence + 1;
- const sealed = await sealedRecord(key, entry.move, state, 'move', sequence);
- const registration = seatReceipt(entry.move, sequence, sealed);
- state.pending = { move: entry.move, intent: entry.intent, ...(entry.command ? { command: entry.command, commandIntent: entry.commandIntent } : {}),
- sequence, sealed, ...(registration ? { registration } : {}) };
- saveLink(file, state);
- let failure;
- try { await sendPending(root, file, state, io); } catch (error) { failure = error; }
- if (failure && !receiptOf()) {
- if (!['RELAY_UNAVAILABLE', 'RELAY_RESPONSE'].includes(failure.code)) throw failure;
- try { await catchUp(root, file, state, io); } catch { throw failure; }
- }
- receipt = receiptOf();
- if (!receipt) throw failure ?? new Refused('RELAY_RESPONSE', 'the relay acknowledged a queued move without its replay receipt; run pullboard status to retry the outbox');
- sequence = receipt.sequence;
- }
- if (state.pending?.move?.id === entry.move.id) delete state.pending;
- if (state.recovered?.move?.id === entry.move.id) delete state.recovered;
- if (receipt.outcome.error) {
- const earlier = localRecords(root, (board) => earlierMoves(board, entry, sequence));
- io.err(`pullboard: ${keepRefusal(state, entry, receipt.outcome.error, earlier)}`);
- }
- outbox.moves.shift();
- if (outbox.moves.length) await writeOutbox(where, key, state.board, outbox);
- else removeOutbox(where);
- saveOutboxSummary(file, state, outbox);
- }
- } catch (error) {
- if (outbox.restored && outbox.moves.length) {
- if (state.pending && outbox.moves.some((entry) => entry.move.id === state.pending.move?.id)) delete state.pending;
- await rebaseOutbox(root, file, state, outbox, key);
- }
- throw error;
- }
+/** The relay transport the ordered outbox queues and replays through (#922). */
+function outboxLink(root, file, state, io) {
+ return { root, file, state, io, request, catchUp, sendPending, sealedRecord, localRecords, saveLink, safePauseCode, safePauseMessage };
}
/** Send one board operation before executing it locally, and report its canonical engine result. */
@@ -1366,7 +1164,7 @@ export async function relayOperation(root, operation, args, io, command, applyLo
moves.add(state.board);
try {
let ordered;
- const queue = (error, move) => queueMove(root, file, state, io, { operation, args, command, move, error, applyLocally: applyPausedMoveLocally });
+ const queue = (error, move) => queueMove(outboxLink(root, file, state, io), { operation, args, command, move, error, applyLocally: applyPausedMoveLocally });
try { ordered = await ensureOrdered(root, file, state, io); }
catch (error) {
if (outboxQueueable(state, error, operation, io)) return await queue(error);
@@ -1381,7 +1179,7 @@ export async function relayOperation(root, operation, args, io, command, applyLo
}
try {
// #922: moves queued while the relay was down go up first, in the order they were made.
- await flushOutbox(root, file, state, io);
+ await flushOutbox(outboxLink(root, file, state, io));
await catchUp(root, file, state, io);
} catch (error) {
if (outboxQueueable(state, error, operation, io)) return await queue(error);
@@ -1473,7 +1271,7 @@ export async function syncRelay(root, io) {
try {
if (state.recovery) throw new Refused('RELAY_RECOVERY_PENDING', `a relay recovery for sequence ${state.recovery.skip} is pending; retry pullboard relay recover --skip ${state.recovery.skip} before syncing`);
if (!await ensureOrdered(root, file, state, io)) return summary(root, state);
- await flushOutbox(root, file, state, io);
+ await flushOutbox(outboxLink(root, file, state, io));
await catchUp(root, file, state, io);
if (state.pending) {
// The complete prefix did not acknowledge this interrupted send. It must not run later.
@@ -1866,7 +1664,7 @@ export async function relayOn(root, address, io, { session, credential, quiet =
if (ready) ready = await ensureOrdered(root, file, state, io);
if (ready) {
rmSync(deletedLinkFile(root), { force: true });
- await flushOutbox(root, file, state, io);
+ await flushOutbox(outboxLink(root, file, state, io));
await catchUp(root, file, state, io);
await deliverDeviceWraps(root, io);
updateRelayMachine(machine => { machine.excluded = machine.excluded.filter(entry => entry !== repoInfo(root).root); });
diff --git a/test/relay-outbox.test.js b/test/relay-outbox.test.js
index 19dab651..261dd670 100644
--- a/test/relay-outbox.test.js
+++ b/test/relay-outbox.test.js
@@ -3,6 +3,7 @@ import { readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';
+import { remapMove } from '../src/relay-outbox.js';
import { runFixtureGit } from './fixture-child.js';
import { relayClientFixture } from './relay-client-fixture.js';
@@ -23,14 +24,20 @@ function shoutTexts(exported) {
return exported.tables.shout.map((row) => row.shout_text);
}
-test('a move on an accepted relay link waits in the ordered outbox while the relay is down, and replays in order [B26,H16,H17]', async (t) => {
+/** Two machines on one accepted relay board with the outbox on, and an item added before the outage. */
+async function twoMachines(t, title) {
const box = await relayClientFixture(t);
enableOutbox(box);
await box.link();
- const contested = (await box.cli('add', box.lane, 'Contested while the relay is down')).document.item.item_id;
+ const contested = (await box.cli('add', box.lane, title)).document.item.item_id;
const printed = await box.cli('relay', 'pair');
const other = await box.otherDeviceJoin(printed.document.code);
assert.equal(JSON.parse(readFileSync(box.linkFile, 'utf8')).baselineAccepted, true, 'the relay accepted this board before the outage');
+ return { box, other, contested };
+}
+
+test('a move on an accepted relay link waits in the ordered outbox while the relay is down, and replays in order [B26,H16,H17]', async (t) => {
+ const { box, other, contested } = await twoMachines(t, 'Contested while the relay is down');
await box.stopRelay();
const first = await box.cli('shout', 'all', 'queued first while the relay is down');
@@ -69,3 +76,46 @@ test('a move on an accepted relay link waits in the ordered outbox while the rel
assert.deepEqual(item(here.document), item(there.document), 'both machines agree on the contested item');
assert.equal(item(there.document).item_status, 'withdrawn');
});
+
+test('an item added while the relay is down keeps its queued moves when another machine adds one first [B26,H16,H17]', async (t) => {
+ const { box, other } = await twoMachines(t, 'Added before the outage');
+ await box.stopRelay();
+ const added = await box.cli('add', box.lane, 'Added here while the relay is down');
+ assert.equal(added.code, 0, JSON.stringify(added.document));
+ const local = added.document.item.item_id;
+ const second = await box.cli('add', box.lane, 'Also added here while the relay is down');
+ assert.equal(second.document.item.item_id, local + 1, 'the second offline add takes the next local number');
+ const withdrawn = await box.cli('withdraw', String(local), 'withdrawn here while the relay is down');
+ assert.equal(withdrawn.code, 0, withdrawn.failure);
+
+ await box.restartRelay();
+ const first = await other.cli('add', box.lane, 'Added there first');
+ assert.equal(first.code, 0, first.failure);
+ assert.equal(first.document.item.item_id, local, 'the other machine takes the number this machine used offline');
+
+ const replayed = await box.cli('status');
+ assert.equal(replayed.code, 0, replayed.failure);
+ assert.equal(replayed.document.relay.outbox, undefined, 'the outbox is empty once it replays');
+ assert.equal(replayed.document.relay.outboxRefusals, undefined, 'nothing is refused: the queued withdraw follows its own item');
+ assert.deepEqual(replayed.document.relay.outboxRenumbered, [{ was: local, is: local + 1 }, { was: local + 1, is: local + 2 }],
+ 'status says each offline item got the next number');
+ for (const machine of [box, other]) {
+ const items = new Map((await machine.cli('export')).document.tables.item.map((row) => [row.item_id, row]));
+ assert.deepEqual([items.get(local).item_title, items.get(local).item_status], ['Added there first', 'open'], 'the other machine\'s item is untouched');
+ assert.deepEqual([items.get(local + 1).item_title, items.get(local + 1).item_status], ['Added here while the relay is down', 'withdrawn'], 'the queued withdraw acts on the item it was made for');
+ assert.deepEqual([items.get(local + 2).item_title, items.get(local + 2).item_status], ['Also added here while the relay is down', 'open'],
+ 'a later renumbering never moves a target that was already mapped');
+ }
+});
+
+test('a queued move maps each item it names once, from the number it was made with [B26,H16]', () => {
+ const remap = { 2: 3, 3: 4 };
+ const withdraw = { id: 'w', operation: 'withdraw', args: [2, { agentId: 'coordinator', reason: 'r' }] };
+ assert.equal(remapMove(withdraw, remap, []).move.args[0], 3, 'local #2 is #3 on the board, not #4');
+ assert.equal(withdraw.args[0], 2, 'the move as it was made is kept, so a retry maps it again from #2');
+ const child = { id: 'c', operation: 'addItem', args: [{ by: 'coordinator', lane: 'app', title: 't', parentId: 3, after: [2, 9] }] };
+ assert.deepEqual(remapMove(child, remap, []).move.args[0], { by: 'coordinator', lane: 'app', title: 't', parentId: 4, after: [3, 9] });
+ const answer = { id: 'a', operation: 'answerDecision', args: [2, { agentId: 'coordinator', answer: 'yes' }] };
+ assert.equal(remapMove(answer, remap, []).move.args[0], 2, 'a decision number is a shout, never an item');
+ assert.deepEqual(remapMove(withdraw, {}, [2]).lost, [2], 'a move on an item whose add was refused is named');
+});