Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(relay): a lost baseline reply is decided before any move [B26,H16]

Corey Olson1bccea31c8ee
diff --git a/src/next-steps.js b/src/next-steps.js
index 68dcb2b3..f34f68f5 100644
--- a/src/next-steps.js
+++ b/src/next-steps.js
@@ -446,6 +446,7 @@ export const NEXT_STEPS = Object.freeze({
RELAY_ACTOR: 'send the move with the token of the agent that makes it',
RELAY_ANSWER: 'use pullboard answer <id> "<answer>" or pullboard pass <id> --note "<reason>"',
RELAY_BASELINE_EXISTS: 'save the board with pullboard export > board.json, then pullboard relay off',
+ RELAY_BASELINE_UNCERTAIN: 'run the command again when the relay answers; it first checks whether the relay kept the board upload',
RELAY_BOARD_DELETED: 'keep working locally, or run pullboard relay on to link the board again',
RELAY_BUSY: 'run the command again after the other worktree finishes sending',
RELAY_CURSOR: 'fetch a consistent snapshot: pullboard relay',
diff --git a/src/relay-baseline-settle.js b/src/relay-baseline-settle.js
new file mode 100644
index 00000000..8e42bb29
--- /dev/null
+++ b/src/relay-baseline-settle.js
@@ -0,0 +1,64 @@
+/** Decide a first board upload whose outcome was lost, before any move may commit outside relay order (#919). */
+import { Refused } from './refused.js';
+import { transientRelayFailure } from './relay-move-fallback.js';
+
+/**
+ * How long an undecided upload is probed, how often the waiting command says so, and the pause between probes.
+ * Why: the criterion bounds the wait at 2 minutes; one probe may still run its own 10 s transport ceiling
+ * after the last start, so probing stops at 110 s.
+ */
+const SETTLE_WAIT_MS = 110000;
+const SETTLE_PROGRESS_MS = 10000;
+const SETTLE_RETRY_MS = 2000;
+
+/**
+ * Whether a failed first-upload send leaves its outcome unknown: no reply, a reply that is no acknowledgement,
+ * a gateway that may have forwarded it, or an error raised by this client after the request went out.
+ * Why: a relay that answers with its own refusal did not store the board, but a lost reply says nothing,
+ * and treating it as "not accepted" let a move commit outside an order the relay already had (#919).
+ */
+export function uncertainUploadFailure(error) {
+ if (!(error instanceof Refused)) return true;
+ if (error.transportFailure || error.code === 'RELAY_RESPONSE') return true;
+ return [502, 504].includes(error.httpStatus);
+}
+
+/** Whether a state read is the relay's authenticated statement that it holds no board here. */
+export function remoteBoardMissing(error) {
+ return error?.httpStatus === 404 && error.relayVersion === 1 && ['NO_BOARD', 'NO_SNAPSHOT'].includes(error.code);
+}
+
+/**
+ * Probe the relay until it says whether it holds this board's pending first snapshot: 'accepted' when its state
+ * is that exact sealed snapshot at sequence 0, 'absent' when it holds no board, 'other' for any other state.
+ * A relay that keeps failing to answer ends in one RELAY_BASELINE_UNCERTAIN refusal naming the cause and the retry.
+ * Why: the caller holds the link lock, so the decision is made once, here, instead of by a later local guess.
+ */
+export async function settleUncertainBaseline(state, io, read) {
+ const startedAt = Date.now();
+ const wait = io?.relayUncertainWaitMs ?? SETTLE_WAIT_MS;
+ const every = io?.relayUncertainProgressMs ?? SETTLE_PROGRESS_MS;
+ const pause = io?.relayUncertainRetryMs ?? SETTLE_RETRY_MS;
+ const seconds = (ms) => Math.round(ms / 1000);
+ const progress = typeof io?.err === 'function'
+ ? setInterval(() => io.err(`pullboard: checking whether the relay kept this board's first upload (${seconds(Date.now() - startedAt)} s of ${seconds(wait)} s)`), every)
+ : null;
+ let cause = 'no answer';
+ try {
+ for (;;) {
+ try {
+ const saved = (await read())?.state;
+ return saved?.sequence === 0 && typeof saved.sealed === 'string' && saved.sealed === state.snapshot?.sealed ? 'accepted' : 'other';
+ } catch (error) {
+ if (remoteBoardMissing(error)) return 'absent';
+ if (!transientRelayFailure(error)) throw error;
+ cause = error.transportFailure ?? (Number.isInteger(error.httpStatus) ? `HTTP ${error.httpStatus}` : error.code);
+ }
+ if (Date.now() - startedAt + pause >= wait) break;
+ await new Promise((resolve) => setTimeout(resolve, pause));
+ }
+ } finally {
+ if (progress) clearInterval(progress);
+ }
+ throw new Refused('RELAY_BASELINE_UNCERTAIN', `the relay may already hold this board's first upload but did not say so within ${seconds(Date.now() - startedAt)} s (${cause}); the move was not saved, so run the command again when the relay answers`);
+}
diff --git a/src/relay-move-fallback.js b/src/relay-move-fallback.js
index a0a2d574..fd11daf6 100644
--- a/src/relay-move-fallback.js
+++ b/src/relay-move-fallback.js
@@ -1,9 +1,12 @@
/** When a relay failure may let a local board move commit without the relay (#919, #341). */
import { Refused } from './refused.js';
-/** Whether the relay has not yet accepted this board's first snapshot, so no ordered history exists to protect. */
+/**
+ * Whether the relay has definitely not accepted this board's first snapshot, so no ordered history exists to protect.
+ * Why: an upload whose reply was lost may already be the relay's baseline, so it never counts as unaccepted (#919).
+ */
export function baselineNotAccepted(state) {
- return state.mode === 'ordered' && state.baselineAccepted === false && !state.baselineConflict;
+ return state.mode === 'ordered' && state.baselineAccepted === false && !state.baselineConflict && !state.baselineUncertain;
}
/**
diff --git a/src/relay.js b/src/relay.js
index 280fb2fd..c3455d94 100644
--- a/src/relay.js
+++ b/src/relay.js
@@ -84,6 +84,11 @@ import {
} from './seal.js';
import { terminalQr } from './qr.js';
import { locked } from './relay-lock.js';
+import {
+ remoteBoardMissing,
+ settleUncertainBaseline,
+ uncertainUploadFailure,
+} from './relay-baseline-settle.js';
import {
baselineNotAccepted,
transientRelayFailure,
@@ -596,7 +601,7 @@ function rememberBaselineFailure(io, file, error) {
async function remoteBaselineAbsent(state, io) {
try { await request(state, '/api/v1/boards/' + state.board + '/state', {}, io); }
catch (error) {
- if (error.httpStatus === 404 && error.relayVersion === 1 && ['NO_BOARD', 'NO_SNAPSHOT'].includes(error.code)) return true;
+ if (remoteBoardMissing(error)) return true;
throw error;
}
return false;
@@ -618,6 +623,7 @@ function fenceBaseline(file, state) {
state.baselineAccepted = true;
state.baselineConflict = true;
delete state.baselinePause;
+ delete state.baselineUncertain;
saveLink(file, state);
requireUnconflictedBaseline(state);
}
@@ -629,28 +635,67 @@ async function requireAbsentBaseline(file, state, io) {
}
}
+/** Record that the relay holds the pending sequence-zero snapshot, from its ACK or an authenticated read of it. */
+function acceptBaseline(root, file, state) {
+ state.baselineAccepted = true;
+ state.cursor = Number.isSafeInteger(state.baselineCursor) ? state.baselineCursor : localRecords(root, (board) => board.db.prepare('SELECT COALESCE(MAX(event_id), 0) AS id FROM event').get().id);
+ delete state.baselineCursor;
+ const resumed = Boolean(state.baselinePause);
+ delete state.baselinePause;
+ delete state.baselineStale;
+ delete state.baselineUncertain;
+ delete state.snapshot;
+ state.presentationDigest = state.snapshotPresentationDigest ?? state.presentationDigest;
+ delete state.snapshotPresentationDigest;
+ state.needsPresentation = false;
+ saveLink(file, state);
+ if (resumed) clearUploadPauseNotices(state);
+}
+
+/**
+ * Decide, under the link lock, whether the relay kept a first upload whose reply was lost: true when it holds
+ * that exact snapshot, false when it holds no board; another board fences, and no answer refuses once.
+ * Why: a client that only saw no reply cannot know the relay has no ordered history, so no move may commit
+ * locally until this says so (#919).
+ */
+async function decideUncertainBaseline(root, file, state, io) {
+ try {
+ const verdict = await settleUncertainBaseline(state, io, () => request(state, '/api/v1/boards/' + state.board + '/state', {}, io));
+ if (verdict === 'other') fenceBaseline(file, state);
+ if (verdict === 'accepted') { acceptBaseline(root, file, state); return true; }
+ delete state.baselineUncertain;
+ saveLink(file, state);
+ return false;
+ } catch (error) {
+ rememberBaselineFailure(io, file, error);
+ throw error;
+ }
+}
+
/** Send the sequence-zero board image; only its first permanent HTTP refusal pauses relay sends. */
async function uploadInitialBaseline(root, file, state, io) {
attemptedBaselineThisCommand(io, file);
+ let sent = false;
try {
await requireAbsentBaseline(file, state, io);
const pending = state.snapshot;
+ // Written ahead: from here until a definite answer, the relay may hold this snapshot (#919).
+ state.baselineUncertain = true;
+ saveLink(file, state);
+ sent = true;
const reply = await request(state, '/api/v1/boards/' + state.board + '/state', { method: 'PUT', body: pending, initialBaseline: true }, io);
if (reply.state?.sequence !== pending.sequence || reply.state.sealed !== pending.sealed) throw new Refused('RELAY_RESPONSE', 'the relay did not acknowledge the initial snapshot; retry the configured relay');
- state.baselineAccepted = true;
- state.cursor = Number.isSafeInteger(state.baselineCursor) ? state.baselineCursor : localRecords(root, (board) => board.db.prepare('SELECT COALESCE(MAX(event_id), 0) AS id FROM event').get().id);
- delete state.baselineCursor;
- const resumed = Boolean(state.baselinePause);
- delete state.baselinePause;
- delete state.baselineStale;
- delete state.snapshot;
- state.presentationDigest = state.snapshotPresentationDigest ?? state.presentationDigest;
- delete state.snapshotPresentationDigest;
- state.needsPresentation = false;
- saveLink(file, state);
- if (resumed) clearUploadPauseNotices(state);
+ acceptBaseline(root, file, state);
return true;
} catch (error) {
+ if (sent && state.baselineUncertain) {
+ if (uncertainUploadFailure(error)) {
+ if (await decideUncertainBaseline(root, file, state, io)) return true;
+ } else {
+ delete state.baselineUncertain;
+ saveLink(file, state);
+ }
+ }
if (error.relayVersion === 1 && error.code === 'BASELINE_EXISTS') fenceBaseline(file, state);
if (!permanentUploadRefusal(error) || state.baselineAccepted) {
rememberBaselineFailure(io, file, error);
@@ -886,6 +931,8 @@ async function ensureOrdered(root, file, state, io) {
if (state.baselineAccepted === undefined) {
state.baselineAccepted = !(state.snapshot?.sequence === 0);
}
+ // An earlier command lost the reply to its first upload: decide it before re-sealing or sending anything.
+ if (state.baselineUncertain && !state.baselineAccepted) await decideUncertainBaseline(root, file, state, io);
if (state.baselinePause || state.baselineStale) {
if (!await retryPausedBaseline(root, file, state, io)) return false;
}
diff --git a/test/relay-client-fixture.js b/test/relay-client-fixture.js
index 85718eb2..9e49ee54 100644
--- a/test/relay-client-fixture.js
+++ b/test/relay-client-fixture.js
@@ -233,6 +233,8 @@ globalThis.fetch = function tracedFetch(input, init) {
let stateReadStarted = 0;
let refuseSnapshotWrites = false;
let stallApi = false;
+ let loseSnapshotReply = null;
+ const lostSnapshotReplies = [];
let rejectSnapshotWrites = null;
let rejectedStateReads = null;
let legacySnapshotApi = false;
@@ -344,7 +346,16 @@ globalThis.fetch = function tracedFetch(input, init) {
return writeHead(acceptedApprovalCompletionReturnedAsRefusal ? 403 : status, ...args);
};
const end = res.end.bind(res);
+ const loseThisReply = snapshotWrite && loseSnapshotReply;
+ if (loseThisReply) loseSnapshotReply = null;
res.end = (chunk, ...args) => {
+ if (loseThisReply && res.statusCode === 200) {
+ // The real service stored the snapshot; only its acknowledgement is lost on the wire (#919).
+ lostSnapshotReplies.push(snapshotRecord.ciphertext);
+ if (loseThisReply.thenStateReads) rejectedStateReads = { ...loseThisReply.thenStateReads };
+ req.socket.destroy();
+ return res;
+ }
if (acceptedApprovalCompletionReturnedAsRefusal) {
chunk = JSON.stringify({ version: 1, error: { code: 'REVIEWER_COMPLETE_REFUSED',
message: 'The completion response was refused after sealed delivery.',
@@ -744,6 +755,13 @@ globalThis.fetch = function tracedFetch(input, init) {
refuseSnapshotWrites(value) { refuseSnapshotWrites = value; },
/** Leave every board API request unanswered until its client times out, like an overloaded relay. */
stallApi(value = true) { stallApi = value; },
+ /**
+ * Let the real relay accept the next API1 snapshot PUT, then drop its reply by closing the socket,
+ * optionally answering later state reads with a chosen refusal, as the #919 lost-ACK review did.
+ */
+ loseNextSnapshotReply(options = {}) { loseSnapshotReply = { thenStateReads: options.thenStateReads ?? null }; },
+ /** SHA-256 digests of the snapshots the relay accepted while their replies were lost. */
+ lostSnapshotReplies() { return [...lostSnapshotReplies]; },
/** Restart only the private service with a changed operator quota, retaining real journals and credentials. */
setStorageQuota(bytes) {
assert.ok(Number.isSafeInteger(bytes) && bytes > 0);
diff --git a/test/relay-sync.test.js b/test/relay-sync.test.js
index 76043a1f..71b6a234 100644
--- a/test/relay-sync.test.js
+++ b/test/relay-sync.test.js
@@ -1,8 +1,10 @@
/** A relay timeout, a busy link lock or an uncertain baseline never fails a local board move [B26,H16,H17]. */
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { describe, test } from 'node:test';
+import { ENGINE_VERSION } from '../src/machine.js';
import { relayClientFixture } from './relay-client-fixture.js';
const RELAY = JSON.stringify(new URL('../src/relay.js', import.meta.url).href);
@@ -45,6 +47,27 @@ async function uncertainLink(t) {
return box;
}
+/** Pause the first baseline with a refusal whose retry reaches the relay, as the #919 lost-ACK review set it up. */
+async function refusedLink(t) {
+ const box = await relayClientFixture(t);
+ box.legacySnapshotApi('bad-route');
+ box.rejectSnapshots({ status: 400, code: 'BAD_UPLOAD', message: 'fixture refused the first upload' });
+ await box.link();
+ box.rejectSnapshots(null);
+ assert.ok(link(box).baselinePause, 'the refused baseline is paused');
+ return box;
+}
+
+/** Read the relay's saved board state with the link's own credential, outside the client under test. */
+async function remoteState(box, state) {
+ const reply = await fetch(box.origin + '/api/v1/boards/' + state.board + '/state', {
+ headers: { authorization: 'Bearer ' + state.token, 'x-pullboard-engine': String(ENGINE_VERSION) },
+ });
+ const document = await reply.json();
+ return { status: reply.status, sequence: document.state?.sequence ?? null,
+ digest: typeof document.state?.sealed === 'string' ? createHash('sha256').update(document.state.sealed).digest('hex') : null };
+}
+
/** After the relay answers again, every local move must be inside the one accepted baseline. */
async function assertUploadedLater(box, texts) {
box.stallApi(false); box.legacySnapshotApi(false);
@@ -86,6 +109,51 @@ for (const [name, setup] of [['a paused link', pausedLink], ['a link whose first
});
}
+for (const [name, setup] of [['a link whose first upload is uncertain', uncertainLink], ['a paused link', refusedLink]]) {
+ test(`a lost baseline reply never lets a move commit outside relay order on ${name} [B26,H16]`, async t => {
+ const box = await setup(t);
+ box.legacySnapshotApi(true);
+ const before = link(box);
+ assert.equal(before.baselineAccepted, false);
+ box.loseNextSnapshotReply({ thenStateReads: { status: 503, code: 'RELAY_UNAVAILABLE' } });
+ const text = 'must stay in relay order once the relay may hold the baseline';
+ const run = await moves(box, [text], { relayUncertainWaitMs: 1500, relayUncertainProgressMs: 400, relayUncertainRetryMs: 200 });
+ assert.equal(run.code, 0, run.stderr);
+ assert.equal(box.lostSnapshotReplies().length, 1, 'the relay accepted one baseline whose reply was lost');
+ assert.equal(run.document.outcomes[0]?.code, 'RELAY_BASELINE_UNCERTAIN', 'an undecided upload refuses instead of committing locally');
+ assert.match(run.document.outcomes[0].message, /run the command again/u);
+ assert.deepEqual(run.document.local, [], 'nothing commits outside the relay order');
+ assert.equal(box.moveAcks.length, 0);
+ assert.ok(run.document.lines.filter(line => /checking whether the relay kept/u.test(line)).length >= 2, 'progress lines appear while deciding');
+ assert.ok(link(box).baselineUncertain, 'the undecided upload is saved for the next command');
+ box.rejectStateReads(null);
+ const remote = await remoteState(box, before);
+ assert.deepEqual([remote.status, remote.sequence, remote.digest], [200, 0, box.lostSnapshotReplies()[0]], 'the relay holds the baseline whose reply was lost');
+ const retried = await moves(box, [text]);
+ assert.deepEqual(retried.document.outcomes, ['ok'], 'the retry decides the upload under the lock, then sends the move');
+ assert.deepEqual(retried.document.local, [text]);
+ assert.equal(box.moveAcks.length, 1, 'the retried move went through relay order');
+ assert.equal(link(box).baselineAccepted, true);
+ assert.equal(link(box).baselineUncertain, undefined);
+ });
+}
+
+test('the public CLI waits out a lost baseline reply and sends the move in relay order [B26,H16]', async t => {
+ const box = await uncertainLink(t);
+ box.legacySnapshotApi(true);
+ box.loseNextSnapshotReply({ thenStateReads: { status: 503, code: 'RELAY_UNAVAILABLE' } });
+ const recover = setTimeout(() => box.rejectStateReads(null), 1500); // budget: hang ceiling for the outage the uncertain upload waits out.
+ t.after(() => clearTimeout(recover));
+ const text = 'public CLI move after a lost baseline reply';
+ const result = await box.cli('shout', 'all', text);
+ assert.equal(result.code, 0, JSON.stringify(result.document));
+ assert.equal(box.lostSnapshotReplies().length, 1, 'the relay accepted one baseline whose reply was lost');
+ assert.equal(box.moveAcks.length, 1, 'the move went through relay order once the upload was decided');
+ assert.equal(link(box).baselineAccepted, true);
+ const exported = await box.cli('export');
+ assert.equal(exported.document.tables.shout.filter(row => row.shout_text === text).length, 1);
+});
+
test('a lock held past the wait refuses once with RELAY_BUSY and a retry step [B26,H16]', async t => {
const box = await relayClientFixture(t);
await box.link();