Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(relay): a relay timeout never fails a local board move [B26,H16]

Corey Olson3325b7576534
diff --git a/src/relay.js b/src/relay.js
index d9593c76..2f5b38eb 100644
--- a/src/relay.js
+++ b/src/relay.js
@@ -120,6 +120,10 @@ export const DEFAULT_RELAY = 'https://app.pullboard.dev';
const LINK_FILES = new WeakMap();
const STATE_FILES = new WeakMap();
const WARNED_COMMANDS = new WeakSet();
+/** Longest wait for a sibling worktree's relay send, and how often the waiting command says so. */
+const LOCK_WAIT_MS = 120000;
+const LOCK_PROGRESS_MS = 10000;
+const MOVE_FALLBACK_NOTICES = new WeakSet();
const KEY_WARNED_COMMANDS = new WeakSet();
const BASELINE_ATTEMPTS = new WeakMap();
const BASELINE_NOTICES = new WeakSet();
@@ -311,11 +315,18 @@ export function saveLink(file, state) {
} finally { rmSync(temporary, { force: true }); }
}
-/** Wait briefly for another worktree's upload, recovering a lock only when its owner is gone. */
-async function locked(file, work) {
+/**
+ * Wait for another worktree's upload, recovering a lock only when its owner is gone.
+ * Why: one large baseline PUT can hold the lock past any short limit, and a board move that gave up
+ * after 15 s failed for a reason the person cannot fix (#919); the wait is up to 2 minutes with a progress line.
+ */
+async function locked(file, work, io) {
const lock = file + '.lock';
mkdirSync(dirname(file), { recursive: true });
- const deadline = Date.now() + 15000;
+ const startedAt = Date.now();
+ const deadline = startedAt + (io?.relayLockWaitMs ?? LOCK_WAIT_MS);
+ const progressEvery = io?.relayLockProgressMs ?? LOCK_PROGRESS_MS;
+ let nextProgress = startedAt + progressEvery;
while (true) {
try { mkdirSync(lock, { mode: 0o700 }); writeFileSync(join(lock, 'pid'), String(process.pid)); break; }
catch (error) {
@@ -329,7 +340,11 @@ async function locked(file, work) {
try { if (Date.now() - lstatSync(lock).mtimeMs > 30000) { rmSync(lock, { recursive: true, force: true }); continue; } }
catch (gone) { if (gone.code === 'ENOENT') continue; throw gone; }
}
- if (Date.now() >= deadline) throw new Refused('RELAY_BUSY', 'another worktree is sending this board; the local move is safe and the next command retries');
+ if (Date.now() >= deadline) throw new Refused('RELAY_BUSY', `another worktree was still sending this board after ${Math.round((Date.now() - startedAt) / 1000)} s; nothing was lost, so run the command again`);
+ if (io?.err && Date.now() >= nextProgress) {
+ nextProgress = Date.now() + progressEvery;
+ io.err(`pullboard: waiting for another worktree to finish sending this board (${Math.round((Date.now() - startedAt) / 1000)} s of ${Math.round((deadline - startedAt) / 1000)} s)`);
+ }
await new Promise((resolve) => setTimeout(resolve, 100));
}
}
@@ -386,7 +401,7 @@ async function rawRequest(state, path, { method = 'GET', body, allowMissing = fa
let requestSignal;
for (let attempt = 0; ; attempt += 1) {
try {
- requestSignal ??= signal ? AbortSignal.any([signal, AbortSignal.timeout(10000)]) : AbortSignal.timeout(10000); // budget: original per-request transport ceiling, shared by a replacement GET.
+ requestSignal ??= signal ? AbortSignal.any([signal, AbortSignal.timeout(io?.relayTimeoutMs ?? 10000)]) : AbortSignal.timeout(io?.relayTimeoutMs ?? 10000); // budget: original per-request transport ceiling, shared by a replacement GET.
response = await fetch(origin + path, {
method, redirect: 'error', signal: requestSignal,
headers: { 'x-pullboard-engine': String(ENGINE_VERSION), ...(token ? { authorization: 'Bearer ' + token } : {}), ...(body === undefined ? {} : { 'content-type': 'application/json' }), ...(initialBaseline ? { 'if-none-match': '*' } : {}) },
@@ -677,6 +692,7 @@ async function uploadInitialBaseline(root, file, state, io) {
delete state.baselineCursor;
const resumed = Boolean(state.baselinePause);
delete state.baselinePause;
+ delete state.baselineStale;
delete state.snapshot;
state.presentationDigest = state.snapshotPresentationDigest ?? state.presentationDigest;
delete state.snapshotPresentationDigest;
@@ -712,7 +728,7 @@ async function uploadInitialBaseline(root, file, state, io) {
/** Probe blocked limits first; only a recovered relay receives a fresh baseline containing all paused local moves. */
async function retryPausedBaseline(root, file, state, io) {
- if (!state.baselinePause || attemptedBaselineThisCommand(io, file)) return !state.baselinePause;
+ if (!(state.baselinePause || state.baselineStale) || attemptedBaselineThisCommand(io, file)) return !(state.baselinePause || state.baselineStale);
try {
if (!await uploadRetryReady(state, state.baselinePause, path => request(state, path, {}, io))) {
saveLink(file, state); reportBaselinePause(state, io); return false;
@@ -920,7 +936,7 @@ async function ensureOrdered(root, file, state, io) {
if (state.baselineAccepted === undefined) {
state.baselineAccepted = !(state.snapshot?.sequence === 0);
}
- if (state.baselinePause) {
+ if (state.baselinePause || state.baselineStale) {
if (!await retryPausedBaseline(root, file, state, io)) return false;
}
if (state.linkPending) {
@@ -1154,6 +1170,33 @@ function recoveredMove(root, state, operation, args, command) {
return recovered.move;
}
+/** Whether the relay has not yet accepted this board's first snapshot, so no ordered history exists to protect. */
+function baselineNotAccepted(state) {
+ return state.mode === 'ordered' && state.baselineAccepted === false && !state.baselineConflict;
+}
+
+/**
+ * Whether a relay failure says nothing definite about the board: a timeout, a lost connection, a busy relay or a 5xx.
+ * Why: only a definite refusal (auth, a conflicting baseline) may stop a local move before the relay accepts a baseline.
+ */
+function transientRelayFailure(error) {
+ if (!(error instanceof Refused)) return false;
+ if (['RELAY_UNAVAILABLE', 'RELAY_RESPONSE', 'RELAY_BUSY'].includes(error.code)) return true;
+ return Number.isInteger(error.httpStatus) && (error.httpStatus >= 500 || [408, 425, 429].includes(error.httpStatus));
+}
+
+/**
+ * Record that a local move happened while the baseline upload was uncertain, and say so once.
+ * Why: the saved snapshot predates the move, so the next command seals a fresh baseline that holds it (#919).
+ */
+function keepMoveForLaterBaseline(file, state, error, io) {
+ state.baselineStale = true;
+ saveLink(file, state);
+ if (MOVE_FALLBACK_NOTICES.has(io)) return;
+ MOVE_FALLBACK_NOTICES.add(io);
+ io.err(`pullboard: [${safePauseCode(error)}] the move is saved on this machine; ${error.transportFailure ? `the relay did not answer (${error.transportFailure})` : safePauseMessage(error)}; it goes up inside the board baseline on a later command once the relay answers`);
+}
+
/** Send one board operation before executing it locally, and report its canonical engine result. */
export async function relayOperation(root, operation, args, io, command, applyLocally) {
const file = linkFile(root);
@@ -1174,8 +1217,16 @@ export async function relayOperation(root, operation, args, io, command, applyLo
if (moves.has(state.board)) CHECKPOINT_NOTICES.get(io)?.delete(state.board);
moves.add(state.board);
try {
- if (!await ensureOrdered(root, file, state, io)) {
- reportBaselinePause(state, io);
+ let ordered;
+ try { ordered = await ensureOrdered(root, file, state, io); }
+ catch (error) {
+ // #341: a baseline the relay has not accepted never blocks a move; the move rides in the later baseline.
+ if (!baselineNotAccepted(state) || !transientRelayFailure(error)) throw error;
+ keepMoveForLaterBaseline(file, state, error, io);
+ ordered = false;
+ }
+ if (!ordered) {
+ if (state.baselinePause) reportBaselinePause(state, io);
return applyPausedMoveLocally();
}
await catchUp(root, file, state, io);
@@ -1241,7 +1292,7 @@ export async function relayOperation(root, operation, args, io, command, applyLo
if (error instanceof Refused && error.code === 'AUTH_REQUIRED') throw expiredSignInMoveRefusal(state, error);
throw error;
}
- });
+ }, io);
}
/** Catch up an opted-in replica before/after a CLI command; offline reads retain local evidence. */
@@ -1280,12 +1331,12 @@ export async function syncRelay(root, io) {
}
} else if (error.code === 'AUTH_REQUIRED') {
if (!orderedMoveCommand(io)) reportExpiredSignIn(state, io);
- } else io.err(`pullboard: ${error.message}; run pullboard status to see pending uploads`);
+ } else if (!(orderedMoveCommand(io) && baselineNotAccepted(state) && transientRelayFailure(error))) io.err(`pullboard: ${error.message}; run pullboard status to see pending uploads`);
return { linked: true, board: state.board, url: state.url, sequence: state.sequence,
behind: (state.mode === 'ordered' ? Number(Boolean(state.pending)) : behind(root, state)) + Number(Boolean(state.snapshot || state.checkpoint)),
recovery: recoverySummary(state) };
}
- });
+ }, io);
}
/** Return token-management context without opening a board-key pairing link or exposing any credential. */
diff --git a/test/relay-client-fixture.js b/test/relay-client-fixture.js
index be1cbea2..85718eb2 100644
--- a/test/relay-client-fixture.js
+++ b/test/relay-client-fixture.js
@@ -232,6 +232,7 @@ globalThis.fetch = function tracedFetch(input, init) {
let stateReadHold = null;
let stateReadStarted = 0;
let refuseSnapshotWrites = false;
+ let stallApi = false;
let rejectSnapshotWrites = null;
let rejectedStateReads = null;
let legacySnapshotApi = false;
@@ -362,6 +363,7 @@ globalThis.fetch = function tracedFetch(input, init) {
request: Buffer.concat(requestChunks), response: Buffer.concat(responseChunks) }));
calls.push({ method: req.method, path: req.url, accept: req.headers.accept ?? '', engine: req.headers['x-pullboard-engine'],
browser: String(req.headers.cookie ?? '').includes('pb_session=') });
+ if (stallApi && (req.url ?? '').startsWith('/api/')) { request.resume(); await new Promise(() => {}); }
if (legacySnapshotApi && req.method === 'GET' && req.url === '/api/v2') {
res.writeHead(legacySnapshotApi === 'bad-route' ? 400 : 200, { 'content-type': 'application/json' });
res.end(JSON.stringify(legacySnapshotApi === 'bad-route'
@@ -740,6 +742,8 @@ globalThis.fetch = function tracedFetch(input, init) {
overrideDelete(value) { override = value; },
failTokenMints(count) { mintFailures = count; },
refuseSnapshotWrites(value) { refuseSnapshotWrites = value; },
+ /** Leave every board API request unanswered until its client times out, like an overloaded relay. */
+ stallApi(value = true) { stallApi = value; },
/** Restart only the private service with a changed operator quota, retaining real journals and credentials. */
setStorageQuota(bytes) {
assert.ok(Number.isSafeInteger(bytes) && bytes > 0);
diff --git a/test/relay-stuck-upload.test.js b/test/relay-stuck-upload.test.js
index ec6dadc0..6e6aaa21 100644
--- a/test/relay-stuck-upload.test.js
+++ b/test/relay-stuck-upload.test.js
@@ -219,14 +219,17 @@ test('a refused upload never blocks board moves [H16,B26]', async (t) => {
transient.rejectSnapshots({ status, code: 'RELAY_UNAVAILABLE', message: `fixture snapshot HTTP ${status}` });
await transient.link();
const title = `not applied before baseline HTTP ${status}`;
- const refused = await transient.cli('add', transient.lane, title);
- assert.equal(refused.code, 1, 'a retryable outage still refuses the linked move');
- assert.equal(refused.document.error.code, 'RELAY_UNAVAILABLE');
- assert.doesNotMatch((refused.document.diagnostics ?? []).join('\n'), /relay paused/u,
+ const kept = await transient.cli('add', transient.lane, title);
+ assert.equal(kept.code, 0, 'a retryable outage never fails a local move before the baseline is accepted [B26]');
+ assert.doesNotMatch((kept.document.diagnostics ?? []).join('\n'), /relay paused/u,
'retryable statuses never activate the permanent baseline pause');
+ assert.equal(JSON.parse(readFileSync(transient.linkFile, 'utf8')).baselinePause, undefined);
transient.rejectSnapshots(null);
- const retried = await transient.cli('add', transient.lane, title);
- assert.equal(retried.code, 0, 'the same command succeeds after the transient failure clears');
+ const retried = await transient.cli('status');
+ assert.equal(retried.code, 0, 'the next command uploads the baseline once the transient failure clears');
+ const exported = await transient.cli('export');
+ assert.equal(exported.document.tables.item.filter(row => row.item_title === title).length, 1, 'the move rides the later baseline once');
+ assert.equal(transient.moveAcks.length, 0);
});
}
@@ -338,9 +341,9 @@ test('a refused upload never blocks board moves [H16,B26]', async (t) => {
for (const refusal of [
{ label: 'unknown', status: 404, code: 'UNKNOWN_STATE' },
{ label: 'auth', status: 401, code: 'AUTH_REQUIRED' },
- { label: 'unavailable', status: 503, code: 'RELAY_UNAVAILABLE' },
+ { label: 'unavailable', status: 503, code: 'RELAY_UNAVAILABLE', transient: true },
{ label: 'wrong-version', status: 404, code: 'NO_BOARD', version: 2 },
- { label: 'unavailable-absence', status: 503, code: 'NO_BOARD' },
+ { label: 'unavailable-absence', status: 503, code: 'NO_BOARD', transient: true },
{ label: 'auth-absence', status: 401, code: 'NO_SNAPSHOT' },
]) {
await t.test(`uncertain ${refusal.label} remote state never authorizes local fallback [H16]`, async subtest => {
@@ -350,6 +353,14 @@ test('a refused upload never blocks board moves [H16,B26]', async (t) => {
assert.ok(JSON.parse(readFileSync(uncertain.linkFile, 'utf8')).baselinePause);
uncertain.rejectStateReads(refusal);
const blocked = await uncertain.cli('add', uncertain.lane, 'never apply a move after uncertain preflight');
+ if (refusal.transient) {
+ // #919: a relay that merely cannot answer never fails a local move; the move waits for a baseline, never an ordered send.
+ assert.equal(blocked.code, 0, 'an unreachable relay never fails a move before the baseline is accepted [B26]');
+ assert.equal(uncertain.moveAcks.length, 0);
+ const kept = await uncertain.cli('export');
+ assert.equal(kept.document.tables.item.some(row => row.item_title === 'never apply a move after uncertain preflight'), true);
+ return;
+ }
assert.equal(blocked.code, 1, 'a failed remote check cannot become local fallback in the same command');
assert.equal(blocked.document.error.code, refusal.code);
assert.equal(uncertain.moveAcks.length, 0);
diff --git a/test/relay-sync.test.js b/test/relay-sync.test.js
new file mode 100644
index 00000000..35fe2f59
--- /dev/null
+++ b/test/relay-sync.test.js
@@ -0,0 +1,115 @@
+/** A relay timeout, a busy link lock or an uncertain baseline never fails a local board move [B26,H16,H17]. */
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { describe, test } from 'node:test';
+import { relayClientFixture } from './relay-client-fixture.js';
+
+const RELAY = JSON.stringify(new URL('../src/relay.js', import.meta.url).href);
+
+/** Read the saved link without including ciphertext or credentials in an assertion diagnostic. */
+function link(box) { return JSON.parse(readFileSync(box.linkFile, 'utf8')); }
+
+/** Run shout moves in one child process with a short transport timeout, reporting codes, notices and local rows. */
+function moves(box, texts, options = {}) {
+ return box.script(`
+ const { relayOperation, localRecords } = await import(${RELAY});
+ const lines = [], io = { err: line => lines.push(line), relayTimeoutMs: 300, ...${JSON.stringify(options)} };
+ const outcomes = [];
+ for (const text of ${JSON.stringify(texts)}) {
+ try { await relayOperation(process.cwd(), 'shout', [{ from: 'coordinator', to: 'all', lanes: [], text }], io); outcomes.push('ok'); }
+ catch (error) { outcomes.push({ code: error.code, message: error.message }); }
+ }
+ const local = localRecords(process.cwd(), board => board.db.prepare('SELECT shout_text AS text FROM shout').all().map(row => row.text));
+ process.stdout.write(JSON.stringify({ outcomes, lines, local }));
+ `);
+}
+
+/** Pause the first baseline for size, as the 19 Oct submit failure did, then stop the relay answering. */
+async function pausedLink(t) {
+ const box = await relayClientFixture(t);
+ box.legacySnapshotApi('bad-route');
+ box.rejectSnapshots({ status: 400, code: 'BAD_REQUEST', message: 'the body exceeds 14000000 bytes; send a smaller snapshot' });
+ await box.link();
+ box.rejectSnapshots(null);
+ assert.ok(link(box).baselinePause, 'the baseline is paused for size');
+ return box;
+}
+
+/** Leave a baseline unaccepted and unpaused, the state a timed-out first upload leaves behind. */
+async function uncertainLink(t) {
+ const box = await pausedLink(t);
+ const state = link(box);
+ delete state.baselinePause;
+ writeFileSync(box.linkFile, JSON.stringify(state) + '\n', { mode: 0o600 });
+ return box;
+}
+
+/** After the relay answers again, every local move must be inside the one accepted baseline. */
+async function assertUploadedLater(box, texts) {
+ box.stallApi(false); box.legacySnapshotApi(false);
+ const status = await box.cli('status');
+ assert.equal(status.code, 0);
+ const state = link(box);
+ assert.equal(state.baselineAccepted, true, 'the next command uploads the baseline');
+ assert.equal(state.baselineStale, undefined);
+ const exported = await box.cli('export');
+ for (const text of texts) assert.ok(exported.document.tables.shout.some(row => row.shout_text === text), `local move kept: ${text}`);
+ assert.equal(box.moveAcks.length, 0, 'the moves went up inside the baseline, not as ordered moves');
+}
+
+describe('relay failures never fail a local move', { concurrency: true }, () => {
+for (const [name, setup] of [['a paused link', pausedLink], ['a link whose first upload is uncertain', uncertainLink]]) {
+ test(`a timing-out relay never fails a move on ${name}, and the move rides the later baseline [B26,H16,H17]`, async t => {
+ const box = await setup(t);
+ box.stallApi(true);
+ const run = await moves(box, ['first move while the relay stalls', 'second move while the relay stalls']);
+ assert.equal(run.code, 0, run.stderr);
+ assert.deepEqual(run.document.outcomes, ['ok', 'ok'], 'both moves commit locally');
+ assert.deepEqual(run.document.local, ['first move while the relay stalls', 'second move while the relay stalls']);
+ assert.equal(run.document.lines.filter(line => /did not answer \(timed out\)/u.test(line)).length, 1, 'the relay failure is reported once');
+ await assertUploadedLater(box, run.document.local);
+ });
+
+ test(`a busy link lock is waited out with progress, then decided under the lock, on ${name} [B26,H16,H17]`, async t => {
+ const box = await setup(t);
+ const holder = spawn('sleep', ['30'], { stdio: 'ignore' });
+ const lock = box.linkFile + '.lock';
+ mkdirSync(lock, { recursive: true, mode: 0o700 });
+ writeFileSync(lock + '/pid', String(holder.pid));
+ const release = setTimeout(() => { holder.kill(); rmSync(lock, { recursive: true, force: true }); }, 2500);
+ t.after(() => { clearTimeout(release); holder.kill(); rmSync(lock, { recursive: true, force: true }); });
+ const run = await moves(box, ['move after the lock cleared'], { relayLockProgressMs: 400 });
+ assert.equal(run.code, 0, run.stderr);
+ assert.deepEqual(run.document.outcomes, ['ok']);
+ assert.ok(run.document.lines.filter(line => /waiting for another worktree/u.test(line)).length >= 2, 'progress lines appear while waiting');
+ });
+}
+
+test('a lock held past the wait refuses once with RELAY_BUSY and a retry step [B26,H16]', async t => {
+ const box = await relayClientFixture(t);
+ await box.link();
+ const holder = spawn('sleep', ['30'], { stdio: 'ignore' });
+ const lock = box.linkFile + '.lock';
+ mkdirSync(lock, { recursive: true, mode: 0o700 });
+ writeFileSync(lock + '/pid', String(holder.pid));
+ t.after(() => { holder.kill(); rmSync(lock, { recursive: true, force: true }); });
+ const run = await moves(box, ['never sent'], { relayLockWaitMs: 800, relayLockProgressMs: 300 });
+ assert.equal(run.document.outcomes[0].code, 'RELAY_BUSY');
+ assert.match(run.document.outcomes[0].message, /run the command again/u);
+ assert.deepEqual(run.document.local, []);
+});
+
+test('on an accepted baseline a stalled relay refuses once with a reason and the retry step, committing nothing [B26,H16]', async t => {
+ const box = await relayClientFixture(t);
+ await box.link();
+ assert.equal((await box.cli('status')).code, 0);
+ assert.equal(link(box).baselineAccepted, true);
+ box.stallApi(true);
+ const run = await moves(box, ['must not commit outside the relay order']);
+ assert.equal(run.document.outcomes[0].code, 'RELAY_UNAVAILABLE');
+ assert.match(run.document.outcomes[0].message, /timed out/u);
+ assert.match(run.document.outcomes[0].message, /retry this move when the relay is reachable/u);
+ assert.deepEqual(run.document.local, [], 'nothing commits outside the relay order');
+});
+});