Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

feat(relay): verify sealed canary round trips and local health [H5,H17]

Corey Olsone31b1d27dc39
diff --git a/src/relay-canary-health.js b/src/relay-canary-health.js
new file mode 100644
index 00000000..2d6cc3bc
--- /dev/null
+++ b/src/relay-canary-health.js
@@ -0,0 +1,133 @@
+/** Client-local round-trip evidence; a relay HTTP health response never establishes sync health [H5,C2]. */
+import { DatabaseSync } from 'node:sqlite';
+import {
+ existsSync,
+ mkdirSync,
+ writeFileSync,
+} from 'node:fs';
+import {
+ dirname,
+ join,
+} from 'node:path';
+import { repoInfo } from './git.js';
+import { Refused } from './refused.js';
+
+const WINDOW_MS = 600000;
+const STEPS = new Set(['sign-in', 'push', 'pull', 'unseal', 'version', 'complete']);
+
+/** Keep observations outside native history and outside sealed snapshots. */
+function healthFile(root) {
+ return join(repoInfo(root).commonDir, 'pullboard', 'relay-health.sqlite');
+}
+
+/** Read one private telemetry record without creating a file for a status read. */
+function historyAt(file) {
+ if (!existsSync(file)) return null;
+ const db = new DatabaseSync(file, { readOnly: true });
+ try {
+ const row = db.prepare('SELECT value FROM relay_health WHERE id=1').get();
+ return row ? JSON.parse(row.value) : null;
+ } finally { db.close(); }
+}
+
+/** Format only observations this client made, with a real locally queued move count. */
+function projection(history, queued, now) {
+ const last = history?.samples?.at(-1);
+ const stale = !last || now - Date.parse(last.at) > 120000;
+ const status = !last || stale ? 'unknown' : history.degradedSince ? 'degraded' : 'healthy';
+ const since = history?.degradedSince ?? null;
+ const time = since ? since.slice(11, 16) : '';
+ const text = status === 'degraded'
+ ? `relay degraded since ${time}; your moves are queued locally (${queued})`
+ : status === 'healthy' ? `relay healthy; last sealed round trip ${Math.round(last.roundTripMs)} ms`
+ : 'relay health unknown; no recent sealed round trip';
+ return { status, since, lastAt: last?.at ?? null, roundTripMs: last?.roundTripMs ?? null,
+ relayVersion: last?.relayVersion ?? null, step: last?.step ?? null, code: last?.code ?? null,
+ consecutiveFailures: history?.consecutiveFailures ?? 0, p95Ms: history?.p95Ms ?? null, queued, text };
+}
+
+/** A status read neither contacts the relay nor treats an old success as current health. */
+export function readRelayHealth(root, { queued = 0, now = Date.now() } = {}) {
+ return readRelayHealthFile(healthFile(root), { queued, now });
+}
+
+/** Read the same client-local record when the transport already knows its shared Git directory. */
+export function readRelayHealthFile(file, { queued = 0, now = Date.now() } = {}) {
+ if (!Number.isSafeInteger(queued) || queued < 0 || !Number.isFinite(now)) {
+ throw new Refused('RELAY_HEALTH', 'read relay health with a nonnegative queued move count and a valid clock');
+ }
+ try { return projection(historyAt(file), queued, now); }
+ catch { return projection(null, queued, now); }
+}
+
+/** Keep the durable record limited to safe protocol facts, never transport error messages or credentials. */
+function sampleOf(result, now) {
+ if (!result || typeof result.ok !== 'boolean' || !STEPS.has(result.step)
+ || !Number.isFinite(result.roundTripMs) || result.roundTripMs < 0 || !Number.isFinite(now)
+ || result.relayVersion !== null && result.relayVersion !== undefined && !Number.isSafeInteger(result.relayVersion)) {
+ throw new Refused('RELAY_HEALTH', 'record a completed bounded relay probe with its failed step and elapsed time');
+ }
+ return { at: new Date(now).toISOString(), ok: result.ok, step: result.step,
+ code: typeof result.code === 'string' && /^[A-Z][A-Z0-9_]{0,79}$/u.test(result.code) ? result.code : 'RELAY_CANARY_FAILURE',
+ roundTripMs: result.roundTripMs, relayVersion: result.relayVersion ?? null };
+}
+
+/** Derive alerts from consecutive probes or a complete ten-minute latency window. */
+function advance(previous, sample, now) {
+ const history = previous ?? { version: 1, monitoringSince: sample.at, samples: [], consecutiveFailures: 0,
+ degradedSince: null, alertSince: null, p95Ms: null };
+ history.samples = history.samples.filter(entry => Date.parse(entry.at) >= now - WINDOW_MS && Date.parse(entry.at) <= now);
+ history.samples.push(sample);
+ history.samples = history.samples.slice(-128);
+ history.consecutiveFailures = sample.ok ? 0 : history.consecutiveFailures + 1;
+ const times = history.samples.filter(entry => entry.ok).map(entry => entry.roundTripMs).sort((a, b) => a - b);
+ const windowReady = now - Date.parse(history.monitoringSince) >= WINDOW_MS && times.length >= 10;
+ history.p95Ms = windowReady ? times[Math.max(0, Math.ceil(times.length * 0.95) - 1)] : null;
+ const slow = history.p95Ms !== null && history.p95Ms > 2000;
+ const wasAlert = Boolean(history.alertSince);
+ if (!sample.ok || slow) history.degradedSince ??= sample.at;
+ else history.degradedSince = null;
+ if (history.consecutiveFailures >= 2 || slow) history.alertSince ??= history.degradedSince ?? sample.at;
+ else if (sample.ok) history.alertSince = null;
+ const transition = !wasAlert && history.alertSince ? 'alert' : wasAlert && !history.alertSince ? 'recovery' : null;
+ const finding = history.alertSince ? { key: 'relay-canary', cause: 'RELAY_CANARY', item: null,
+ subject: 'relay', title: 'sealed round trips are degraded', since: history.alertSince,
+ why: slow ? `Ten-minute p95 is ${Math.round(history.p95Ms)} ms` : `Two consecutive probes failed at ${sample.step} (${sample.code})`,
+ next: 'inspect pullboard relay canary; keep working locally while queued moves await the relay', owner: 'coordinator' } : null;
+ return { history, transition, finding };
+}
+
+/** Serialize local observations in a separate SQLite file; the real board and relay cursor are untouched. */
+export function observeRelayHealth(root, result, options = {}) {
+ return observeRelayHealthFile(healthFile(root), result, options);
+}
+
+/** Record a transport observation when its link path is already available, without rediscovering Git. */
+export function observeRelayHealthFile(file, result, { now = Date.now(), alerts = true } = {}) {
+ const sample = sampleOf(result, now);
+ mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+ try { writeFileSync(file, '', { flag: 'wx', mode: 0o600 }); }
+ catch (error) { if (error.code !== 'EEXIST') throw error; }
+ const db = new DatabaseSync(file);
+ try {
+ db.exec('PRAGMA busy_timeout=1000; CREATE TABLE IF NOT EXISTS relay_health(id INTEGER PRIMARY KEY CHECK(id=1), value TEXT NOT NULL); BEGIN IMMEDIATE');
+ const saved = db.prepare('SELECT value FROM relay_health WHERE id=1').get();
+ const previous = saved ? JSON.parse(saved.value) : null;
+ const priorAlert = previous?.alertSince ?? null;
+ const priorFailures = previous?.consecutiveFailures ?? 0;
+ const updated = advance(previous, sample, now);
+ // Ordinary client transport failures update status; only the configured canary raises monitor alerts.
+ if (!alerts) {
+ updated.transition = null; updated.finding = null;
+ updated.history.alertSince = priorAlert;
+ updated.history.consecutiveFailures = priorFailures;
+ }
+ db.prepare('INSERT INTO relay_health(id,value) VALUES (1,?) ON CONFLICT(id) DO UPDATE SET value=excluded.value')
+ .run(JSON.stringify(updated.history));
+ db.exec('COMMIT');
+ return { health: projection(updated.history, 0, now), transition: updated.transition, finding: updated.finding };
+ } catch (error) {
+ try { db.exec('ROLLBACK'); } catch { /* A schema refusal may precede the transaction. */ }
+ throw error;
+ } finally { db.close(); }
+}
diff --git a/src/relay-canary-probe.js b/src/relay-canary-probe.js
new file mode 100644
index 00000000..1346158b
--- /dev/null
+++ b/src/relay-canary-probe.js
@@ -0,0 +1,150 @@
+/** A bounded ordinary sealed-move round trip on a person-linked canary board [H2,H15,C7]. */
+import { randomUUID } from 'node:crypto';
+import * as store from './board.js';
+import {
+ engineReceipt,
+ requireSupportedEngine,
+} from './engine.js';
+import { ENGINE_VERSION } from './machine.js';
+import { repoInfo } from './git.js';
+import { relayLinkFile } from './local-state.js';
+import { actorTransport } from './relay-seats.js';
+import {
+ localRecords,
+ relayMachineContext,
+ relayOperation,
+ request,
+} from './relay.js';
+import { readBoardKey } from './relay-key.js';
+import { unseal } from './seal.js';
+import { Refused } from './refused.js';
+
+/** Keep the recorded result bounded and credential-free, even for unexpected errors. */
+function safeCode(error) {
+ return typeof error?.code === 'string' && /^[A-Z][A-Z0-9_]{0,79}$/u.test(error.code)
+ ? error.code : 'RELAY_CANARY_FAILURE';
+}
+
+/** Verify the exact returned sealed move, not a local write or a successful health response. */
+async function verifyReturnedMove(root, state, remote, expected) {
+ if (!Array.isArray(remote.events)) throw new Refused('RELAY_RESPONSE', 'the canary pull returned no event list');
+ const matches = remote.events.filter(record => record.event_id === expected.sequence);
+ if (matches.length !== 1 || matches[0].kind !== 'move') {
+ throw new Refused('RELAY_RESPONSE', 'the canary pull did not return the exact acknowledged move');
+ }
+ const record = matches[0];
+ if (typeof record.sealed !== 'string') throw new Refused('RELAY_RESPONSE', 'the canary move has no sealed bytes');
+ const sealed = Buffer.from(record.sealed, 'base64url');
+ if (!sealed.length || sealed.toString('base64url') !== record.sealed) {
+ throw new Refused('RELAY_RESPONSE', 'the canary move has invalid sealed bytes');
+ }
+ expected.stage('unseal');
+ const plain = await unseal(readBoardKey(state.board), sealed,
+ { boardId: state.board, kind: 'move', sequence: record.event_id });
+ expected.stage('version');
+ let move;
+ try { move = JSON.parse(new TextDecoder().decode(plain)); }
+ catch { throw new Refused('RELAY_MOVE', 'the canary move is not a supported sealed document'); }
+ if (move?.version !== 1 || !Number.isSafeInteger(move.engine) || move.engine < 1) {
+ throw new Refused('RELAY_MOVE', 'the canary move format is unsupported');
+ }
+ requireSupportedEngine(move);
+ const shout = move.args?.[0];
+ if (move.id !== expected.moveId || move.operation !== 'shout' || move.actor !== expected.agentId
+ || shout?.from !== expected.agentId || shout.to !== 'coordinator' || shout.text !== expected.text
+ || move.engine > ENGINE_VERSION) {
+ throw new Refused('RELAY_RESPONSE', 'the canary returned a different move or nonce');
+ }
+ const receipt = localRecords(root, board => engineReceipt(board, move.id));
+ if (!receipt || receipt.sequence !== expected.sequence || receipt.outcome?.error) {
+ throw new Refused('RELAY_RESPONSE', 'the canary has no successful native receipt for its returned move');
+ }
+}
+
+/**
+ * Exercise saved authentication, ordinary actor push, fresh pull and device-only unsealing under one deadline.
+ * A failed probe may already have recorded its nonce; it never retries that write or changes credentials.
+ */
+export async function probeRelayCanary(root, { agentId, deadlineMs = 5000, io = {}, clock = () => Date.now() } = {}) {
+ const started = clock();
+ const at = new Date(started).toISOString();
+ let step = 'sign-in', relayVersion = null, sequence = null, moveId = null;
+ const controller = new AbortController();
+ let timer;
+ /** Enforce the absolute deadline between local phases as well as inside requests and lock waits. */
+ function remaining() {
+ const left = deadlineMs - (clock() - started);
+ if (controller.signal.aborted || left <= 0) throw new Refused('RELAY_CANARY_DEADLINE', 'the relay canary deadline elapsed');
+ return Math.max(1, Math.ceil(left));
+ }
+ /** Bound the public result to protocol facts; never retain error messages, tokens or key material. */
+ function result(ok, code) {
+ return { at, ok, step: ok ? 'complete' : step, code, roundTripMs: Math.max(0, clock() - started),
+ relayVersion, sequence, moveId };
+ }
+ try {
+ if (!Number.isSafeInteger(deadlineMs) || deadlineMs < 1 || deadlineMs > 2_147_483_647) {
+ throw new Refused('RELAY_CANARY_CONFIG', 'configure a positive bounded canary deadline');
+ }
+ timer = setTimeout(() => controller.abort(), deadlineMs);
+ const state = relayMachineContext(root);
+ if (!state || state.unlinking || state.environment) {
+ throw new Refused('RELAY_OFF', 'link this dedicated canary board through the person before probing it');
+ }
+ const actor = localRecords(root, board => store.listAgents(board).find(agent => agent.agent_id === agentId));
+ if (!actor || actor.agent_lane === 'coordinator') {
+ throw new Refused('NOT_JOINED', 'join an ordinary noncoordinator seat on the dedicated canary board');
+ }
+ const previousSequence = state.sequence;
+ if (!Number.isSafeInteger(previousSequence) || previousSequence < 0) {
+ throw new Refused('RELAY_STORAGE', 'restore the canary link sequence before probing it');
+ }
+ const probeIo = Object.create(io);
+ Object.assign(probeIo, { relayCanary: true, relaySignal: controller.signal, stderr: io.stderr ?? { write() {} },
+ stdout: io.stdout ?? { write() {} }, err: io.err ?? (() => {}), say: io.say ?? (() => {}) });
+ Object.defineProperty(probeIo, 'relayTimeoutMs', { get: remaining });
+ Object.defineProperty(probeIo, 'relayLockWaitMs', { get: remaining });
+ /** Observe the actual transport phase without affecting routing, credential choice or responses. */
+ probeIo.onRelayRequest = ({ method, path }) => {
+ remaining();
+ if (path.startsWith('/auth/')) step = 'sign-in';
+ else if (method === 'GET' && /\/(?:events|state)(?:\?|$)/u.test(path)) step = 'pull';
+ else if (method === 'POST' && path.endsWith('/moves')) step = 'push';
+ io.onRelayRequest?.({ method, path });
+ };
+ const transport = await actorTransport(root, relayLinkFile(repoInfo(root).commonDir), state, agentId, probeIo);
+ const signed = await request(transport, '/auth/session?board=' + state.board, { signal: controller.signal }, probeIo);
+ relayVersion = signed.version;
+ if (signed.session?.kind !== 'board' || signed.session.agent !== agentId) {
+ throw new Refused('RELAY_ACTOR', 'the canary sign-in does not belong to its configured seat');
+ }
+ const text = 'Relay canary ' + randomUUID();
+ let verified;
+ /** Inspect each durable acknowledgement once, before legitimate checkpoint compaction. */
+ probeIo.onRelayMoveRecorded = ({ moveId: recordedId, sequence: recordedSequence }) => {
+ if (verified) return verified;
+ moveId = recordedId; sequence = recordedSequence;
+ verified = (async () => {
+ remaining(); step = 'pull';
+ const pulled = await request(transport, '/api/v1/boards/' + state.board + '/events?after=' + previousSequence,
+ { signal: controller.signal }, probeIo);
+ relayVersion = pulled.version;
+ await verifyReturnedMove(root, state, pulled, { moveId, sequence, agentId, text,
+ stage(value) { remaining(); step = value; } });
+ remaining();
+ })();
+ return verified;
+ };
+ remaining(); step = 'push';
+ await relayOperation(root, 'shout', [{ from: agentId, to: 'coordinator', text, lanes: [] }], probeIo);
+ if (!verified) throw new Refused('RELAY_RESPONSE', 'the canary write was not acknowledged by the relay');
+ await verified;
+ remaining();
+ return result(true, 'OK');
+ } catch (error) {
+ if (['RELAY_VERSION', 'ENGINE_VERSION', 'SEAL_VERSION'].includes(error?.code)) step = 'version';
+ else if (/^SEAL_/u.test(error?.code ?? '') || error?.code === 'RELAY_KEY_MISSING') step = 'unseal';
+ if (Number.isSafeInteger(error?.relayVersion)) relayVersion = error.relayVersion;
+ return result(false, controller.signal.aborted ? 'RELAY_CANARY_DEADLINE' : safeCode(error));
+ } finally { clearTimeout(timer); }
+}
diff --git a/src/relay-lock.js b/src/relay-lock.js
index 15129089..17d22e29 100644
--- a/src/relay-lock.js
+++ b/src/relay-lock.js
@@ -39,9 +39,9 @@ function gone(pid) {
* transaction that checks the holder and replaces it only when that holder's process is gone, so of any number of
* waiters exactly one holds it, and a live holder is never displaced: there is no separate removal step to race.
*/
-function openLease(file) {
+function openLease(file, timeoutMs = 10000) {
const db = new DatabaseSync(`${file}.lease.sqlite`);
- db.exec(`PRAGMA busy_timeout = 10000;
+ db.exec(`PRAGMA busy_timeout = ${Math.max(1, Math.min(10000, Math.floor(timeoutMs)))};
CREATE TABLE IF NOT EXISTS relay_lease(name TEXT PRIMARY KEY, pid INTEGER NOT NULL, token TEXT NOT NULL);`);
return db;
}
@@ -101,15 +101,22 @@ export async function locked(file, work, io) {
const finishWait = beginCommandPhase('waiting-lock');
try {
const lock = file + '.lock';
+ /** A caller deadline stops waiting without displacing or signaling another lock holder. */
+ const checkAbort = () => {
+ if (io?.relaySignal?.aborted) throw new Refused('RELAY_TIMEOUT', 'the relay deadline expired; this waiter did not take over another sender');
+ };
+ checkAbort();
mkdirSync(dirname(file), { recursive: true });
const startedAt = Date.now();
const deadline = startedAt + (io?.relayLockWaitMs ?? LOCK_WAIT_MS);
const progressEvery = io?.relayLockProgressMs ?? LOCK_PROGRESS_MS;
let nextProgress = startedAt + progressEvery;
- const lease = openLease(file);
+ const lease = openLease(file, io?.relaySignal ? deadline - Date.now() : 10000);
const token = randomUUID();
try {
- while (!(takeLease(lease, token) && takeDirectory(lock))) {
+ while (true) {
+ checkAbort();
+ if (takeLease(lease, token) && takeDirectory(lock)) break;
if (Date.now() >= deadline) throw new Refused('RELAY_BUSY', `another worktree was still sending this board after ${Math.round((Date.now() - startedAt) / 1000)} s; this attempt was not recorded; run the command again`);
if (io?.err && Date.now() >= nextProgress) {
nextProgress = Date.now() + progressEvery;
@@ -119,6 +126,7 @@ export async function locked(file, work, io) {
}
finishWait();
try {
+ checkAbort();
// Only the current lock holder can have a live link write; remove crashed legacy bearer fragments.
for (const name of readdirSync(dirname(file))) {
if (/^relay\.json\.[0-9a-f-]+\.tmp$/u.test(name)) rmSync(join(dirname(file), name), { force: true });
diff --git a/src/relay-output.js b/src/relay-output.js
index 96e738a1..95446e63 100644
--- a/src/relay-output.js
+++ b/src/relay-output.js
@@ -10,6 +10,7 @@ import {
export function briefLines(state) {
const lines = [];
if (state.linked) lines.push(`relay: sequence ${state.sequence}; ${state.behind} pending uploads${state.recovery?.pending ? `; recovery pending for sequence ${state.recovery.skip}; retry ${state.recovery.next}` : ''}`);
+ if (state.health) lines.push(state.health.text);
if (state.oversizedSnapshot && !state.refusedCheckpoint) lines.push(formatSnapshotLimit(state.oversizedSnapshot));
if (state.storage) lines.push(relayStorageLine(state.storage));
lines.push(...outboxLines(state));
diff --git a/src/relay.js b/src/relay.js
index 08b30e8c..5ba3a3de 100644
--- a/src/relay.js
+++ b/src/relay.js
@@ -9,6 +9,10 @@ import {
} from './relay-snapshot-size.js';
export { formatSnapshotLimit, originRepository };
import { beginCommandPhase } from './command-phase.js';
+import {
+ observeRelayHealth,
+ readRelayHealth,
+} from './relay-canary-health.js';
import { deliverPersonEnrollments } from './relay-person-enrollment.js';
import { recordedMoveRefusal } from './recorded-move.js';
import {
@@ -354,6 +358,8 @@ export async function request(state, path, options = {}, io) {
/** The sole raw HTTP transport: explicit sign-in or a saved, opted-in board link, with bounded deadlines. */
async function rawRequest(state, path, { method = 'GET', body, allowMissing = false, initialBaseline = false, signal } = {}, io) {
+ signal ??= io?.relaySignal;
+ io?.onRelayRequest?.({ method, path });
const supplied = io?.ignoreRelayEnvironmentToken ? undefined : process.env.PULLBOARD_RELAY_TOKEN;
const readToken = method === 'GET' && /^\/api\/v[12]\/boards\//u.test(path) ? supplied : undefined;
if (readToken !== undefined && !/^pa_[A-Za-z0-9_-]{43}$/.test(readToken)) throw new Refused('AUTH_REQUIRED', 'supply a current scoped board token without including it in command arguments');
@@ -407,7 +413,11 @@ async function rawRequest(state, path, { method = 'GET', body, allowMissing = fa
if (storage) error.storage = storage;
throw error;
}
- if (document.version !== 1) throw new Refused('RELAY_VERSION', 'the relay API version is unsupported; upgrade Pullboard before syncing');
+ if (document.version !== 1) {
+ const error = new Refused('RELAY_VERSION', 'the relay API version is unsupported; upgrade Pullboard before syncing');
+ if (Number.isSafeInteger(document.version)) error.relayVersion = document.version;
+ throw error;
+ }
return document;
} finally { finishPhase(); }
}
@@ -448,6 +458,17 @@ function behind(root, state) {
return localRecords(root, (board) => board.db.prepare('SELECT COUNT(*) AS count FROM event WHERE event_id > ?').get(state.cursor).count);
}
+/** Best-effort telemetry never changes the outcome or credential authority of a native move. */
+function recordClientHealth(root, ok, error, roundTripMs, step = 'push') {
+ const code = error?.code ?? 'OK';
+ if (/VERSION/u.test(code)) step = 'version';
+ else if (/AUTH|OAUTH|NOT_JOINED|RELAY_OFF/u.test(code)) step = 'sign-in';
+ else if (/SEAL|DECRYPT|RELAY_KEY/u.test(code)) step = 'unseal';
+ try { observeRelayHealth(root, { ok, step: ok ? 'complete' : step, code, roundTripMs,
+ relayVersion: ok ? 1 : error?.relayVersion ?? null }, { alerts: false }); }
+ catch { /* Losing optional local telemetry cannot undo a requested operation. */ }
+}
+
/** Expose key-free link status; only an explicit link operation includes its pairing carrier. */
function summary(root, state, { pairing = false } = {}) {
if (!state) {
@@ -462,6 +483,7 @@ function summary(root, state, { pairing = false } = {}) {
sequence: state.sequence, behind: (state.baselinePause ? behind(root, state) : state.mode === 'ordered' ? Number(Boolean(state.pending)) : behind(root, state)) + Number(Boolean(state.snapshot || state.checkpoint))
+ (state.outbox?.moves?.length ?? 0),
recovery: recoverySummary(state),
+ health: readRelayHealth(root, { queued: state.outbox?.moves?.length ?? 0 }),
...outboxFields(state),
...(relayStorage(state.storage) ? { storage: relayStorage(state.storage) } : {}),
...(oversizedSnapshot ? { oversizedSnapshot } : {}),
@@ -1108,6 +1130,7 @@ function outboxLink(root, file, state, io) {
/** Send one board operation before executing it locally, and report its canonical engine result. */
export async function relayOperation(root, operation, args, io, command, applyLocally, onRecorded) {
+ const roundTripStarted = Date.now();
const file = linkFile(root);
return locked(file, async () => {
const state = loadLink(file);
@@ -1199,7 +1222,11 @@ export async function relayOperation(root, operation, args, io, command, applyLo
}
const receipt = localRecords(root, (board) => engineReceipt(board, move.id));
if (!receipt) throw new Refused('RELAY_RESPONSE', 'the acknowledged operation has no replay receipt; fetch a consistent relay snapshot');
- if (!receipt.outcome.error) onRecorded?.();
+ if (!receipt.outcome.error) {
+ onRecorded?.();
+ await io.onRelayMoveRecorded?.({ moveId: move.id, sequence: state.sequence });
+ if (!io.relayCanary) recordClientHealth(root, true, null, Date.now() - roundTripStarted);
+ }
delete state.recovered;
saveLink(file, state);
// A phone reads the next checkpoint; the background publisher sends it once moves go quiet (#1014).
@@ -1222,6 +1249,7 @@ export async function relayOperation(root, operation, args, io, command, applyLo
return receipt.outcome.result;
} catch (error) {
// Catch-up may commit the requested receipt before a later link write refuses. A refused receipt counts as nothing.
+ if (!io.relayCanary) recordClientHealth(root, false, error, Date.now() - roundTripStarted);
if (requestedMove && onRecorded) {
try {
const committed = localRecords(root, board => engineReceipt(board, requestedMove.id));
@@ -1240,6 +1268,7 @@ export async function relayOperation(root, operation, args, io, command, applyLo
/** Catch up an opted-in replica before/after a CLI command; offline reads retain local evidence. */
export async function syncRelay(root, io) {
+ const roundTripStarted = Date.now();
// Reuse Git discovery only within this command; still re-read the link before each retry.
const files = LINK_FILES.get(io) ?? LINK_FILES.set(io, new Map()).get(io);
const file = files.get(root) ?? linkFile(root);
@@ -1271,6 +1300,7 @@ export async function syncRelay(root, io) {
}
catch (error) {
if (!(error instanceof Refused)) throw error;
+ if (!io.relayCanary) recordClientHealth(root, false, error, Date.now() - roundTripStarted, 'pull');
if (confirmedDeletion(state, error)) { forgetDeletedCopy(root, file, state, io); return summary(root, null); }
if (error.code === 'RELAY_KEY_MISSING') {
if (!KEY_WARNED_COMMANDS.has(io)) {
@@ -1282,7 +1312,8 @@ export async function syncRelay(root, io) {
} else if (!(orderedMoveCommand(io) && (baselineNotAccepted(state) || outboxQueueable(state, error, null, io)) && transientRelayFailure(error))) io.err(`pullboard: ${error.message}; run pullboard status to see pending uploads`);
return { linked: true, board: state.board, url: state.url, sequence: state.sequence,
behind: (state.mode === 'ordered' ? Number(Boolean(state.pending)) : behind(root, state)) + Number(Boolean(state.snapshot || state.checkpoint)) + (state.outbox?.moves?.length ?? 0),
- recovery: recoverySummary(state), ...outboxFields(state) };
+ recovery: recoverySummary(state), ...outboxFields(state),
+ health: readRelayHealth(root, { queued: state.outbox?.moves?.length ?? 0 }) };
}
}, io);
}
diff --git a/test/relay-canary.test.js b/test/relay-canary.test.js
index 48c52cc0..05587f51 100644
--- a/test/relay-canary.test.js
+++ b/test/relay-canary.test.js
@@ -72,7 +72,7 @@ function displayText(value) {
return '';
}
-test('a failed round trip alerts within two minutes and names the failed step [H2,H15,C7]', { timeout: 60_000 }, async t => { // budget: private relay startup, two real probes, and fixture cleanup; alert timing uses the injected clock.
+test('a failed round trip alerts within two minutes and names the failed step [H2,H15,C7]', { timeout: 60_000 }, async t => { // budget: private relay startup, four real probes, and fixture cleanup; alert timing uses the injected clock.
const { probeRelayCanary, observeRelayHealth, readRelayHealth } = await canaryApi();
const box = await relayClientFixture(t);
await box.link();
@@ -83,24 +83,35 @@ test('a failed round trip alerts within two minutes and names the failed step [H
assert.equal(typeof agentId, 'string', 'the probe uses a real registered machine seat');
let now = 1_000;
- const probe = async () => {
- // The fixture lets this authenticated write finish, then refuses the probe's fresh event pull.
- box.beforeNextMove(() => box.refuseReads(true));
+ const probe = async ({ refusePull = false } = {}) => {
+ // The fixture lets this authenticated write finish, then refuses only the probe's fresh event pull.
+ if (refusePull) box.beforeNextMove(() => box.refuseReads(true));
let result;
try { result = await withFixtureEnvironment(box.env, () => probeRelayCanary(root, { agentId, io: {}, clock: () => now })); }
- finally { box.refuseReads(false); }
- assert.equal(result.ok, false, 'the canary reports the deliberately failed fresh pull');
- assert.ok(typeof result.step === 'string' && result.step.length > 0 && result.step !== 'complete', 'the failing stage is explicit');
+ finally { if (refusePull) box.refuseReads(false); }
+ if (refusePull) {
+ assert.equal(result.ok, false, 'the canary reports the deliberately failed fresh pull');
+ assert.equal(result.step, 'pull', 'the failure is attributed to the fresh event pull, not setup or seat registration');
+ } else {
+ assert.equal(result.ok, true, 'a normal authenticated sealed round trip establishes the healthy baseline and recovery');
+ assert.equal(result.step, 'complete');
+ }
return result;
};
- const first = await probe();
+ const healthy = observeRelayHealth(box.root, await probe(), { now });
+ assert.equal(healthy.health.status, 'healthy', 'a real sealed round trip establishes healthy status');
+ assert.equal(healthy.transition, null);
+ assert.equal(healthy.finding, null);
+
+ now += 1_000;
+ const first = await probe({ refusePull: true });
const initial = observeRelayHealth(box.root, first, { now });
assert.equal(initial.transition, null, 'one failure degrades the relay without an alert');
assert.equal(initial.health.status, 'degraded');
now += 119_000;
- const second = await probe();
+ const second = await probe({ refusePull: true });
const alerted = observeRelayHealth(box.root, second, { now });
assert.equal(alerted.transition, 'alert', 'two consecutive failures alert inside the two-minute window');
assert.equal(alerted.health.status, 'degraded');
@@ -112,6 +123,38 @@ test('a failed round trip alerts within two minutes and names the failed step [H
assert.ok(displayText(read).includes(second.step), 'the persisted health projection retains the failed step');
const link = JSON.parse(readFileSync(box.linkFile, 'utf8'));
assert.equal(JSON.stringify(read).includes(link.token), false, 'health projection never exposes the relay credential');
+
+ now += 1_000;
+ const recovered = observeRelayHealth(box.root, await probe(), { now });
+ assert.equal(recovered.transition, 'recovery', 'a subsequent complete sealed round trip clears the active alert');
+ assert.equal(recovered.health.status, 'healthy');
+ assert.equal(recovered.health.consecutiveFailures, 0);
+ assert.equal(recovered.finding, null);
+ assert.equal(readRelayHealth(box.root, { now }).status, 'healthy', 'the cleared state is durable for a later status read');
+});
+
+test('the health history waits for ten minutes of slow successful round-trip samples before p95 alerting [H5,H15,C7]', async t => {
+ const { observeRelayHealth, readRelayHealth } = await canaryApi();
+ const box = await relayClientFixture(t);
+ const start = Date.parse('2026-01-01T00:00:00.000Z');
+ let last;
+ // Use deterministic timestamps in the real private SQLite history; the primary named case proves transport success.
+ for (let index = 0; index <= 10; index += 1) {
+ const now = start + index * 60_000;
+ last = observeRelayHealth(box.root, {
+ ok: true, step: 'complete', code: 'OK', roundTripMs: 2_101, relayVersion: 1,
+ }, { now });
+ if (index < 10) {
+ assert.equal(last.transition, null, 'a partial latency window cannot alert early');
+ assert.equal(last.health.status, 'healthy');
+ assert.equal(last.health.p95Ms, null);
+ }
+ }
+ assert.equal(last.transition, 'alert', 'the first full ten-minute window applies the p95 threshold');
+ assert.equal(last.health.status, 'degraded');
+ assert.equal(last.health.p95Ms, 2_101);
+ assert.match(last.finding.why, /ten-minute p95.*2101 ms/iu);
+ assert.equal(readRelayHealth(box.root, { now: start + 600_000 }).status, 'degraded');
});
test('the client shows queued moves while the relay is degraded [H16,H17,C7]', { timeout: 60_000 }, async t => { // budget: linked fixture setup, two queued moves, two real probes, and cleanup.