Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

feat(relay): schedule local canary alerts and retain delivery [H5,H17]

Corey Olsonbf757b95d3ad
diff --git a/src/health.js b/src/health.js
index 913a1684..6baaa2ca 100644
--- a/src/health.js
+++ b/src/health.js
@@ -35,6 +35,7 @@ import { localReleaseRank } from './release-queue.js';
import { workQueue } from './queue.js';
import { trunkRef } from './trusted-policy.js';
import { routeWorkShortages } from './route-capacity.js';
+import { readRelayHealth } from './relay-canary-health.js';
/** Each threshold's pullboard.json key under "health" and its default. */
export const HEALTH_DEFAULTS = Object.freeze({ stuck: '1h', holds: '1d', release: '1d' });
@@ -88,6 +89,22 @@ export function healthReport({ db, root, config, now }) {
const report = { asOf, thresholds: { stuck: limits.stuck, holds: limits.holds, release: limits.release }, stuck: [], waiting: [] };
// Gate alarms read the git directory, so they report even before a board exists (U41).
report.stuck.push(...gateControlFindings(root, config ?? {}, now));
+ const canary = readRelayHealth(root);
+ if (typeof canary.alertSince === 'string' && Number.isFinite(Date.parse(canary.alertSince))) {
+ const step = typeof canary.step === 'string' ? canary.step : 'unknown';
+ const code = typeof canary.code === 'string' ? canary.code : 'RELAY_CANARY_FAILURE';
+ const slow = canary.p95Ms !== null && canary.p95Ms > 2000;
+ const why = slow ? `Ten-minute p95 is ${Math.round(canary.p95Ms)} ms`
+ : canary.consecutiveFailures >= 2 ? `The last canary probe failed during ${step} (${code})`
+ : `The relay canary alert remains active; the latest recorded step is ${step} (${code})`;
+ report.stuck.push(finding('RELAY_CANARY', null, {
+ subject: 'relay canary', key: 'RELAY_CANARY:relay', since: canary.alertSince, now: now.getTime(),
+ title: 'sealed round trips are degraded',
+ why,
+ next: 'inspect pullboard relay canary; keep working locally while queued moves await the relay',
+ owner: { role: 'coordinator', who: COORDINATOR }, detail: { step, code, roundTripMs: canary.roundTripMs, p95Ms: canary.p95Ms },
+ }));
+ }
if (!db) return report;
const at = now.getTime();
const age = (iso) => at - Date.parse(iso);
diff --git a/src/needs-summary.js b/src/needs-summary.js
index 7a13640a..0f3f6975 100644
--- a/src/needs-summary.js
+++ b/src/needs-summary.js
@@ -15,6 +15,7 @@ import {
sentinelDraftNeeds,
sentinels,
} from './board-query.js';
+import { relayCanaryNeeds } from './relay-canary-needs.js';
/** Keep status obligations; reuse its same-call seat rows, otherwise read them freshly. */
export function personNeedsSummary(root, { board, config, agents }) {
@@ -25,7 +26,8 @@ export function personNeedsSummary(root, { board, config, agents }) {
agents: (agents ?? store.listAgents(board)).map(agent => ({ ...agent, displayName: displayAgentName(agent, config.agents.names) })),
rigorDrafts: rigorDraftNeeds(rows, store.rowDrafts(board), store.rowDecisions(board)), sentinelDrafts: sentinelDraftNeeds(sentinels(board)), blocks: recentBlocks(repoInfo(root).commonDir, board),
}, specApprovalState(root, { board, config, spec }));
- return projected;
+ const relayHealthAlerts = relayCanaryNeeds(root);
+ return { ...projected, relayHealthAlerts, needs: [...projected.needs, ...relayHealthAlerts] };
}
/** The person's blocks from the last day, by the board's clock, for Needs you [C6]. */
diff --git a/src/needs.js b/src/needs.js
index c3e7dace..3823811a 100644
--- a/src/needs.js
+++ b/src/needs.js
@@ -46,6 +46,7 @@ export function personNeeds(state) {
...personBlocks(state.blocks ?? []),
...(drafts ? [{ kind: 'draft', id: 'drafts', count: drafts, ref: String(drafts),
text: 'draft spec rows to approve or drop' + rows.filter(row => row.staleDecision).map(row => `; ${row.stage}`).join(''), what: 'review in SPEC.md', target: 'tab:spec' }] : []),
+ ...(state.relayHealthAlerts ?? []),
];
}
diff --git a/src/relay-canary-config.js b/src/relay-canary-config.js
new file mode 100644
index 00000000..a5576bcd
--- /dev/null
+++ b/src/relay-canary-config.js
@@ -0,0 +1,19 @@
+/** Machine-local configuration for a dedicated, ordinarily linked canary seat [H5,H17]. */
+import { isAbsolute } from 'node:path';
+import { Refused } from './refused.js';
+
+/** Validate only execution settings; credentials remain in the existing board link and device key store. */
+export function relayCanaryConfig(value) {
+ if (value === undefined || value === null) return null;
+ if (!value || typeof value !== 'object' || Array.isArray(value)
+ || typeof value.enabled !== 'boolean' || typeof value.root !== 'string' || !isAbsolute(value.root)
+ || typeof value.agentId !== 'string' || !value.agentId || /\s/u.test(value.agentId)
+ || Object.keys(value).some(key => !['enabled', 'root', 'agentId', 'deadlineMs'].includes(key))) {
+ throw new Refused('RELAY_CANARY_CONFIG', 'configure a dedicated linked seat with pullboard relay canary on; credentials are never command arguments');
+ }
+ const deadlineMs = value.deadlineMs ?? 5000;
+ if (!Number.isSafeInteger(deadlineMs) || deadlineMs < 100 || deadlineMs > 25000) {
+ throw new Refused('RELAY_CANARY_CONFIG', 'set the canary timeout between 100 and 25000 milliseconds, below its one-minute cadence');
+ }
+ return { enabled: value.enabled, root: value.root, agentId: value.agentId, deadlineMs };
+}
diff --git a/src/relay-canary-health.js b/src/relay-canary-health.js
index 2d6cc3bc..03d43edf 100644
--- a/src/relay-canary-health.js
+++ b/src/relay-canary-health.js
@@ -1,5 +1,6 @@
/** Client-local round-trip evidence; a relay HTTP health response never establishes sync health [H5,C2]. */
import { DatabaseSync } from 'node:sqlite';
+import { randomUUID } from 'node:crypto';
import {
existsSync,
mkdirSync,
@@ -42,6 +43,7 @@ function projection(history, queued, now) {
: status === 'healthy' ? `relay healthy; last sealed round trip ${Math.round(last.roundTripMs)} ms`
: 'relay health unknown; no recent sealed round trip';
return { status, since, lastAt: last?.at ?? null, roundTripMs: last?.roundTripMs ?? null,
+ alertSince: history?.alertSince ?? null, pendingNotifications: history?.pendingNotices?.length ?? 0,
relayVersion: last?.relayVersion ?? null, step: last?.step ?? null, code: last?.code ?? null,
consecutiveFailures: history?.consecutiveFailures ?? 0, p95Ms: history?.p95Ms ?? null, queued, text };
}
@@ -94,6 +96,11 @@ function advance(previous, sample, now) {
subject: 'relay', title: 'sealed round trips are degraded', since: history.alertSince,
why: slow ? `Ten-minute p95 is ${Math.round(history.p95Ms)} ms` : `Two consecutive probes failed at ${sample.step} (${sample.code})`,
next: 'inspect pullboard relay canary; keep working locally while queued moves await the relay', owner: 'coordinator' } : null;
+ if (transition) {
+ history.pendingNotices ??= [];
+ history.pendingNotices.push({ id: randomUUID(), kind: transition, at: sample.at, finding,
+ step: sample.step, code: sample.code });
+ }
return { history, transition, finding };
}
@@ -115,12 +122,14 @@ export function observeRelayHealthFile(file, result, { now = Date.now(), alerts
const previous = saved ? JSON.parse(saved.value) : null;
const priorAlert = previous?.alertSince ?? null;
const priorFailures = previous?.consecutiveFailures ?? 0;
+ const priorNotices = structuredClone(previous?.pendingNotices ?? []);
const updated = advance(previous, sample, now);
// Ordinary client transport failures update status; only the configured canary raises monitor alerts.
if (!alerts) {
updated.transition = null; updated.finding = null;
updated.history.alertSince = priorAlert;
updated.history.consecutiveFailures = priorFailures;
+ updated.history.pendingNotices = priorNotices;
}
db.prepare('INSERT INTO relay_health(id,value) VALUES (1,?) ON CONFLICT(id) DO UPDATE SET value=excluded.value')
.run(JSON.stringify(updated.history));
@@ -131,3 +140,41 @@ export function observeRelayHealthFile(file, result, { now = Date.now(), alerts
throw error;
} finally { db.close(); }
}
+
+/** Retry alert/recovery delivery after an outage; a local observation is never a remote receipt. */
+export function pendingRelayCanaryNotices(root) {
+ return historyAt(healthFile(root))?.pendingNotices ?? [];
+}
+
+/** Remove only a notice the normal native move recorded, including its ordinary offline outbox. */
+export function markCanaryNoticeRecorded(root, id) {
+ const db = new DatabaseSync(healthFile(root));
+ try {
+ db.exec('PRAGMA busy_timeout=1000; BEGIN IMMEDIATE');
+ const row = db.prepare('SELECT value FROM relay_health WHERE id=1').get();
+ if (row) {
+ const history = JSON.parse(row.value);
+ history.pendingNotices = (history.pendingNotices ?? []).filter(notice => notice.id !== id);
+ db.prepare('UPDATE relay_health SET value=? WHERE id=1').run(JSON.stringify(history));
+ }
+ db.exec('COMMIT');
+ } catch (error) {
+ try { db.exec('ROLLBACK'); } catch { /* Keep the original storage refusal. */ }
+ throw error;
+ } finally { db.close(); }
+}
+
+/** Claim one due minute atomically so multiple local views cannot run duplicate monitor ticks. */
+export function claimRelayCanaryTick(root, now = Date.now()) {
+ if (!Number.isFinite(now)) throw new Refused('RELAY_HEALTH', 'run the canary with a valid local clock');
+ const file = healthFile(root);
+ mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+ try { writeFileSync(file, '', { flag: 'wx', mode: 0o600 }); }
+ catch (error) { if (error.code !== 'EEXIST') throw error; }
+ const db = new DatabaseSync(file);
+ try {
+ db.exec('PRAGMA busy_timeout=1000; CREATE TABLE IF NOT EXISTS canary_schedule(id INTEGER PRIMARY KEY CHECK(id=1), next_due REAL NOT NULL)');
+ return db.prepare('INSERT INTO canary_schedule(id,next_due) VALUES (1,?) ON CONFLICT(id) DO UPDATE SET next_due=excluded.next_due WHERE canary_schedule.next_due<=?')
+ .run(now + 60000, now).changes === 1;
+ } finally { db.close(); }
+}
diff --git a/src/relay-canary-needs.js b/src/relay-canary-needs.js
new file mode 100644
index 00000000..34d42b76
--- /dev/null
+++ b/src/relay-canary-needs.js
@@ -0,0 +1,27 @@
+/** Project this checkout's active relay-canary alert into its local Needs-you list [H5,H15,C7]. */
+import { readRelayHealth } from './relay-canary-health.js';
+import { relayStatus } from './relay.js';
+
+/** Count only this local link's queued moves; status is read-only and never contacts the relay. */
+function queuedMoves(root) {
+ try { return relayStatus(root).outbox?.length ?? 0; }
+ catch { return 0; }
+}
+
+/** Keep private client telemetry local, so a publishing host's probe never enters a sealed presentation. */
+export function relayCanaryNeeds(root) {
+ const health = readRelayHealth(root, { queued: queuedMoves(root) });
+ if (typeof health.alertSince !== 'string' || !Number.isFinite(Date.parse(health.alertSince))) return [];
+ const step = typeof health.step === 'string' && /^[a-z-]{1,24}$/u.test(health.step) ? health.step : 'unknown';
+ const code = typeof health.code === 'string' && /^[A-Z][A-Z0-9_]{0,79}$/u.test(health.code) ? health.code : 'RELAY_CANARY_FAILURE';
+ const time = health.alertSince.slice(11, 16);
+ const queued = health.queued;
+ const detail = health.p95Ms !== null && health.p95Ms > 2000
+ ? `the ten-minute p95 is ${Math.round(health.p95Ms)} ms`
+ : health.consecutiveFailures >= 2 ? `the last canary probe failed during ${step} (${code})`
+ : `the alert remains active; the latest recorded step is ${step} (${code})`;
+ return [{ kind: 'relay-health', id: 'relay-canary', ref: 'relay', at: health.alertSince,
+ text: `relay degraded since ${time}; your moves are queued locally (${queued})`,
+ what: `${detail}; inspect pullboard relay canary and keep working locally while queued moves await the relay`,
+ target: 'relay:relay-canary', step, code, queued }];
+}
diff --git a/src/relay-canary-probe.js b/src/relay-canary-probe.js
index b97c5399..228e8b3f 100644
--- a/src/relay-canary-probe.js
+++ b/src/relay-canary-probe.js
@@ -70,6 +70,10 @@ export async function probeRelayCanary(root, { agentId, deadlineMs = 5000, io =
const at = new Date(started).toISOString();
let step = 'sign-in', relayVersion = null, sequence = null, moveId = null;
const controller = new AbortController();
+ /** Stop this probe's own I/O when its watch is shutting down. */
+ const stop = () => controller.abort();
+ if (io.relaySignal?.aborted) stop();
+ io.relaySignal?.addEventListener('abort', stop, { once: true });
let timer;
/** Enforce the absolute deadline between local phases as well as inside requests and lock waits. */
function remaining() {
@@ -159,5 +163,8 @@ export async function probeRelayCanary(root, { agentId, deadlineMs = 5000, io =
else if (/^SEAL_/u.test(error?.code ?? '') || error?.code === 'RELAY_KEY_MISSING') step = 'unseal';
if (Number.isSafeInteger(error?.relayVersion)) relayVersion = error.relayVersion;
return result(false, controller.signal.aborted ? 'RELAY_CANARY_DEADLINE' : safeCode(error));
- } finally { clearTimeout(timer); }
+ } finally {
+ clearTimeout(timer);
+ io.relaySignal?.removeEventListener('abort', stop);
+ }
}
diff --git a/src/relay-canary-watch.js b/src/relay-canary-watch.js
new file mode 100644
index 00000000..5cf02d35
--- /dev/null
+++ b/src/relay-canary-watch.js
@@ -0,0 +1,54 @@
+/** A machine-configured one-minute watch, independent of ordinary ten-minute health scans [H5,H17]. */
+import { probeRelayCanary } from './relay-canary-probe.js';
+import {
+ claimRelayCanaryTick,
+ observeRelayHealth,
+ pendingRelayCanaryNotices,
+ markCanaryNoticeRecorded,
+} from './relay-canary-health.js';
+import { Refused } from './refused.js';
+import { relayCanaryConfig } from './relay-canary-config.js';
+import { loadMachineSettings } from './settings.js';
+
+export const RELAY_CANARY_INTERVAL_MS = 60000;
+
+/** Observe exactly one complete probe; optional notices use the ordinary command's board authority. */
+export async function runRelayCanary(value, { io = {}, now = Date.now, notify } = {}) {
+ const config = relayCanaryConfig(value);
+ if (!config) throw new Refused('RELAY_CANARY_CONFIG', 'configure this machine with pullboard relay canary on before running its watch');
+ const observation = await probeRelayCanary(config.root, { agentId: config.agentId,
+ deadlineMs: config.deadlineMs, io });
+ const update = observeRelayHealth(config.root, observation, { now: now() });
+ if (notify) for (const notice of pendingRelayCanaryNotices(config.root)) {
+ await notify(notice, observation);
+ markCanaryNoticeRecorded(config.root, notice.id);
+ }
+ return { observation, ...update };
+}
+
+/** Start one due local tick; shutdown aborts and awaits the active probe instead of abandoning a write. */
+export function startRelayCanaryWatch({ config = () => loadMachineSettings().relay?.canary,
+ run = runRelayCanary, now = Date.now, every = setInterval, cancel = clearInterval, onError = () => {}, notify } = {}) {
+ const lifetime = new AbortController();
+ let pending = null;
+ /** Read current machine opt-in on every tick and admit at most one local run for its minute. */
+ const tick = () => {
+ if (pending || lifetime.signal.aborted) return pending;
+ pending = Promise.resolve().then(async () => {
+ const selected = relayCanaryConfig(config());
+ if (!selected?.enabled || !claimRelayCanaryTick(selected.root, now())) return;
+ await run(selected, { io: { relaySignal: lifetime.signal }, now, notify });
+ }).catch(error => { if (!lifetime.signal.aborted) onError(error); }).finally(() => { pending = null; });
+ return pending;
+ };
+ const timer = every(tick, RELAY_CANARY_INTERVAL_MS);
+ timer.unref?.();
+ void tick();
+ return { tick,
+ /** Stop scheduling first, then drain the exact in-flight probe under its shared deadline. */
+ async close() {
+ cancel(timer); lifetime.abort();
+ if (pending) await pending;
+ },
+ };
+}
diff --git a/src/settings.js b/src/settings.js
index a30f147c..088425dc 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -13,6 +13,7 @@ import {
} from './resources.js';
import { validateRunner } from './runners.js';
import { Refused } from './refused.js';
+import { relayCanaryConfig } from './relay-canary-config.js';
import {
carriedBrowserByLaunch,
gateQueueSettings,
@@ -58,6 +59,7 @@ export function loadMachineSettings() {
if (value.relay?.offerAtInit !== undefined && typeof value.relay.offerAtInit !== 'boolean') {
throw new Refused('BAD_MACHINE_SETTINGS', `relay.offerAtInit in ${file} must be true or false`);
}
+ relayCanaryConfig(value.relay?.canary);
if (value.gate?.memoryFloorMB !== undefined && (!Number.isSafeInteger(value.gate.memoryFloorMB) || value.gate.memoryFloorMB < 0)) {
throw new Refused('BAD_MACHINE_SETTINGS', `gate.memoryFloorMB in ${file} must be a whole number of megabytes, 0 or more; fix it before running Pullboard`);
}
@@ -149,6 +151,15 @@ function writeMachineSettings(settings) {
} finally { rmSync(temporary, { force: true }); }
}
+/** Persist an explicitly configured canary without changing any board's credentials or project policy. */
+export function setRelayCanary(value) {
+ const canary = relayCanaryConfig(value);
+ const settings = loadMachineSettings();
+ const next = { ...settings, relay: { ...settings.relay, canary } };
+ writeMachineSettings(next);
+ return canary;
+}
+
/** Opt this machine into one board's landing pool without cancelling any admitted work. */
export function setLandingPool(board, enabled) {
if (!/^[a-f0-9]{32}$/u.test(board) || typeof enabled !== 'boolean') throw new Refused('BAD_MACHINE_SETTINGS', 'a landing pool needs this board identifier and on or off');