Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

feat(relay): run canary watches through ordinary seats [H5,H17]

Corey Olson22eded8b22ac
diff --git a/src/api.js b/src/api.js
index c9ba6b86..a4c36599 100644
--- a/src/api.js
+++ b/src/api.js
@@ -86,11 +86,13 @@ import { createApiHandler } from './api-http.js';
import { settingsInventory } from './settings-inventory.js';
import { moveArgs } from './api-moves.js';
import { personRequestStatuses } from './relay-requests.js';
+import { relayCanaryNeeds } from './relay-canary-needs.js';
+import { relayFiles } from './local-state.js';
export { moveArgs } from './api-moves.js';
/** Inputs shared by the complete state cache and the lightweight live-change watcher. */
const STATE_REVISION_INPUTS = Object.freeze([
- 'identity', 'head', 'trunk', 'files', 'event', 'agents', 'leases', 'requestReceipts', 'milestones', 'externalRoadmap', 'eventLogVersion', 'ruleProofWindow',
+ 'identity', 'head', 'trunk', 'files', 'event', 'agents', 'leases', 'requestReceipts', 'milestones', 'externalRoadmap', 'eventLogVersion', 'ruleProofWindow', 'localRelayMarker',
]);
/** Serialize named state inputs in one stable order for cache and stream comparisons. */
@@ -179,6 +181,7 @@ function boardStateRevision(root, id, isMain, { projects = listApiProjects, scop
externalRoadmap: local.externalMilestones,
eventLogVersion: store.EVENT_LOG_VERSION,
ruleProofWindow: local.ruleProofWindow,
+ localRelayMarker: localRelayMarker(info),
};
const revision = stateRevision(inputs);
return { cacheId: info.commonDir + '\0' + id, revision, event: local.event, milestones: local.externalMilestones,
@@ -207,6 +210,12 @@ function fileMarkerForPath(path) {
}
}
+/** Invalidate only this local API state when its private canary record or queued link changes. */
+function localRelayMarker(info) {
+ const files = relayFiles(info.commonDir);
+ return [fileMarkerForPath(join(info.commonDir, 'pullboard', 'relay-health.sqlite')), fileMarkerForPath(files.link)];
+}
+
/** Read HEAD and its selected ref without spawning Git on every 200 ms poll. */
function headFileRevision(info) {
const head = readFileSync(join(info.gitDir, 'HEAD'), 'utf8').trim();
@@ -428,6 +437,7 @@ function watchLocalStateRevision(board, { scopeCommonDir = null } = {}) {
externalRoadmap,
eventLogVersion: store.EVENT_LOG_VERSION,
ruleProofWindow: local.ruleProofWindow,
+ localRelayMarker: localRelayMarker(info),
});
}
@@ -500,6 +510,8 @@ function buildLocalBoardState(root, boardId, board, codeIsMain, { previous = nul
state.personRequests = projectData.personRequests;
state.milestones = projectData.milestones;
state.blocks = personBlockWords(repoInfo(root).commonDir, board.clock.now()); // Needs you below is recomputed with them [C6].
+ // Private telemetry belongs only to this local API host, not a sealed relay presentation.
+ state.relayHealthAlerts = relayCanaryNeeds(root);
const decorated = projectSpecApprovals(projectFit(pendingFit(repoInfo(root).commonDir), projectRowDecisions(projectData.rowDecisions, state)), specApprovalState(root, { board, config }));
decorated.ruleCounts = ruleProofCounts(root, board, decorated.spec);
const bases = new Map(decorated.items.map((item) => [item.id, item]));
diff --git a/src/commands/relay-canary.js b/src/commands/relay-canary.js
new file mode 100644
index 00000000..83c34d8c
--- /dev/null
+++ b/src/commands/relay-canary.js
@@ -0,0 +1,132 @@
+/** Explicit machine-local canary commands; all messages retain the caller's ordinary authority [H5,H17]. */
+import { realpathSync } from 'node:fs';
+import { context, ordered, withBoard, whoAmI } from '../cli.js';
+import { COORDINATOR } from '../config.js';
+import { PERSON } from '../board.js';
+import { Refused } from '../refused.js';
+import { relayMachineContext, relayStatus } from '../relay.js';
+import { loadMachineSettings, setRelayCanary } from '../settings.js';
+import { relayCanaryConfig } from '../relay-canary-config.js';
+import { readRelayHealth } from '../relay-canary-health.js';
+import { startRelayCanaryWatch, runRelayCanary } from '../relay-canary-watch.js';
+
+/** Read telemetry without contacting the relay or creating a health observation. */
+function status(root, config) {
+ let queued = 0;
+ try { queued = relayStatus(root).outbox?.length ?? 0; }
+ catch { /* An absent local link cannot establish current relay health. */ }
+ return { config, health: readRelayHealth(root, { queued }) };
+}
+
+/** Bind an explicit run to the configured checkout and its ordinary joined seat. */
+function requireSeat(ctx, board, config) {
+ const me = whoAmI(ctx, board);
+ if (me.id === COORDINATOR || me.id === PERSON || !me.id
+ || realpathSync(ctx.info.root) !== realpathSync(config.root) || me.id !== config.agentId) {
+ throw new Refused('RELAY_CANARY_SEAT', 'run the canary from its configured ordinary joined seat; do not borrow a coordinator or another seat');
+ }
+ return me;
+}
+
+/** Record each notification through ordinary native transport; failure leaves the durable notice pending. */
+async function notify(ctx, config, notice, observation) {
+ const delayed = !observation.ok || Date.parse(observation.at) > Date.parse(notice.at) ? 'DELAYED: relay delivery is pending; this is a local native notice, not a phone acknowledgement. ' : '';
+ const text = `${delayed}Relay canary ${notice.kind} at ${notice.at}: ${notice.step} (${notice.code}). Inspect pullboard relay canary status; keep working locally.`;
+ await withBoard(ctx, async board => {
+ const me = requireSeat(ctx, board, config);
+ for (const to of [COORDINATOR, PERSON]) {
+ await ordered(ctx, board, 'shout', [{ from: me.id, to, text, decision: false }]);
+ }
+ });
+}
+
+/** Run `relay canary on|off|status|run`; the lazy relay dispatcher supplies the subcommand in first. */
+export async function run(io, { first = 'status', second, rest = [], values = {} }) {
+ if (!['on', 'off', 'status', 'run', 'watch'].includes(first) || second || rest.length
+ || values.url || values.all || values.skip || (values.timeout !== undefined && first !== 'on')) {
+ throw new Refused('USAGE', 'pullboard relay canary on [--timeout <milliseconds>], off, status, run, or watch');
+ }
+ const ctx = context(io);
+ let config = relayCanaryConfig(loadMachineSettings().relay?.canary);
+ if (first === 'watch') return watch(ctx, config);
+ if (first === 'on') {
+ const linked = relayMachineContext(ctx.info.root);
+ if (!linked || linked.environment) throw new Refused('RELAY_NOT_LINKED', 'link this dedicated canary seat normally before enabling its watch; environment credentials do not configure a canary');
+ config = withBoard(ctx, board => {
+ const me = whoAmI(ctx, board);
+ const selected = relayCanaryConfig({ enabled: true, root: realpathSync(ctx.info.root), agentId: me.id,
+ deadlineMs: values.timeout === undefined ? 5000 : Number(values.timeout) });
+ requireSeat(ctx, board, selected);
+ return setRelayCanary(selected);
+ });
+ } else if (first === 'off') {
+ config = setRelayCanary(config ? { ...config, enabled: false } : null);
+ } else if (first === 'run') {
+ if (!config?.enabled) throw new Refused('RELAY_CANARY_CONFIG', 'enable this machine’s dedicated canary with pullboard relay canary on');
+ withBoard(ctx, board => requireSeat(ctx, board, config));
+ const controller = new AbortController();
+ let interrupted = 0;
+ /** Abort owned transport first; the awaited probe drains before the command returns. */
+ const interrupt = code => { interrupted ||= code; controller.abort(); };
+ const onInterrupt = () => interrupt(130);
+ const onTerminate = () => interrupt(143);
+ const onParentAbort = () => controller.abort();
+ process.once('SIGINT', onInterrupt);
+ process.once('SIGTERM', onTerminate);
+ io.relaySignal?.addEventListener('abort', onParentAbort, { once: true });
+ if (io.relaySignal?.aborted) controller.abort();
+ let result;
+ try {
+ result = await runRelayCanary(config, { io: { ...io, relaySignal: controller.signal }, notify: (notice, observation) => {
+ if (controller.signal.aborted) throw new Refused('RELAY_CANARY_STOPPED', 'the canary stopped; pending notifications remain for its next ordinary run');
+ return notify({ ...ctx, io: { ...io, relaySignal: controller.signal } }, config, notice, observation);
+ } });
+ } catch (error) {
+ io.say('DELAYED: the probe or native notification did not finish; durable pending notices are retained, with no phone acknowledgement claimed.');
+ if (interrupted) return interrupted;
+ throw error;
+ } finally {
+ process.removeListener('SIGINT', onInterrupt);
+ process.removeListener('SIGTERM', onTerminate);
+ io.relaySignal?.removeEventListener('abort', onParentAbort);
+ }
+ io.result?.(result);
+ io.say(result.health.text);
+ if (result.health.pendingNotifications) io.say('DELAYED: canary notifications remain pending; no phone acknowledgement has been observed.');
+ return interrupted || (result.observation.ok ? 0 : 1);
+ }
+ const result = status(ctx.info.root, config);
+ io.result?.(result);
+ io.say(`Relay canary ${config?.enabled ? 'on' : 'off'}. ${result.health.text}`);
+ if (result.health.pendingNotifications) io.say('DELAYED: canary notifications remain pending; no phone acknowledgement has been observed.');
+ return 0;
+}
+
+/** Keep one explicitly requested watch alive, stopping only after its active ordinary probe drains. */
+async function watch(ctx, config) {
+ if (!config?.enabled) throw new Refused('RELAY_CANARY_CONFIG', 'enable this machine’s dedicated canary with pullboard relay canary on');
+ withBoard(ctx, board => requireSeat(ctx, board, config));
+ let stopped;
+ let status = 0;
+ const done = new Promise(resolve => { stopped = resolve; });
+ const observer = startRelayCanaryWatch({ keepAlive: true,
+ run: (selected, { io }) => {
+ withBoard(ctx, board => requireSeat(ctx, board, selected));
+ const current = { ...ctx, io: { ...ctx.io, ...io } };
+ return runRelayCanary(selected, { io: current.io, notify: (notice, observation) => notify(current, selected, notice, observation) });
+ },
+ onError: error => ctx.io.err(`Relay canary has a pending observation or notice (${error.code ?? 'RELAY_CANARY_FAILURE'}); the next minute retries it.`),
+ });
+ /** Stop further scheduling, abort owned I/O and let native receipt cleanup finish before returning. */
+ const stop = code => { status ||= code; void observer.close().then(stopped, stopped); };
+ const interrupt = () => stop(130);
+ const terminate = () => stop(143);
+ process.once('SIGINT', interrupt); process.once('SIGTERM', terminate);
+ ctx.io.say('Relay canary watch runs every minute; stop it with Ctrl-C.');
+ try { await done; }
+ finally {
+ process.removeListener('SIGINT', interrupt); process.removeListener('SIGTERM', terminate);
+ await observer.close();
+ }
+ return status;
+}
diff --git a/src/commands/relay.js b/src/commands/relay.js
index f48c6c1a..dd3c739d 100644
--- a/src/commands/relay.js
+++ b/src/commands/relay.js
@@ -52,6 +52,7 @@ import {
/** Run `pullboard relay` and return its exit code. */
export async function run(io, { first, second, rest, values }) {
+ if (first === 'canary') return (await import('./relay-canary.js')).run(io, { first: second, second: rest[0], rest: rest.slice(1), values });
if (['on', 'join'].includes(first) && gitSyncLinked(io.cwd)) throw new Refused('GIT_SYNC_LINKED', 'this board already shares moves through Git; it cannot start a second move order', 'keep using pullboard sync for this board');
if (values['new-browser'] && first !== 'pair') throw new Refused('USAGE', 'pullboard relay pair --new-browser');
if (first === 'members') {
diff --git a/src/help.js b/src/help.js
index 54cac812..d0b13dc1 100644
--- a/src/help.js
+++ b/src/help.js
@@ -59,6 +59,7 @@ Set up
pair prints a one-use machine code; join stores it in another clone
pullboard relay members set <login> [<login> ...] | clear publish an approver-signed membership list
pullboard relay pair --new-browser sign in and pair one more browser; existing browsers stay paired
+ pullboard relay canary [on|off|status|run|watch] opt this linked seat into minute sealed round-trip monitoring
pullboard relay on --all [--url <address>] [--name <computer>] link every registered project after one sign-in; pair the phone once
keep this command open until pairing finishes; later projects link on registration
pullboard relay devices [--verbose] paired phones and joined computers by name, with paired date, last seen and boards
diff --git a/src/relay-canary-probe.js b/src/relay-canary-probe.js
index 228e8b3f..8ba304c6 100644
--- a/src/relay-canary-probe.js
+++ b/src/relay-canary-probe.js
@@ -63,7 +63,7 @@ async function verifyReturnedMove(root, state, remote, expected) {
/**
* Exercise saved authentication, ordinary actor push, fresh pull and device-only unsealing under one deadline.
- * A failed probe may already have recorded its nonce; it never retries that write or changes credentials.
+ * A failed probe may already have recorded its nonce; the next run finishes that exact receipt before a fresh nonce.
*/
export async function probeRelayCanary(root, { agentId, deadlineMs = 5000, io = {}, clock = () => Date.now() } = {}) {
const started = clock();
diff --git a/src/relay-canary-watch.js b/src/relay-canary-watch.js
index 5cf02d35..9314094f 100644
--- a/src/relay-canary-watch.js
+++ b/src/relay-canary-watch.js
@@ -28,7 +28,7 @@ export async function runRelayCanary(value, { io = {}, now = Date.now, notify }
/** Start one due local tick; shutdown aborts and awaits the active probe instead of abandoning a write. */
export function startRelayCanaryWatch({ config = () => loadMachineSettings().relay?.canary,
- run = runRelayCanary, now = Date.now, every = setInterval, cancel = clearInterval, onError = () => {}, notify } = {}) {
+ run = runRelayCanary, now = Date.now, every = setInterval, cancel = clearInterval, onError = () => {}, notify, keepAlive = false } = {}) {
const lifetime = new AbortController();
let pending = null;
/** Read current machine opt-in on every tick and admit at most one local run for its minute. */
@@ -42,7 +42,7 @@ export function startRelayCanaryWatch({ config = () => loadMachineSettings().rel
return pending;
};
const timer = every(tick, RELAY_CANARY_INTERVAL_MS);
- timer.unref?.();
+ if (!keepAlive) timer.unref?.();
void tick();
return { tick,
/** Stop scheduling first, then drain the exact in-flight probe under its shared deadline. */
diff --git a/src/relay-output.js b/src/relay-output.js
index 95446e63..aa560b9c 100644
--- a/src/relay-output.js
+++ b/src/relay-output.js
@@ -33,6 +33,8 @@ export function outboxLines(state) {
export function detailLines(result, boardName) {
const lines = [];
lines.push(result.linked ? `${result.link}\nrelay: ${boardName} (board ${result.board}) is linked for GitHub ${result.login || `account ${result.account}`}; sequence ${result.sequence}; ${result.behind} pending uploads${result.recovery?.pending ? `; recovery pending for sequence ${result.recovery.skip}; retry ${result.recovery.next}` : ''}` : (result.notice || 'relay off; the local board is complete'));
+ if (result.health) lines.push(result.health.text);
+ lines.push(...outboxLines(result));
if (result.storage) lines.push(relayStorageLine(result.storage));
return lines;
}
diff --git a/src/serve-sentinel.js b/src/serve-sentinel.js
index f7f080b2..13c392a7 100644
--- a/src/serve-sentinel.js
+++ b/src/serve-sentinel.js
@@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url';
import { listProjects } from './projects.js';
import { loadConfig } from './config.js';
import { healthSentinelEnabled } from './settings.js';
+import { startRelayCanaryWatch } from './relay-canary-watch.js';
/** Ten minutes is the declared background cadence, not a test performance budget. */
export const HEALTH_SENTINEL_INTERVAL_MS = 10 * 60_000;
@@ -15,9 +16,9 @@ function sentinelRunsFor(root) {
}
/** Run only the ordered health scan in one owned child and drain it on view shutdown. */
-function refreshBoard(root, signal) {
+function refreshBoard(root, signal, words = ['sentinel', 'run', '--health-only', '--json'], degradedIsExpected = false) {
return new Promise((resolve, reject) => {
- const child = spawn(process.execPath, [CLI, 'sentinel', 'run', '--health-only', '--json'], {
+ const child = spawn(process.execPath, [CLI, ...words], {
cwd: root, env: process.env, stdio: 'ignore', signal,
});
let timer;
@@ -30,15 +31,21 @@ function refreshBoard(root, signal) {
child.once('error', error => { if (error.name !== 'AbortError') reject(error); });
child.once('close', code => {
clearTimeout(timer); signal.removeEventListener('abort', stopping);
- if (code && !signal.aborted) reject(new Error(`health sentinel refresh exited ${code}`));
+ if (code && !(degradedIsExpected && code === 1) && !signal.aborted) reject(new Error(`health sentinel refresh exited ${code}`));
else resolve();
});
});
}
+/** Run the configured ordinary canary command; a failed round trip is a recorded observation. */
+function refreshCanary(root, signal) {
+ return refreshBoard(root, signal, ['relay', 'canary', 'run', '--json'], true);
+}
+
/** Start after the view listens; scans use the ordinary sentinel command so each board's off switch and ordering apply. */
export function startServeSentinel({ projectRoot, run = refreshBoard, projects = listProjects,
- every = setInterval, cancel = clearInterval, enabled = sentinelRunsFor, onError = () => {} } = {}) {
+ every = setInterval, cancel = clearInterval, enabled = sentinelRunsFor, onError = () => {}, runCanary = refreshCanary, canary = startRelayCanaryWatch } = {}) {
+ const canaryWatch = canary({ run: (config, { io }) => runCanary(config.root, io.relaySignal), onError });
const lifetime = new AbortController();
let pending = null;
/** Never start a second scan while the prior one or shutdown is still in flight. */
@@ -60,7 +67,7 @@ export function startServeSentinel({ projectRoot, run = refreshBoard, projects =
/** Stop the interval and wait for the exact active child to finish shutting down. */
async close() {
cancel(timer); lifetime.abort();
- if (pending) await pending;
+ await Promise.all([pending, canaryWatch.close()]);
},
};
}
diff --git a/test/relay-canary-watch.test.js b/test/relay-canary-watch.test.js
new file mode 100644
index 00000000..78c52202
--- /dev/null
+++ b/test/relay-canary-watch.test.js
@@ -0,0 +1,207 @@
+/** Minute canary scheduling, cross-watch admission, shutdown draining and durable notice retry [H5,H17,C7]. */
+import assert from 'node:assert/strict';
+import { DatabaseSync } from 'node:sqlite';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { delimiter, dirname, join } from 'node:path';
+import test from 'node:test';
+import { fileURLToPath } from 'node:url';
+import { repoInfo } from '../src/git.js';
+import { pendingRelayCanaryNotices } from '../src/relay-canary-health.js';
+import { startRelayCanaryWatch, RELAY_CANARY_INTERVAL_MS, runRelayCanary } from '../src/relay-canary-watch.js';
+import { relayClientFixture } from './relay-client-fixture.js';
+import { runFixtureGit } from './fixture-child.js';
+
+/** Capture the scheduler callback so cadence tests advance time without sleeping. */
+function fakeInterval() {
+ const timers = [];
+ return {
+ timers,
+ every(callback, delay) {
+ const timer = { callback, delay, unrefed: false, unref() { this.unrefed = true; } };
+ timers.push(timer);
+ return timer;
+ },
+ cancel(timer) { timer.cancelled = true; },
+ };
+}
+
+/** Temporarily give a direct probe only a private fixture's machine paths and restore every parent value. */
+async function withFixtureEnvironment(env, action) {
+ const names = ['HOME', 'PULLBOARD_HOME', 'PULLBOARD_MACHINE_HOME', 'PULLBOARD_RELAY_KEY', 'PULLBOARD_RELAY_TOKEN',
+ 'GIT_CONFIG_GLOBAL', 'GIT_CONFIG_NOSYSTEM', 'PATH'];
+ const saved = Object.fromEntries(names.map(name => [name, process.env[name]]));
+ for (const name of names) {
+ if (Object.hasOwn(env, name)) process.env[name] = env[name];
+ else delete process.env[name];
+ }
+ try { return await action(); }
+ finally {
+ for (const name of names) {
+ if (saved[name] === undefined) delete process.env[name];
+ else process.env[name] = saved[name];
+ }
+ }
+}
+
+/** Commit the fixture project configuration before link setup or creating a sibling worktree. */
+function commitConfiguration(box) {
+ const file = join(box.root, 'pullboard.json');
+ const config = JSON.parse(readFileSync(file, 'utf8'));
+ writeFileSync(file, JSON.stringify(config, null, 2) + '\n');
+ const cli = fileURLToPath(new URL('../bin/pullboard.js', import.meta.url));
+ const privateBin = box.env.PATH.split(delimiter)[0];
+ const pullboard = join(privateBin, 'pullboard');
+ assert.equal(existsSync(pullboard), false, 'the fixture has no preexisting private Pullboard wrapper to overwrite');
+ writeFileSync(pullboard, `#!/bin/sh\nexec node "${cli}" "$@"\n`, { mode: 0o700 });
+ runFixtureGit(['add', 'pullboard.json'], { cwd: box.root, env: box.env });
+ runFixtureGit(['commit', '-q', '-m', 'chore(config): prepare the canary project'], { cwd: box.root, env: box.env });
+}
+
+/** Create a real noncoordinator agent worktree sharing the fixture's local board and relay link. */
+async function joinedCanarySeat(box) {
+ const root = join(box.root, '..', 'watch-seat');
+ runFixtureGit(['worktree', 'add', '-q', '-b', 'agent/watch-canary', root], { cwd: box.root, env: box.env });
+ const joined = await box.cliAt(root, 'join', box.lane, '--model', 'watch-fixture');
+ assert.equal(joined.code, 0, joined.failure);
+ return { root, agentId: joined.document.agent };
+}
+
+test('the watch runs once per minute only while opted in [H5,H17]', { timeout: 45_000 }, async t => { // budget: private Git fixture setup and bounded CLI cleanup; cadence advances with a fake clock.
+ const box = await relayClientFixture(t);
+ const clock = fakeInterval();
+ let now = 10_000;
+ let calls = 0;
+ const watch = startRelayCanaryWatch({
+ config: () => ({ enabled: false, root: box.root, agentId: 'fixture-seat' }),
+ run: async () => { calls += 1; }, now: () => now,
+ every: clock.every, cancel: clock.cancel,
+ });
+ await watch.tick();
+ assert.equal(calls, 0, 'an explicit off setting does not probe or claim a minute');
+ assert.equal(existsSync(join(repoInfo(box.root).commonDir, 'pullboard', 'relay-health.sqlite')), false,
+ 'off configuration creates no canary schedule state');
+ await watch.close();
+ assert.equal(clock.timers[0].delay, RELAY_CANARY_INTERVAL_MS);
+ assert.equal(clock.timers[0].unrefed, true, 'the minute timer does not keep the client alive');
+ assert.equal(clock.timers[0].cancelled, true);
+
+ const enabledClock = fakeInterval();
+ const enabled = startRelayCanaryWatch({
+ config: () => ({ enabled: true, root: box.root, agentId: 'fixture-seat' }),
+ run: async () => { calls += 1; }, now: () => now,
+ every: enabledClock.every, cancel: enabledClock.cancel,
+ });
+ await enabled.tick();
+ assert.equal(calls, 1, 'the first enabled tick runs immediately');
+ await enabledClock.timers[0].callback();
+ assert.equal(calls, 1, 'a second tick inside the claimed minute is skipped');
+ now += RELAY_CANARY_INTERVAL_MS;
+ await enabledClock.timers[0].callback();
+ assert.equal(calls, 2, 'the next due minute is admitted');
+ await enabled.close();
+});
+
+test('same-root watches share atomic admission and close aborts and drains a single run [H5,H17]', { timeout: 45_000 }, async t => { // budget: private Git/SQLite setup and a manually released in-flight run; no polling delay.
+ const box = await relayClientFixture(t);
+ const clockA = fakeInterval();
+ const clockB = fakeInterval();
+ const now = 50_000;
+ let calls = 0;
+ let active = 0;
+ let maximumActive = 0;
+ let signal;
+ let releaseRun;
+ let resolveStarted;
+ let drained = false;
+ const started = new Promise(resolve => { resolveStarted = resolve; });
+ const held = new Promise(resolve => { releaseRun = resolve; });
+ const run = async (_config, { io }) => {
+ calls += 1;
+ active += 1;
+ maximumActive = Math.max(maximumActive, active);
+ signal = io.relaySignal;
+ resolveStarted();
+ await held;
+ active -= 1;
+ drained = true;
+ };
+ const config = () => ({ enabled: true, root: box.root, agentId: 'fixture-seat' });
+ const first = startRelayCanaryWatch({ config, run, now: () => now, every: clockA.every, cancel: clockA.cancel });
+ const second = startRelayCanaryWatch({ config, run, now: () => now, every: clockB.every, cancel: clockB.cancel });
+ await started;
+ // Trigger competing/repeated ticks without awaiting a potentially winning held run.
+ const competingTick = second.tick();
+ const repeatedTicks = [first.tick(), first.tick()];
+ await new Promise(resolve => setImmediate(resolve));
+ assert.equal(calls, 1, 'the shared SQLite minute claim admits one of two same-root watches');
+ assert.equal(maximumActive, 1, 'repeated ticks never overlap the watch’s in-flight run');
+
+ let closed = false;
+ const closing = Promise.all([first.close(), second.close()]).then(() => { closed = true; });
+ assert.equal(signal.aborted, true, 'closing the owning watch aborts the active probe signal');
+ await Promise.resolve();
+ assert.equal(closed, false, 'close waits for the active run to finish its cleanup');
+ assert.equal(drained, false);
+ releaseRun();
+ await Promise.all([closing, competingTick, ...repeatedTicks]);
+ assert.equal(drained, true);
+ assert.equal(active, 0);
+
+ const file = join(repoInfo(box.root).commonDir, 'pullboard', 'relay-health.sqlite');
+ const db = new DatabaseSync(file, { readOnly: true });
+ try {
+ assert.deepEqual(db.prepare('SELECT id, next_due FROM canary_schedule').all().map(row => ({ ...row })), [{ id: 1, next_due: now + RELAY_CANARY_INTERVAL_MS }],
+ 'both watch processes use one durable SQLite schedule row');
+ } finally { db.close(); }
+});
+
+test('a pending alert notice retries after delivery failure and is removed only after success [H5,H17]', { timeout: 60_000 }, async t => { // budget: linked loopback fixture, three bounded probes, and fixture cleanup.
+ const box = await relayClientFixture(t);
+ commitConfiguration(box);
+ await box.link();
+ const seat = await joinedCanarySeat(box);
+ await box.stopRelay();
+ const config = { enabled: true, root: seat.root, agentId: seat.agentId, deadlineMs: 5_000 };
+ let now = Date.now();
+ const attemptIds = [];
+ await withFixtureEnvironment(box.env, () => runRelayCanary(config, { now: () => now,
+ notify: async notice => { attemptIds.push(notice.id); } }));
+
+ now += 60_000;
+ await assert.rejects(withFixtureEnvironment(box.env, () => runRelayCanary(config, { now: () => now,
+ notify: async notice => { attemptIds.push(notice.id); throw new Error('fixture delivery failure'); } })), /fixture delivery failure/u);
+ const pending = pendingRelayCanaryNotices(box.root);
+ assert.equal(pending.length, 1, 'the unrecorded alert stays durable after delivery throws');
+ assert.equal(attemptIds.length, 1, 'the first failed probe did not create a notification');
+
+ now += 60_000;
+ await withFixtureEnvironment(box.env, () => runRelayCanary(config, { now: () => now,
+ notify: async notice => { attemptIds.push(notice.id); } }));
+ assert.deepEqual(attemptIds, [pending[0].id, pending[0].id], 'retry uses the same durable notice identity');
+ assert.deepEqual(pendingRelayCanaryNotices(box.root), [], 'successful ordinary notice recording clears the pending delivery');
+
+});
+
+
+test('the canary command binds machine opt-in to its linked ordinary seat [H5,H17]', { timeout: 60_000 }, async t => { // budget: real linked Git fixture and bounded ordinary sealed probe, not a timing assertion.
+ const box = await relayClientFixture(t);
+ commitConfiguration(box);
+ await box.link();
+ const refused = await box.cli('relay', 'canary', 'on');
+ assert.equal(refused.code, 1, 'a coordinator cannot substitute for the dedicated ordinary canary seat');
+ assert.match(refused.failure, /RELAY_CANARY_SEAT/u);
+ const seat = await joinedCanarySeat(box);
+ const enabled = await box.cliAt(seat.root, 'relay', 'canary', 'on', '--timeout', '5000');
+ assert.equal(enabled.code, 0, enabled.failure);
+ assert.equal(enabled.document.config.agentId, seat.agentId);
+ const probed = await box.cliAt(seat.root, 'relay', 'canary', 'run');
+ assert.equal(probed.code, 0, probed.failure);
+ assert.equal(probed.document.observation.ok, true);
+ assert.equal(probed.document.observation.step, 'complete');
+ const off = await box.cliAt(seat.root, 'relay', 'canary', 'off');
+ assert.equal(off.code, 0, off.failure);
+ assert.equal(off.document.config.enabled, false);
+ const stopped = await box.cliAt(seat.root, 'relay', 'canary', 'run');
+ assert.equal(stopped.code, 1);
+ assert.match(stopped.failure, /RELAY_CANARY_CONFIG/u);
+});