Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

test(relay): browser fault study and per-move native diff [H5,H17]

Corey Olsonaf3bcb993a3c
diff --git a/test/relay-study-browser.study.js b/test/relay-study-browser.study.js
new file mode 100644
index 00000000..6a6a28f8
--- /dev/null
+++ b/test/relay-study-browser.study.js
@@ -0,0 +1,72 @@
+/** @pullboard-test-gui */
+/** What a paired browser does when its relay is down, slow, failing or refusing its engine [H5,H17]. Run it explicitly: the default gate skips *.study.js. */
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { performance } from 'node:perf_hooks';
+import { test } from 'node:test';
+import { relayClientFixture } from './relay-client-fixture.js';
+import { findChromeExecutable, startChrome } from './relay-browser-fixture.js';
+import { faultProxy } from './relay-study-fixture.js';
+
+// Budget: declared observation window; the bar names a 5 s slow relay, so each fault is watched for four of those.
+const OBSERVE_MS = 20_000;
+// Budget: declared recovery ceiling after the fault clears, measured, never asserted.
+const RECOVER_MS = 30_000;
+
+/** Read what the page shows now: the relay notice, the live line, and whether each title is on the board. */
+async function pageState(chrome, titles) {
+ return JSON.parse(await chrome.evaluate(`JSON.stringify({
+ notice: (document.querySelector('#relay-notice')?.textContent ?? '').trim().slice(0, 200),
+ live: (document.querySelector('#live')?.textContent ?? '').trim().slice(0, 200),
+ shown: ${JSON.stringify(titles)}.map(title => (document.querySelector('#chain')?.textContent ?? '').includes(title)),
+ })`));
+}
+
+/** Poll the page until a condition holds or the window ends, keeping every distinct state it passed through. */
+async function watch(chrome, titles, windowMs, done) {
+ const started = performance.now(), seen = [];
+ let state;
+ do {
+ state = await pageState(chrome, titles);
+ const key = JSON.stringify(state);
+ if (seen.at(-1)?.key !== key) seen.push({ key, atMs: Math.round(performance.now() - started), ...state });
+ if (done(state)) return { reachedMs: Math.round(performance.now() - started), seen };
+ await new Promise(resolve => setTimeout(resolve, 500));
+ } while (performance.now() - started < windowMs);
+ return { reachedMs: null, seen };
+}
+
+test('relay study measures a paired browser across relay faults [H5,H17]', {
+ timeout: 600_000, skip: !findChromeExecutable() && 'Chrome is not installed',
+}, async t => {
+ const box = await relayClientFixture(t);
+ const first = 'Browser study seed';
+ assert.equal((await box.cli('add', box.lane, first)).code, 0);
+ await box.link();
+ const link = JSON.parse(readFileSync(box.linkFile, 'utf8'));
+ const key = readFileSync(box.keyFile, 'utf8').trim();
+ const proxy = await faultProxy(t, box, { rewriteLink: false });
+ const chrome = await startChrome();
+ t.after(() => chrome.close());
+ const cookie = await chrome.send('Network.setCookie', { name: 'pb_session', value: (await box.phoneSession()).token,
+ url: proxy.origin, httpOnly: true, sameSite: 'Lax' });
+ assert.equal(cookie.success, true);
+ await chrome.navigate(proxy.origin + '/#board=' + link.board + '&key=' + key);
+ await chrome.waitFor("document.querySelector('#chain')?.textContent.includes(" + JSON.stringify(first) + ')');
+ for (const fault of ['down', 'slow-5s', '5xx', 'stale-engine']) {
+ proxy.set(fault === 'down' ? { down: true } : { delayMs: fault === 'slow-5s' ? 5000 : 0,
+ status: fault === '5xx' ? 503 : fault === 'stale-engine' ? 409 : null,
+ code: fault === 'stale-engine' ? 'ENGINE_VERSION' : 'RELAY_UNAVAILABLE' });
+ proxy.drop();
+ const title = 'Browser study move during ' + fault;
+ assert.equal((await box.cli('add', box.lane, title)).code, 0, 'the native board keeps moving on its direct link');
+ const during = await watch(chrome, [first, title], OBSERVE_MS, state => state.shown[1]);
+ proxy.set();
+ proxy.drop();
+ const after = during.reachedMs === null ? await watch(chrome, [first, title], RECOVER_MS, state => state.shown[1]) : null;
+ t.diagnostic(JSON.stringify({ phase: 'browser-transport-failure', fault, node: process.versions.node,
+ newMoveShownDuringFaultMs: during.reachedMs, statesDuringFault: during.seen.map(({ key, ...state }) => state),
+ recoveredAfterClearMs: after?.reachedMs ?? null, statesAfterClear: after?.seen.map(({ key, ...state }) => state) ?? [],
+ scope: 'Real Chrome paired through a loopback fault proxy in front of the private relay; the CLI publishes directly.' }));
+ }
+});
diff --git a/test/relay-study-fixture.js b/test/relay-study-fixture.js
index 00c26b86..ba25dca5 100644
--- a/test/relay-study-fixture.js
+++ b/test/relay-study-fixture.js
@@ -12,7 +12,7 @@ import { openPagedSnapshot } from '../src/relay-page-layout.js';
* publisher's and the comparing reader's values differ by the seconds between them (coordinator ruling 31625).
*/
function asOfField(path, name) {
- return ['observedAt', 'asOf', 'ageMinutes'].includes(name) || name === 'minutes' && path.includes('["stages"]');
+ return ['observedAt', 'asOf', 'ageMinutes'].includes(name) || name === 'minutes' && (path.includes('["stages"]') || path.includes('.stages.'));
}
/** Compare every JSON field except as-of clock readings, retaining paths but never printing board contents or credentials. */
@@ -70,13 +70,14 @@ export async function readRelay(box) {
* Put a real HTTP fault proxy between the linked CLI and the fixture relay by pointing this scratch board's link at
* it. API requests can be delayed, or answered with a refusal, without changing the shared relay fixture.
*/
-export async function faultProxy(t, box) {
+export async function faultProxy(t, box, { rewriteLink = true } = {}) {
const target = new URL(box.origin);
let fault = null;
const server = createServer((req, res) => {
/** Forward one request unchanged, after any configured delay or instead of a configured refusal. */
async function handle() {
const api = (req.url ?? '').startsWith('/api/');
+ if (api && fault?.down) { req.socket.destroy(); return; }
if (api && fault?.delayMs) await new Promise(resolve => setTimeout(resolve, fault.delayMs));
if (api && fault?.status) {
req.resume();
@@ -85,7 +86,7 @@ export async function faultProxy(t, box) {
return;
}
const upstream = httpRequest({ host: target.hostname, port: target.port, method: req.method, path: req.url,
- headers: { ...req.headers, host: target.host } }, reply => { res.writeHead(reply.statusCode, reply.headers); reply.pipe(res); });
+ headers: { ...req.headers, host: target.host, ...(req.headers.origin ? { origin: box.origin } : {}) } }, reply => { res.writeHead(reply.statusCode, reply.headers); reply.pipe(res); });
upstream.on('error', () => { if (!res.headersSent) res.writeHead(502); res.end(); });
req.pipe(upstream);
}
@@ -93,16 +94,51 @@ export async function faultProxy(t, box) {
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
t.after(() => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }));
- const original = readFileSync(box.linkFile, 'utf8');
- const state = JSON.parse(original);
- state.url = 'http://127.0.0.1:' + server.address().port;
- writeFileSync(box.linkFile, JSON.stringify(state), { mode: 0o600 });
+ const origin = 'http://127.0.0.1:' + server.address().port;
+ if (rewriteLink) writeFileSync(box.linkFile, JSON.stringify({ ...JSON.parse(readFileSync(box.linkFile, 'utf8')), url: origin }), { mode: 0o600 });
return {
- /** Set the fault for later API requests: { delayMs, status, code }, or null to pass them through. */
+ origin,
+ /** Drop every open connection, so a live stream must reconnect through the current fault. */
+ drop() { server.closeAllConnections(); },
+ /** Set the fault for later API requests: { down } or { delayMs, status, code }, or null to pass them through. */
set(value = null) {
- assert.ok(value === null || Number.isSafeInteger(value.delayMs) && value.delayMs >= 0
+ assert.ok(value === null || value.down === true || Number.isSafeInteger(value.delayMs) && value.delayMs >= 0
&& (value.status === null || [409, 503].includes(value.status)));
fault = value;
},
};
}
+
+/** Collect every key path, with array members folded to [], so shapes compare without board contents. */
+export function paths(value, prefix = '$', found = new Set()) {
+ found.add(prefix);
+ if (Array.isArray(value)) for (const member of value) paths(member, prefix + '[]', found);
+ else if (value && typeof value === 'object') for (const [name, child] of Object.entries(value)) paths(child, prefix + '.' + name, found);
+ return found;
+}
+
+/** Name the common scalar paths whose values differ between the two states, sampled by item and agent id. */
+export function valueDifferences(native, relay) {
+ const found = new Set();
+ /** Compare two values below one path, matching array members by their id when they carry one. */
+ function walk(left, right, path) {
+ if (Array.isArray(left) && Array.isArray(right)) {
+ const key = member => member && typeof member === 'object' ? member.id ?? member.agent_id ?? member.shout_id ?? null : null;
+ const byKey = new Map(right.map(member => [key(member), member]));
+ left.forEach((member, index) => {
+ const other = key(member) === null ? right[index] : byKey.get(key(member));
+ if (other !== undefined) walk(member, other, path + '[]');
+ });
+ return;
+ }
+ if (left && right && typeof left === 'object' && typeof right === 'object') {
+ for (const name of Object.keys(left)) {
+ if (Object.hasOwn(right, name) && !asOfField(path, name)) walk(left[name], right[name], path + '.' + name);
+ }
+ return;
+ }
+ if (JSON.stringify(left) !== JSON.stringify(right)) found.add(path);
+ }
+ walk(native, relay, '$');
+ return [...found].sort();
+}
diff --git a/test/relay-study-moves.study.js b/test/relay-study-moves.study.js
index a84f67ef..f27323e6 100644
--- a/test/relay-study-moves.study.js
+++ b/test/relay-study-moves.study.js
@@ -8,7 +8,11 @@ import * as milestones from '../src/milestones.js';
import { MOVE_OPERATIONS } from '../src/move-registry.js';
import { relaySnapshot } from '../src/relay-presentation.js';
import { relayClientFixture } from './relay-client-fixture.js';
-import { differences, readRelay } from './relay-study-fixture.js';
+import { differences, paths, readRelay, valueDifferences } from './relay-study-fixture.js';
+import { serveApi } from '../src/api.js';
+import { main } from '../src/cli.js';
+import { presentationState } from '../relay/browser-model.js';
+import { apiFetch } from './api-fixture.js';
import { submitFixture } from './board-submit-fixture.js';
import { syntheticFlakeObservation } from './flake-observation-fixture.js';
@@ -17,7 +21,28 @@ test('relay study compares each exercised move effect through the real API2 read
await box.link();
const board = store.openBoard(join(box.root, '.git', 'pullboard', 'board.sqlite'));
t.after(() => store.closeBoard(board));
- const covered = new Set(), setupRefusals = [];
+ const covered = new Set(), setupRefusals = [], nativeParity = [];
+ const api = await serveApi({ runCommand: main, projects: () => [{ root: box.root, name: 'study', project: '', added: '2026-10-26T00:00:00.000Z' }], stateDetail: 'full' });
+ t.after(() => api.close());
+ const address = new URL(api.url);
+ const nativeApi = { origin: address.origin, secret: address.searchParams.get('k') };
+ const boardId = (await (await apiFetch(nativeApi, '/api/v1/boards')).json()).boards[0].id;
+ /** Diff the native view's state against what a relay reader renders from the decrypted presentation. */
+ async function compareNative(operation, presentation) {
+ const native = (await (await apiFetch(nativeApi, '/api/v1/boards/' + boardId + '/state')).json()).state;
+ const reader = presentationState(presentation, boardId);
+ const nativePaths = paths(native), readerPaths = paths(reader);
+ nativeParity.push({ operation, onlyNative: [...nativePaths].filter(path => !readerPaths.has(path)).sort(),
+ onlyReader: [...readerPaths].filter(path => !nativePaths.has(path)).sort(), differingValues: valueDifferences(native, reader) });
+ }
+ t.after(() => {
+ const union = field => [...new Set(nativeParity.flatMap(entry => entry[field]))].sort();
+ t.diagnostic(JSON.stringify({ phase: 'native-versus-reader-per-move', node: process.versions.node, operations: nativeParity.length,
+ onlyNative: union('onlyNative'), onlyReader: union('onlyReader'), differingValues: union('differingValues'),
+ perOperation: nativeParity.map(entry => ({ operation: entry.operation, onlyNative: entry.onlyNative.length,
+ onlyReader: entry.onlyReader, differingValues: entry.differingValues })),
+ scope: 'After each exercised move: GET /api/v1/boards/:id/state (stateDetail full) against presentationState of the decrypted relay presentation; as-of clock readings ignored.' }));
+ });
t.after(() => t.diagnostic(JSON.stringify({ phase: 'move-effect-parity', node: process.versions.node,
coveredMoves: [...covered], setupRefusals, uncoveredMoves: MOVE_OPERATIONS.filter(operation => !covered.has(operation)),
limitation: 'Native board-state fixtures with actual API2 transport; synthetic submit is explicitly not a gate proof. Only successful event-producing calls counted.' })));
@@ -41,6 +66,7 @@ test('relay study compares each exercised move effect through the real API2 read
const read = await readRelay(box);
assert.deepEqual(differences(expected.presentation, read.document.presentation), [], operation + ' full presentation');
assert.deepEqual(differences(expected.tables, read.document.tables), [], operation + ' native tables');
+ await compareNative(operation, read.document.presentation);
covered.add(operation);
return result;
}
diff --git a/test/relay-study-native.study.js b/test/relay-study-native.study.js
index 3c340595..dac63678 100644
--- a/test/relay-study-native.study.js
+++ b/test/relay-study-native.study.js
@@ -8,38 +8,7 @@ import { relayPresentation } from '../src/relay-presentation.js';
import { presentationState } from '../relay/browser-model.js';
import { relayClientFixture } from './relay-client-fixture.js';
import { apiFetch } from './api-fixture.js';
-
-/** Collect every key path, with array members folded to [], so shapes compare without board contents. */
-function paths(value, prefix = '$', found = new Set()) {
- found.add(prefix);
- if (Array.isArray(value)) for (const member of value) paths(member, prefix + '[]', found);
- else if (value && typeof value === 'object') for (const [name, child] of Object.entries(value)) paths(child, prefix + '.' + name, found);
- return found;
-}
-
-/** Name the common scalar paths whose values differ between the two states, sampled by item and agent id. */
-function valueDifferences(native, relay) {
- const found = new Set();
- /** Compare two values below one path, matching array members by their id when they carry one. */
- function walk(left, right, path) {
- if (Array.isArray(left) && Array.isArray(right)) {
- const key = member => member && typeof member === 'object' ? member.id ?? member.agent_id ?? member.shout_id ?? null : null;
- const byKey = new Map(right.map(member => [key(member), member]));
- left.forEach((member, index) => {
- const other = key(member) === null ? right[index] : byKey.get(key(member));
- if (other !== undefined) walk(member, other, path + '[]');
- });
- return;
- }
- if (left && right && typeof left === 'object' && typeof right === 'object') {
- for (const name of Object.keys(left)) if (Object.hasOwn(right, name)) walk(left[name], right[name], path + '.' + name);
- return;
- }
- if (JSON.stringify(left) !== JSON.stringify(right)) found.add(path);
- }
- walk(native, relay, '$');
- return [...found].sort();
-}
+import { paths, valueDifferences } from './relay-study-fixture.js';
test('relay study diffs the native view state and the relay reader state [H5,H17]', { timeout: 180_000 }, async t => {
const box = await relayClientFixture(t);