Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

feat(relay): resume bounded native snapshots [H15,H17]

Corey Olson2b21cf5d3b96
diff --git a/relay/page-store.js b/relay/page-store.js
index b2b87f1a..fe20fce6 100644
--- a/relay/page-store.js
+++ b/relay/page-store.js
@@ -186,7 +186,11 @@ export function createRelayPages({ directory, boardId, now = Date.now, database,
const staged = upload(id, principal);
if (reference.sequence !== staged.sequence) throw new Refused('BAD_SEQUENCE', 'Publish the coverage sequence named when this upload began.');
const root = db.prepare('SELECT sequence,payload FROM page_object WHERE id=?').get(reference.id);
- if (!root || root.sequence !== reference.sequence) throw new Refused('SNAPSHOT_PAGE_MISSING', 'Upload this snapshot’s sealed root before publishing it.');
+ if (!root || root.sequence !== reference.sequence) {
+ const error = new Refused('SNAPSHOT_PAGE_MISSING', 'Upload this snapshot’s sealed root before publishing it.');
+ error.page = reference.id;
+ throw error;
+ }
graph([reference.id], true);
const result = publishHead(reference, Buffer.from(root.payload));
if (result && typeof result.then === 'function') throw new Refused('RELAY_CONFIG', 'Commit the journal head synchronously before finishing publication.');
diff --git a/relay/pages-http.js b/relay/pages-http.js
index ac84bff1..9b25872f 100644
--- a/relay/pages-http.js
+++ b/relay/pages-http.js
@@ -43,11 +43,11 @@ export function createRelayPagesHandler({ directory, now, authorize, withJournal
const snapshot = journal.snapshot();
if (!snapshot) throw new Refused('NO_SNAPSHOT', 'Upload a sealed snapshot before reading this board.');
const root = snapshotPageRoot(snapshot);
- if (!root) return { ...annotated(snapshot, id), apiVersion: 1 };
+ if (!root) return { ...annotated(snapshot, id), apiVersion: 1, storage: quota.usage(id) };
const read = pages.pin(principal, root);
const warning = quota.notice(id);
return { apiVersion: 2, sequence: snapshot.sequence, receivedAt: snapshot.receivedAt, sender: snapshot.sender,
- root, read, ...(warning ? { warning } : {}) };
+ root, read, storage: quota.usage(id), ...(warning ? { warning } : {}) };
}
/** Route only bounded opaque operations; no request supplies the account charged for them. */
diff --git a/src/cli.js b/src/cli.js
index 2cf33203..c7362c12 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -420,6 +420,7 @@ import {
relayRecovered,
relayRevoke,
relayStatus,
+ relayStorageLine,
relayTokens,
syncRelay,
} from './relay.js';
@@ -2538,6 +2539,7 @@ function readCommands(io, { first, second, rest, values }) {
}
if (summary.relay.linked) io.say(`relay: sequence ${summary.relay.sequence}; ${summary.relay.behind} pending uploads${summary.relay.recovery?.pending ? `; recovery pending for sequence ${summary.relay.recovery.skip}; retry ${summary.relay.recovery.next}` : ''}`);
if (summary.relay.oversizedSnapshot && !summary.relay.refusedCheckpoint) io.say(formatSnapshotLimit(summary.relay.oversizedSnapshot));
+ if (summary.relay.storage) io.say(relayStorageLine(summary.relay.storage));
const { items, accepted, rejected } = summary.stats;
io.say(`${summary.me.displayName}: ${summary.unread} unread shouts; holding ${summary.mine.map((item) => `#${item.item_id}`).join(', ') || 'nothing'}`);
io.say(`board: ${items.open} open, ${items.claimed} claimed, ${items.submitted} awaiting verification, ${items.verified} verified, ${items.withdrawn} withdrawn`);
@@ -4468,6 +4470,7 @@ async function runCommand(argv, io) {
io.result?.(result);
const boardName = listApiProjects().find((project) => project.root === ctx.info.root)?.name ?? basename(ctx.info.root);
io.say(result.linked ? `${result.link}\nrelay: ${boardName} (board ${result.board}) is linked for GitHub ${result.login || `account ${result.account}`}; sequence ${result.sequence}; ${result.behind} pending uploads${result.recovery?.pending ? `; recovery pending for sequence ${result.recovery.skip}; retry ${result.recovery.next}` : ''}` : (result.notice || 'relay off; the local board is complete'));
+ if (result.storage) io.say(relayStorageLine(result.storage));
return 0;
}
if (command === 'blueprint' && first === 'save') {
diff --git a/src/relay-page-cache.js b/src/relay-page-cache.js
new file mode 100644
index 00000000..92d166b9
--- /dev/null
+++ b/src/relay-page-cache.js
@@ -0,0 +1,159 @@
+/** Private device-only cipher cache and durable upload graph; no board key or decrypted record is stored [H15,H17]. */
+import { createHash } from 'node:crypto';
+import { closeSync, lstatSync, mkdirSync, openSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { Refused } from './refused.js';
+import { PAGE_CIPHER_BYTES, PAGE_FANOUT, pageReference, sealedPageFingerprint } from './sealed-pages.js';
+
+/** Refuse a damaged private cache without repeating any bytes or private fingerprints. */
+function invalid() { return new Refused('RELAY_PAGE_CACHE', 'Restore private relay page cache storage, or remove this device cache and retry the saved snapshot.'); }
+
+/** Hash a local identity without making a repository name, token or relay address a filename. */
+export function snapshotSourceId(value) { return createHash('sha256').update(value).digest('hex'); }
+
+/** Keep one board and relay origin's immutable cipher objects, private reuse mappings and unfinished uploads. */
+export function createRelayPageCache(state) {
+ if (!/^[0-9a-f]{32}$/u.test(state?.board ?? '') || typeof state.url !== 'string') throw invalid();
+ const directory = join(process.env.PULLBOARD_HOME || join(homedir(), '.pullboard'), 'relay-pages');
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
+ const parent = lstatSync(directory);
+ if (!parent.isDirectory() || parent.isSymbolicLink() || parent.mode & 0o077) throw invalid();
+ const origin = snapshotSourceId(state.url);
+ const file = join(directory, state.board + '-' + origin + '.sqlite');
+ try { closeSync(openSync(file, 'wx', 0o600)); } catch (error) { if (error.code !== 'EEXIST') throw error; }
+ const stat = lstatSync(file);
+ if (!stat.isFile() || stat.isSymbolicLink() || stat.mode & 0o077) throw invalid();
+ const db = new DatabaseSync(file);
+ try {
+ db.exec(`PRAGMA busy_timeout=30000; PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;
+ CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY,value TEXT NOT NULL);
+ CREATE TABLE IF NOT EXISTS object (id TEXT PRIMARY KEY,sequence INTEGER NOT NULL,payload BLOB NOT NULL,children TEXT NOT NULL,ack INTEGER NOT NULL DEFAULT 0);
+ CREATE TABLE IF NOT EXISTS fingerprint (hash TEXT PRIMARY KEY,id TEXT NOT NULL REFERENCES object(id) ON DELETE CASCADE,reference TEXT NOT NULL);
+ CREATE TABLE IF NOT EXISTS job (source TEXT PRIMARY KEY,reference TEXT NOT NULL,upload TEXT);
+ CREATE TABLE IF NOT EXISTS job_object (source TEXT NOT NULL REFERENCES job(source) ON DELETE CASCADE,id TEXT NOT NULL REFERENCES object(id),PRIMARY KEY(source,id));`);
+ const saved = db.prepare('SELECT key,value FROM meta ORDER BY key').all();
+ if (!saved.length) db.prepare('INSERT INTO meta VALUES (?,?),(?,?),(?,?)').run('board', state.board, 'format', '1', 'origin', origin);
+ else if (JSON.stringify(saved.map(row => [row.key, row.value])) !== JSON.stringify([['board', state.board], ['format', '1'], ['origin', origin]])) throw invalid();
+ } catch (error) { db.close(); throw error; }
+
+ /** Validate both ciphertext identity and opaque metadata before reusing a private object. */
+ function object(id) {
+ if (!/^[0-9a-f]{64}$/u.test(id ?? '')) throw invalid();
+ const row = db.prepare('SELECT * FROM object WHERE id=?').get(id);
+ if (!row || !Number.isSafeInteger(row.sequence) || row.sequence < 0 || row.payload.byteLength > PAGE_CIPHER_BYTES || snapshotSourceId(row.payload) !== id) throw invalid();
+ let children;
+ try { children = JSON.parse(row.children); } catch { throw invalid(); }
+ if (!Array.isArray(children) || children.length > PAGE_FANOUT || children.some(child => !/^[0-9a-f]{64}$/u.test(child))) throw invalid();
+ return { id, sequence: row.sequence, bytes: Uint8Array.from(row.payload), children, acknowledged: row.ack === 1 };
+ }
+
+ /** Store immutable bytes before recording any reference or upload progress. */
+ function put(value) {
+ const id = snapshotSourceId(value.bytes);
+ if (id !== value.id || value.bytes.byteLength > PAGE_CIPHER_BYTES || !Number.isSafeInteger(value.sequence) || value.sequence < 0 ||
+ !Array.isArray(value.children) || value.children.length > PAGE_FANOUT || value.children.some(child => !/^[0-9a-f]{64}$/u.test(child))) throw invalid();
+ const saved = db.prepare('SELECT id FROM object WHERE id=?').get(id);
+ if (saved) {
+ const previous = object(id);
+ if (previous.sequence !== value.sequence || JSON.stringify(previous.children) !== JSON.stringify(value.children)) throw invalid();
+ return;
+ }
+ db.prepare('INSERT INTO object(id,sequence,payload,children) VALUES (?,?,?,?)').run(id, value.sequence, value.bytes, JSON.stringify(value.children));
+ }
+
+ /** Verify decrypted cache identity instead of trusting a stale or misbound private fingerprint row. */
+ async function lookup(hash, context) {
+ const saved = db.prepare('SELECT reference FROM fingerprint WHERE hash=?').get(hash);
+ if (!saved) return null;
+ let reference;
+ try { reference = pageReference(JSON.parse(saved.reference)); } catch { throw invalid(); }
+ const actual = await sealedPageFingerprint(reference, { ...context, get: id => object(id).bytes, parents: [] });
+ if (actual !== hash) throw invalid();
+ return reference;
+ }
+
+ /** A plaintext hash is private device cache metadata and never part of an upload object. */
+ function remember(hash, reference) {
+ if (!/^[0-9a-f]{64}$/u.test(hash ?? '')) throw invalid();
+ const ref = pageReference(reference);
+ db.prepare('INSERT INTO fingerprint VALUES (?,?,?) ON CONFLICT(hash) DO UPDATE SET id=excluded.id,reference=excluded.reference').run(hash, ref.id, JSON.stringify(ref));
+ }
+
+ /** Read a durable job only under its exact original pending-cipher source identity. */
+ function job(source) {
+ if (!/^[0-9a-f]{64}$/u.test(source ?? '')) throw invalid();
+ const row = db.prepare('SELECT reference,upload FROM job WHERE source=?').get(source);
+ if (!row) return null;
+ let reference;
+ try { reference = pageReference(JSON.parse(row.reference)); } catch { throw invalid(); }
+ return { reference, upload: row.upload };
+ }
+
+ /** Atomically retain only this exact prepared graph as the job's upload/resume set. */
+ function prepare(source, reference) {
+ pageReference(reference);
+ db.exec('BEGIN IMMEDIATE');
+ try {
+ db.prepare('INSERT INTO job VALUES (?,?,NULL) ON CONFLICT(source) DO UPDATE SET reference=excluded.reference,upload=NULL').run(source, JSON.stringify(reference));
+ db.prepare('DELETE FROM job_object WHERE source=?').run(source);
+ const found = new Set(), visiting = new Set();
+ /** Visit each local child once, refusing cycles and missing data before publishing. */
+ function visit(id, depth = 0) {
+ if (visiting.has(id) || depth >= 64) throw invalid();
+ if (found.has(id)) return;
+ visiting.add(id);
+ const value = object(id);
+ for (const child of value.children) visit(child, depth + 1);
+ visiting.delete(id); found.add(id);
+ db.prepare('INSERT INTO job_object VALUES (?,?)').run(source, id);
+ }
+ visit(reference.id);
+ db.exec('COMMIT');
+ } catch (error) { db.exec('ROLLBACK'); throw error; }
+ }
+
+ /** Record the server's principal-bound upload id before the first object request. */
+ function upload(source, id) {
+ if (!/^[A-Za-z0-9-]{1,128}$/u.test(id ?? '')) throw invalid();
+ db.prepare('UPDATE job SET upload=? WHERE source=?').run(id, source);
+ }
+
+ /** Stream only objects not yet acknowledged by this relay, without an aggregate request body. */
+ function* pending(source) {
+ for (const row of db.prepare('SELECT object.id FROM object JOIN job_object USING(id) WHERE source=? AND ack=0 ORDER BY object.rowid').iterate(source)) yield object(row.id);
+ }
+
+ /** A lost ACK safely retries the same immutable object; only an actual ACK marks it done. */
+ function acknowledge(id) { db.prepare('UPDATE object SET ack=1 WHERE id=?').run(id); }
+
+ /** A collected remote subtree must be re-staged; an old successful upload is not perpetual remote custody. */
+ function missing(id) {
+ const found = new Set();
+ /** Reset the complete local subtree so publication never depends on a stale server-ACK cache. */
+ function visit(child, depth = 0) {
+ if (depth >= 64) throw invalid();
+ if (found.has(child)) return;
+ found.add(child);
+ const value = object(child);
+ db.prepare('UPDATE object SET ack=0 WHERE id=?').run(child);
+ for (const next of value.children) visit(next, depth + 1);
+ }
+ visit(id);
+ }
+
+ /** After a real root ACK, discard superseded jobs and locally cached pages unreachable from the current graph. */
+ function complete(source) {
+ db.exec('BEGIN IMMEDIATE');
+ try {
+ db.prepare('DELETE FROM job WHERE source<>?').run(source);
+ db.exec('DELETE FROM object WHERE id NOT IN (SELECT id FROM job_object)');
+ db.exec('COMMIT');
+ } catch (error) { db.exec('ROLLBACK'); throw error; }
+ }
+
+ /** Close the private cache after every request, including refused or interrupted uploads. */
+ function close() { db.close(); }
+ return { file, object, put, lookup, remember, job, prepare, upload, pending, acknowledge, missing, complete, close };
+}
diff --git a/src/relay-page-client.js b/src/relay-page-client.js
new file mode 100644
index 00000000..19ecc7e7
--- /dev/null
+++ b/src/relay-page-client.js
@@ -0,0 +1,173 @@
+/** Native bounded snapshot upload/resume and complete restore; no private cache fingerprint crosses HTTP [H15,H17]. */
+import { Refused } from './refused.js';
+import { createRelayPageCache, snapshotSourceId } from './relay-page-cache.js';
+import { openPagedSnapshot, writePagedSnapshot } from './relay-page-layout.js';
+import { PAGE_CIPHER_BYTES, pageReference } from './sealed-pages.js';
+
+const PAGE_CAPABILITIES = new WeakMap();
+
+/** Refuse a malformed ACK instead of clearing the durable native pending snapshot. */
+function invalid() { return new Refused('RELAY_RESPONSE', 'The relay did not acknowledge this sealed snapshot page. Retry the saved upload with the configured relay.'); }
+
+/** Decode only bounded canonical cipher bytes received under a content-addressed object id. */
+function bytes(value) {
+ if (typeof value !== 'string' || value.length > 600_000 || !/^[A-Za-z0-9_-]+$/u.test(value)) throw invalid();
+ const decoded = Buffer.from(value, 'base64url');
+ if (!decoded.byteLength || decoded.byteLength > PAGE_CIPHER_BYTES || decoded.toString('base64url') !== value) throw invalid();
+ return Uint8Array.from(decoded);
+}
+
+/** Keep account quota status only after validating its credential-free response fields. */
+export function relayStorage(value) {
+ if (!value || typeof value.account?.id !== 'string' || typeof value.account?.login !== 'string' ||
+ !/^[A-Za-z0-9_-]{1,100}$/u.test(value.account.id) || !/^[A-Za-z0-9_-]{1,100}$/u.test(value.account.login) ||
+ !Number.isSafeInteger(value.usedBytes) || value.usedBytes < 0 || !Number.isSafeInteger(value.limitBytes) || value.limitBytes < 1) return null;
+ return { account: { id: value.account.id, login: value.account.login }, usedBytes: value.usedBytes, limitBytes: value.limitBytes };
+}
+
+/** Print account, used bytes and the operator-set limit without importing server-only quota storage into the CLI. */
+export function relayStorageLine(value) {
+ const storage = relayStorage(value);
+ return storage ? `relay storage: account ${storage.account.login}, ${(storage.usedBytes / 1_000_000).toFixed(1)} MB used of ${(storage.limitBytes / 1_000_000).toFixed(1)} MB` : '';
+}
+
+/** Adapt the old internal pending-record interface only after the new protocol has proved an exact root ACK. */
+export async function pagedSnapshotRequest(state, options, { request, key, decode, encode, maxEngine }) {
+ const path = '/api/v2/boards/' + state.board;
+ /** Retain the server-derived account totals from successful writes and reads. */
+ function storage(value) { const result = relayStorage(value); if (result) state.storage = result; }
+ if ((options.method ?? 'GET') === 'GET') {
+ const document = await request(path + '/state', options);
+ storage(document.state?.storage);
+ const saved = document.state;
+ if (document.apiVersion !== 2 || !saved || !Number.isSafeInteger(saved.sequence) || saved.sequence < 0) throw invalid();
+ if (saved.apiVersion === 1) return { ...document, state: saved };
+ if (saved.apiVersion !== 2 || !/^[A-Za-z0-9-]{1,128}$/u.test(saved.read?.id ?? '')) throw invalid();
+ const root = pageReference(saved.root);
+ if (root.sequence !== saved.sequence) throw invalid();
+ const context = {
+ key, boardId: state.board, sequence: saved.sequence,
+ /** The server checks this path against the principal's pinned root on every request. */
+ async get(id, parents) {
+ const result = await request(path + '/objects/' + id + '?pin=' + encodeURIComponent(saved.read.id) + '&via=' + parents.join(','));
+ if (result.apiVersion !== 2 || result.page?.id !== id) throw invalid();
+ return bytes(result.page.sealed);
+ },
+ };
+ const page = await openPagedSnapshot(root, context, { maxEngine });
+ const native = await page.native();
+ // Local resealing adapts existing restore callers, not the wire protocol. The authenticated root, cursor and
+ // sender remain in the response; no locally generated bytes are reported as a server ciphertext ACK.
+ let sealed = await encode(native, saved.sequence);
+ const cache = createRelayPageCache(state);
+ try {
+ for (const pending of [state.snapshot, state.checkpoint]) if (pending?.sequence === saved.sequence && typeof pending.sealed === 'string') {
+ const prepared = cache.job(snapshotSourceId(pending.sealed));
+ if (prepared?.reference.id === root.id) { sealed = pending.sealed; break; }
+ }
+ } finally { cache.close(); }
+ return { ...document, state: { ...saved, sealed } };
+ }
+ if (options.method !== 'PUT' || !Number.isSafeInteger(options.body?.sequence) || typeof options.body.sealed !== 'string') throw invalid();
+ const pending = options.body;
+ const source = snapshotSourceId(pending.sealed);
+ const cache = createRelayPageCache(state);
+ try {
+ let prepared = cache.job(source);
+ if (!prepared) {
+ const native = await decode(pending);
+ const context = { key, boardId: state.board, sequence: pending.sequence, put: cache.put, remember: cache.remember };
+ context.lookup = fingerprint => cache.lookup(fingerprint, context);
+ const reference = await writePagedSnapshot(native, context);
+ cache.prepare(source, reference);
+ prepared = cache.job(source);
+ }
+ if (prepared.reference.sequence !== pending.sequence) throw invalid();
+ /** Resume only the same authenticated principal's live upload, otherwise start a new bounded session. */
+ async function upload() {
+ let result;
+ try {
+ result = await request(path + '/uploads', { method: 'POST', body: { sequence: pending.sequence, ...(prepared.upload ? { resume: prepared.upload } : {}) } });
+ } catch (error) {
+ if (error.code !== 'SNAPSHOT_UPLOAD_EXPIRED' || !prepared.upload) throw error;
+ result = await request(path + '/uploads', { method: 'POST', body: { sequence: pending.sequence } });
+ }
+ storage(result.storage);
+ if (result.apiVersion !== 2 || result.upload?.sequence !== pending.sequence || !/^[A-Za-z0-9-]{1,128}$/u.test(result.upload?.id ?? '')) throw invalid();
+ cache.upload(source, result.upload.id);
+ return result.upload.id;
+ }
+ let uploadId = await upload();
+ const repaired = new Set();
+ let renewed = false;
+ for (;;) {
+ try {
+ for (const object of cache.pending(source)) {
+ const result = await request(path + '/uploads/' + uploadId + '/objects/' + object.id, {
+ method: 'PUT', body: { sequence: object.sequence, sealed: Buffer.from(object.bytes).toString('base64url'), children: object.children },
+ });
+ if (result.apiVersion !== 2 || result.page?.id !== object.id) throw invalid();
+ cache.acknowledge(object.id);
+ }
+ const result = await request(path + '/uploads/' + uploadId + '/publish', {
+ method: 'POST', body: { root: prepared.reference, ...(options.initialBaseline ? { initialOnly: true } : {}) },
+ });
+ const accepted = pageReference(result.state?.root);
+ if (result.apiVersion !== 2 || result.state?.sequence !== pending.sequence || JSON.stringify(accepted) !== JSON.stringify(prepared.reference)) throw invalid();
+ storage(result.state.storage);
+ cache.complete(source);
+ return { ...result, state: { ...result.state, sealed: pending.sealed } };
+ } catch (error) {
+ if (error.code === 'SNAPSHOT_UPLOAD_EXPIRED' && !renewed) {
+ renewed = true; prepared.upload = uploadId; uploadId = await upload(); continue;
+ }
+ if (error.code !== 'SNAPSHOT_PAGE_MISSING' || !/^[0-9a-f]{64}$/u.test(error.page ?? '') || repaired.has(error.page)) throw error;
+ repaired.add(error.page);
+ cache.missing(error.page);
+ }
+ }
+ } finally { cache.close(); }
+}
+
+/** Negotiate bounded snapshots only when needed; small legacy heads keep their existing wire interface. */
+export async function relaySnapshotRequest(state, path, options = {}, io, hooks) {
+ const snapshot = path === '/api/v1/boards/' + state.board + '/state' && ['GET', 'PUT'].includes(options.method ?? 'GET');
+ /** Probe the configured relay once per command without downgrading a board already known to use pages. */
+ async function supportsPages() {
+ const owner = io && typeof io === 'object' ? io : state;
+ let origins = PAGE_CAPABILITIES.get(owner);
+ if (!origins) { origins = new Map(); PAGE_CAPABILITIES.set(owner, origins); }
+ if (!origins.has(state.url)) origins.set(state.url, (async () => {
+ try { return (await hooks.raw('/api/v2', {})).apiVersion === 2; }
+ catch (error) { if (error.httpStatus === 404 && error.relayVersion === 1) return false; throw error; }
+ })());
+ const supported = await origins.get(state.url);
+ if (!supported && state.snapshotApiVersion === 2) throw new Refused('RELAY_API_VERSION', 'This board requires relay API version 2. Upgrade the configured relay before syncing.');
+ return supported;
+ }
+ /** Keep the old internal retry carrier, while only bounded sealed objects cross the new HTTP interface. */
+ async function paged() {
+ state.snapshotApiVersion = 2;
+ const key = hooks.key();
+ try {
+ return await pagedSnapshotRequest(state, options, { key, maxEngine: hooks.maxEngine,
+ request: hooks.raw,
+ decode: record => hooks.decode(record, key),
+ encode: (value, sequence) => hooks.encode(value, sequence, key),
+ });
+ } catch (error) {
+ const storage = relayStorage(error.storage);
+ if (storage) state.storage = storage;
+ throw error;
+ } finally {
+ hooks.persist();
+ }
+ }
+ const oversized = options.method === 'PUT' && typeof options.body?.sealed === 'string' && options.body.sealed.length > Math.floor(hooks.legacyLimit * 4 / 3);
+ if (snapshot && (state.snapshotApiVersion === 2 || oversized) && await supportsPages()) return paged();
+ try { return await hooks.raw(path, options); }
+ catch (error) {
+ if (!snapshot || error.code !== 'RELAY_API_VERSION' || !await supportsPages()) throw error;
+ return paged();
+ }
+}
diff --git a/src/relay.js b/src/relay.js
index 3f0bd09b..6da555a6 100644
--- a/src/relay.js
+++ b/src/relay.js
@@ -101,10 +101,13 @@ import {
requireRelayOnChannel,
} from './person.js';
import { listApiProjects } from './projects.js';
+import { relaySnapshotRequest, relayStorage, relayStorageLine } from './relay-page-client.js';
+export { relayStorageLine };
export const DEFAULT_RELAY = 'https://app.pullboard.dev';
const SEALED_SNAPSHOT_LIMIT = 10_000_000;
const LINK_FILES = new WeakMap();
+const STATE_FILES = new WeakMap();
const WARNED_COMMANDS = new WeakSet();
const KEY_WARNED_COMMANDS = new WeakSet();
const BASELINE_ATTEMPTS = new WeakMap();
@@ -283,11 +286,13 @@ function loadLink(file) {
state.personReauth = true;
saveLink(file, state);
}
+ STATE_FILES.set(state, file);
return state;
}
/** Atomically persist acknowledgements and an exact sealed retry before attempting its send. */
export function saveLink(file, state) {
+ STATE_FILES.set(state, file);
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
const temporary = file + '.' + randomUUID() + '.tmp';
try {
@@ -347,10 +352,21 @@ function transportFailure(error) {
return typeof code === 'string' && /^[A-Z][A-Z0-9_]{1,40}$/u.test(code) ? code : 'no connection';
}
-/** The sole outbound transport in the CLI: explicit sign-in or a saved, opted-in board link. */
-export async function request(state, path, { method = 'GET', body, allowMissing = false, initialBaseline = false, signal } = {}, io) {
+/** Use the page adapter only for snapshot requests; the saved opt-in still owns every network call. */
+export async function request(state, path, options = {}, io) {
+ return relaySnapshotRequest(state, path, options, io, {
+ raw: (target, settings) => rawRequest(state, target, settings, io), key: () => readBoardKey(state.board),
+ decode: (record, key) => decoded(key, record, state, 'snapshot', record.sequence),
+ encode: (value, sequence, key) => sealedRecord(key, value, state, 'snapshot', sequence),
+ maxEngine: ENGINE_VERSION, legacyLimit: SEALED_SNAPSHOT_LIMIT,
+ persist: () => { const file = STATE_FILES.get(state); if (file) saveLink(file, state); },
+ });
+}
+
+/** The sole raw HTTP transport: explicit sign-in or a saved, opted-in board link, with bounded deadlines. */
+async function rawRequest(state, path, { method = 'GET', body, allowMissing = false, initialBaseline = false, signal } = {}, io) {
const supplied = io?.ignoreRelayEnvironmentToken ? undefined : process.env.PULLBOARD_RELAY_TOKEN;
- const readToken = method === 'GET' && path.startsWith('/api/v1/boards/') ? supplied : undefined;
+ const readToken = method === 'GET' && /^\/api\/v[12]\/boards\//u.test(path) ? supplied : undefined;
if (readToken !== undefined && !/^pa_[A-Za-z0-9_-]{43}$/.test(readToken)) throw new Refused('AUTH_REQUIRED', 'supply a current scoped board token without including it in command arguments');
const token = readToken ?? state.token;
let response;
@@ -394,6 +410,9 @@ export async function request(state, path, { method = 'GET', body, allowMissing
error.relayVersion = document.version;
error.relayCode = document.error?.code;
error.relayMessage = document.error?.message;
+ if (/^[0-9a-f]{64}$/u.test(document.error?.page ?? '')) error.page = document.error.page;
+ const storage = relayStorage(document.error?.storage);
+ if (storage) error.storage = storage;
throw error;
}
if (document.version !== 1) throw new Refused('RELAY_VERSION', 'the relay API version is unsupported; upgrade Pullboard before syncing');
@@ -456,6 +475,7 @@ function summary(root, state, { pairing = false } = {}) {
link: 'https://app.pullboard.dev/#board=' + state.board + fragment,
sequence: state.sequence, behind: (state.baselinePause ? behind(root, state) : state.mode === 'ordered' ? Number(Boolean(state.pending)) : behind(root, state)) + Number(Boolean(state.snapshot || state.checkpoint)),
recovery: recoverySummary(state),
+ ...(relayStorage(state.storage) ? { storage: relayStorage(state.storage) } : {}),
...(oversizedSnapshot ? { oversizedSnapshot } : {}),
...(state.baselinePause ? { pausedUpload: structuredClone(state.baselinePause) } : {}),
...(state.checkpointRefusal ? { refusedCheckpoint: checkpointProblem(state) } : {}) };
@@ -463,6 +483,7 @@ function summary(root, state, { pairing = false } = {}) {
/** Measure a pending sealed snapshot against the relay's decoded journal limit. */
function snapshotSize(state) {
+ if (state?.snapshotApiVersion === 2) return null;
const sealed = state?.snapshot?.sealed ?? state?.checkpoint?.sealed;
if (typeof sealed !== 'string') return null;
const bytes = Buffer.from(sealed, 'base64url');
diff --git a/src/sealed-pages.js b/src/sealed-pages.js
index 90e0911b..b3abddc5 100644
--- a/src/sealed-pages.js
+++ b/src/sealed-pages.js
@@ -136,16 +136,17 @@ async function readPage(reference, context, parents = []) {
if (plain.byteLength <= PREFIX.byteLength || PREFIX.some((byte, index) => plain[index] !== byte)) throw invalid();
const kind = plain[PREFIX.byteLength];
const body = plain.subarray(PREFIX.byteLength + 1);
+ const fingerprint = await pageDigest(plain);
if (kind === 1) {
if (body.byteLength !== reference.bytes) throw invalid();
- return { body };
+ return { body, fingerprint };
}
if (kind === 2) {
if (body.byteLength !== reference.bytes) throw invalid();
let descriptor;
try { descriptor = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(body)); } catch { throw invalid(); }
if (!descriptor || !Object.hasOwn(descriptor, 'value') || !Array.isArray(descriptor.links) || descriptor.links.length > PAGE_FANOUT) throw invalid();
- return { descriptor: { value: descriptor.value, links: descriptor.links.map(pageReference) } };
+ return { descriptor: { value: descriptor.value, links: descriptor.links.map(pageReference) }, fingerprint };
}
if (kind !== 0) throw invalid();
let children;
@@ -154,9 +155,12 @@ async function readPage(reference, context, parents = []) {
children = children.map(pageReference);
const length = children.reduce((sum, child) => sum + child.bytes, 0);
if (!Number.isSafeInteger(length) || length !== reference.bytes) throw invalid();
- return { children };
+ return { children, fingerprint };
}
+/** Authenticate a private cache hit before trusting its plaintext fingerprint; never upload the result. */
+export async function sealedPageFingerprint(reference, context) { return (await readPage(pageReference(reference), context)).fingerprint; }
+
/**
* Yield authenticated JSON bytes one bounded leaf at a time. A shared subtree may appear twice; only ancestor cycles refuse.
* maxBytes is a consumer's allocation budget for this value, independent of the size of the board's other paged values.
diff --git a/test/relay-page-client.test.js b/test/relay-page-client.test.js
new file mode 100644
index 00000000..5a5b1deb
--- /dev/null
+++ b/test/relay-page-client.test.js
@@ -0,0 +1,185 @@
+/** Durable device cache and bounded native protocol retries, using real crypto and journal storage [H15,H17]. */
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { randomBytes } from 'node:crypto';
+import { test } from 'node:test';
+import { createRelayJournal } from '../relay/journal.js';
+import { snapshotPageRoot } from '../relay/pages-http.js';
+import { createRelayPageCache, snapshotSourceId } from '../src/relay-page-cache.js';
+import { pagedSnapshotRequest, relaySnapshotRequest, relayStorage, relayStorageLine } from '../src/relay-page-client.js';
+import { generateBoardKey } from '../src/seal.js';
+import { writeSealedValue } from '../src/sealed-pages.js';
+import { Refused } from '../src/refused.js';
+
+/** Use only an isolated device home, restoring the surrounding test process after its cache closes. */
+function privateHome(t) {
+ const root = mkdtempSync(join(tmpdir(), 'pullboard-private-page-client-'));
+ const old = process.env.PULLBOARD_HOME;
+ process.env.PULLBOARD_HOME = join(root, 'device');
+ t.after(() => { if (old === undefined) delete process.env.PULLBOARD_HOME; else process.env.PULLBOARD_HOME = old; rmSync(root, { recursive: true, force: true }); });
+ return root;
+}
+
+/** Dispatch the client's actual API2 objects against real journal/page transactions; HTTP auth has its separate tests. */
+async function fixture(t) {
+ const root = privateHome(t);
+ const state = { board: 'a'.repeat(32), url: 'https://synthetic-relay.example' };
+ const key = await generateBoardKey();
+ const journal = createRelayJournal({ directory: join(root, 'journal'), boardId: state.board });
+ const pages = journal.pages();
+ t.after(() => { pages.close(); journal.close(); });
+ const principal = { kind: 'machine', userId: 'fixture-account', machine: 'fixture-machine' };
+ const storage = { account: { id: 'fixture-account', login: 'fixture-login' }, usedBytes: 0, limitBytes: 256_000_000 };
+ const calls = [];
+ let loseObjectAck = false, corruptRootAck = false;
+ /** Preserve a strict bounded upload shape while returning the real storage's idempotent acknowledgements. */
+ async function request(path, options = {}) {
+ const body = options.body;
+ calls.push({ path, method: options.method ?? 'GET', bytes: body ? Buffer.byteLength(JSON.stringify(body)) : 0,
+ digest: body?.sealed ? snapshotSourceId(body.sealed) : null });
+ const suffix = path.slice(('/api/v2/boards/' + state.board).length);
+ if (suffix === '/uploads') return { apiVersion: 2, storage, upload: body.resume ? pages.resume(body.resume, principal, body.sequence) : pages.begin(principal, body.sequence) };
+ const put = /^\/uploads\/([^/]+)\/objects\/([0-9a-f]{64})$/u.exec(suffix);
+ if (put) {
+ assert.deepEqual(Object.keys(body).sort(), ['children', 'sealed', 'sequence']);
+ assert.ok(calls.at(-1).bytes < 600_000);
+ const stored = pages.put(put[1], principal, { id: put[2], sequence: body.sequence, bytes: Buffer.from(body.sealed, 'base64url'), children: body.children });
+ if (loseObjectAck) { loseObjectAck = false; throw new Refused('RELAY_UNAVAILABLE', 'Synthetic lost object ACK.'); }
+ return { apiVersion: 2, page: stored };
+ }
+ const publish = /^\/uploads\/([^/]+)\/publish$/u.exec(suffix);
+ if (publish) {
+ const saved = pages.publish(publish[1], principal, body.root, (ref, bytes) => journal.saveSnapshot(ref.sequence, bytes, principal, { apiVersion: 2, pageBytes: ref.bytes }));
+ pages.prune([body.root.id]);
+ const result = { apiVersion: 2, state: { root: snapshotPageRoot(saved), sequence: saved.sequence, storage } };
+ if (corruptRootAck) result.state.root.id = 'f'.repeat(64);
+ return result;
+ }
+ if (suffix === '/state') {
+ const saved = journal.snapshot();
+ const reference = snapshotPageRoot(saved);
+ return { apiVersion: 2, state: { ...saved, apiVersion: 2, root: reference, read: pages.pin(principal, reference), storage } };
+ }
+ const get = /^\/objects\/([0-9a-f]{64})\?(.+)$/u.exec(suffix);
+ if (get) {
+ const query = new URLSearchParams(get[2]);
+ const object = pages.get(query.get('pin'), principal, get[1], query.get('via') ? query.get('via').split(',') : []);
+ return { apiVersion: 2, page: { id: object.id, sequence: object.sequence, sealed: object.bytes.toString('base64url'), children: object.children } };
+ }
+ throw new Error('Unexpected synthetic API2 route.');
+ }
+ const shouts = [{ shout_id: 1, shout_text: randomBytes(700_000).toString('base64'), shout_decision: 0, shout_answers: null }];
+ const document = { version: 1, tables: {
+ board_meta: [{ meta_key: 'board_id', meta_value: state.board }, { meta_key: 'relay_engine_version', meta_value: '56' }, { meta_key: 'relay_applied_sequence', meta_value: '0' }],
+ shout: shouts, event: [], item: [], agent: [], verdict: [], sqlite_sequence: [{ name: 'shout', seq: 19 }],
+ }, presentation: { version: 1, config: {}, state: { items: [], agents: [], shouts: [], decisions: [], asked: [], events: [], holds: [], spec: [], practice: [], lanes: [], owning: [] }, shouts } };
+ const adapter = { request, key, maxEngine: 56, decode: async () => structuredClone(document), encode: async value => JSON.stringify(value) };
+ return { state, document, adapter, calls, pages, journal, principal,
+ loseObjectAck() { loseObjectAck = true; }, corruptRootAck() { corruptRootAck = true; } };
+}
+
+test('a lost object ACK resumes the exact durable graph and native restore preserves all rows [H15,H17]', async t => {
+ const box = await fixture(t);
+ const pending = { sequence: 0, sealed: 'synthetic-original-pending-cipher' };
+ box.loseObjectAck();
+ await assert.rejects(pagedSnapshotRequest(box.state, { method: 'PUT', body: pending }, box.adapter), { code: 'RELAY_UNAVAILABLE' });
+ assert.equal(box.journal.snapshot(), null, 'a partially uploaded graph is never installed');
+ const first = box.calls.find(call => call.method === 'PUT');
+ const result = await pagedSnapshotRequest(box.state, { method: 'PUT', body: pending }, box.adapter);
+ assert.equal(result.state.sealed, pending.sealed, 'the existing retry carrier is acknowledged only after an exact root ACK');
+ const retries = box.calls.filter(call => call.path === first.path && call.method === 'PUT');
+ assert.equal(retries.length, 2); assert.equal(retries[0].digest, retries[1].digest);
+ assert.ok(box.calls.some(call => call.path.endsWith('/uploads') && call.method === 'POST'));
+ const read = await pagedSnapshotRequest(box.state, {}, box.adapter);
+ assert.deepEqual(JSON.parse(read.state.sealed), box.document);
+ assert.equal(box.state.storage.account.login, 'fixture-login');
+});
+
+test('a collected remote subtree is restaged despite old cached ACKs, without replacing the source graph [H15,H17]', async t => {
+ const box = await fixture(t);
+ const pending = { sequence: 0, sealed: 'synthetic-stable-pending-cipher' };
+ const first = await pagedSnapshotRequest(box.state, { method: 'PUT', body: pending }, box.adapter);
+ const count = box.calls.filter(call => call.method === 'PUT').length;
+ box.pages.prune([]);
+ const second = await pagedSnapshotRequest(box.state, { method: 'PUT', body: pending }, box.adapter);
+ assert.equal(second.state.root.id, first.state.root.id);
+ assert.ok(box.calls.filter(call => call.method === 'PUT').length > count);
+ assert.deepEqual(JSON.parse((await pagedSnapshotRequest(box.state, {}, box.adapter)).state.sealed), box.document);
+});
+
+test('an incorrect root ACK never completes or discards the private durable upload [H15,H17]', async t => {
+ const box = await fixture(t);
+ const pending = { sequence: 0, sealed: 'synthetic-ack-check' };
+ box.corruptRootAck();
+ await assert.rejects(pagedSnapshotRequest(box.state, { method: 'PUT', body: pending }, box.adapter), { code: 'RELAY_RESPONSE' });
+ const cache = createRelayPageCache(box.state);
+ try { assert.ok(cache.job(snapshotSourceId(pending.sealed))); } finally { cache.close(); }
+});
+
+test('private reuse authenticates the fingerprint binding rather than trusting an unrelated cached page [H15,H17]', async t => {
+ privateHome(t);
+ const state = { board: 'b'.repeat(32), url: 'https://synthetic-relay.example' };
+ const cache = createRelayPageCache(state);
+ const context = { key: await generateBoardKey(), boardId: state.board, sequence: 0, put: cache.put };
+ try {
+ const ref = await writeSealedValue({ marker: 'private-data' }, context);
+ cache.remember('f'.repeat(64), ref);
+ await assert.rejects(cache.lookup('f'.repeat(64), context), { code: 'RELAY_PAGE_CACHE' });
+ } finally { cache.close(); }
+});
+
+test('account status has a validated identity, exact byte counts and a clear operator-set limit [H15,H17]', () => {
+ const storage = { account: { id: 'account-1', login: 'owner-login' }, usedBytes: 251_000_000, limitBytes: 256_000_000 };
+ assert.deepEqual(relayStorage(storage), storage);
+ assert.equal(relayStorageLine(storage), 'relay storage: account owner-login, 251.0 MB used of 256.0 MB');
+ assert.equal(relayStorage({ ...storage, account: { ...storage.account, login: 'Bearer private text' } }), null);
+ assert.equal(relayStorage({ ...storage, usedBytes: -1 }), null);
+});
+
+test('oversize native uploads negotiate API2, native reads upgrade explicitly, and a paged board never downgrades [H15,H17]', async t => {
+ const box = await fixture(t);
+ const pending = { sequence: 0, sealed: 'synthetic-large-legacy-carrier' };
+ const oldPath = '/api/v1/boards/' + box.state.board + '/state';
+ const calls = [];
+ let advertised = 2, persisted = 0;
+ const hooks = { ...box.adapter, key: () => box.adapter.key, legacyLimit: 1, persist() { persisted++; },
+ /** The adapter must never send an oversized legacy body before negotiating bounded pages. */
+ async raw(path, options) {
+ calls.push(path);
+ if (path === '/api/v2') return { version: 1, apiVersion: advertised };
+ if (path === oldPath) { assert.notEqual(options.method, 'PUT'); throw new Refused('RELAY_API_VERSION', 'Use API2.'); }
+ return box.adapter.request(path, options);
+ },
+ };
+ const written = await relaySnapshotRequest(box.state, oldPath, { method: 'PUT', body: pending }, {}, hooks);
+ assert.equal(written.state.sealed, pending.sealed);
+ assert.equal(box.state.snapshotApiVersion, 2);
+ assert.ok(!calls.includes(oldPath), 'the oversized wire body never reaches API1');
+ delete box.state.snapshotApiVersion;
+ const read = await relaySnapshotRequest(box.state, oldPath, {}, {}, hooks);
+ assert.deepEqual(JSON.parse(read.state.sealed), box.document);
+ assert.equal(box.state.snapshotApiVersion, 2);
+ assert.equal(persisted, 2);
+ advertised = 1;
+ await assert.rejects(relaySnapshotRequest(box.state, oldPath, {}, {}, hooks), { code: 'RELAY_API_VERSION' });
+});
+
+test('a relay without API2 retains legacy snapshot behavior rather than guessing a new wire format [H15,H17]', async () => {
+ const state = { board: 'a'.repeat(32), url: 'https://synthetic-legacy.example' };
+ const path = '/api/v1/boards/' + state.board + '/state';
+ const calls = [];
+ const response = { version: 1, state: { sealed: 'legacy-ack' } };
+ const hooks = { legacyLimit: 1,
+ /** An old relay has no capability route and only accepts its original snapshot interface. */
+ async raw(target) {
+ calls.push(target);
+ if (target === '/api/v2') throw Object.assign(new Refused('BAD_REQUEST', 'Legacy relay route.'), { httpStatus: 404, relayVersion: 1 });
+ assert.equal(target, path); return response;
+ },
+ };
+ assert.equal(await relaySnapshotRequest(state, path, { method: 'PUT', body: { sealed: 'long' } }, {}, hooks), response);
+ assert.deepEqual(calls, ['/api/v2', path]);
+ assert.equal(state.snapshotApiVersion, undefined);
+});