Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(relay): publish page graphs with their journal head [H15,H18]
Corey Olsonb16c190c927c
diff --git a/relay/journal.js b/relay/journal.jsindex ee3e29b1..b2a966db 100644--- a/relay/journal.js+++ b/relay/journal.js@@ -3,6 +3,7 @@ import { DatabaseSync } from 'node:sqlite';import { closeSync, lstatSync, mkdirSync, openSync } from 'node:fs';import { join, resolve } from 'node:path';import { Refused } from '../src/refused.js';+import { createRelayPages } from './page-store.js';const FORMAT = 4;const DEFAULT_BYTES = 100_000;@@ -32,8 +33,9 @@ export function createRelayJournal({ directory, boardId, maxBytes = DEFAULT_BYTEif (!stat.isFile() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) throw new Refused('RELAY_STORAGE', 'the journal must be a private regular database file');const db = new DatabaseSync(file);let closed = false;+ let writing = false;try {- db.exec('PRAGMA busy_timeout = 30000; PRAGMA journal_mode = WAL; PRAGMA secure_delete = ON; BEGIN IMMEDIATE');+ db.exec('PRAGMA busy_timeout = 30000; PRAGMA journal_mode = WAL; PRAGMA foreign_keys=ON; PRAGMA secure_delete = ON; BEGIN IMMEDIATE');db.exec(`CREATE TABLE IF NOT EXISTS journal_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);CREATE TABLE IF NOT EXISTS journal_record (sequence INTEGER PRIMARY KEY, received_at TEXT NOT NULL,kind TEXT NOT NULL CHECK(kind IN ('move','request')), payload BLOB NOT NULL,@@ -42,6 +44,10 @@ export function createRelayJournal({ directory, boardId, maxBytes = DEFAULT_BYTE(sender_kind IN ('agent','machine') AND sender_agent IS NOT NULL)))`);const saved = db.prepare('SELECT key, value FROM journal_meta ORDER BY key').all();if (saved.length === 0) {+ if (db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='page_meta'").get()) {+ const owner = db.prepare("SELECT value FROM page_meta WHERE key='board'").get();+ if (owner && owner.value !== id) throw new Refused('RELAY_STORAGE', 'Restore sealed pages with their original journal board identity.');+ }if (db.prepare('SELECT 1 FROM journal_record LIMIT 1').get()) throw new Refused('RELAY_STORAGE', 'restore the journal with its original identity and format metadata');db.prepare('INSERT INTO journal_meta (key,value) VALUES (?,?), (?,?)').run('board', id, 'format', String(FORMAT));} else if (saved.length !== 2 || saved[0].key !== 'board' || saved[0].value !== id || saved[1].key !== 'format' || !['2', '3', String(FORMAT)].includes(saved[1].value)) {@@ -99,6 +105,25 @@ export function createRelayJournal({ directory, boardId, maxBytes = DEFAULT_BYTE/** Keep calls after shutdown from exposing a native SQLite error. */function open() { if (closed) throw new Refused('RELAY_CLOSED', 'open the journal before reading or appending records'); }+ /** Share one SQLite transaction between a sealed-object publication and its journal coverage cursor. */+ function transaction(work) {+ open();+ if (writing) return work();+ db.exec('BEGIN IMMEDIATE'); writing = true;+ try {+ const result = work();+ if (result && typeof result.then === 'function') throw new Refused('RELAY_CONFIG', 'Finish the journal transaction before awaiting.');+ db.exec('COMMIT'); return result;+ } catch (error) { db.exec('ROLLBACK'); throw error; }+ finally { writing = false; }+ }++ /** Store page objects beside the head so existing consistent backups and unlink cleanup cover both. */+ function pages(options = {}) {+ open();+ return createRelayPages({ directory: root, boardId: id, now: options.now ?? Date.now, database: db, transaction });+ }+/** Convert an SQLite row to detached opaque bytes so callers cannot mutate a stored record. */function record(row) {return { sequence: row.sequence, receivedAt: row.received_at, kind: row.kind ?? 'snapshot',@@ -227,8 +252,7 @@ export function createRelayJournal({ directory, boardId, maxBytes = DEFAULT_BYTEif (![1, 2].includes(apiVersion) || apiVersion === 2 && (!Number.isSafeInteger(pageBytes) || pageBytes < 1)) throw new Refused('SNAPSHOT_VERSION', 'Publish a supported snapshot transport version and its sealed root length.');const sealed = payload(bytes);const who = sender(principal, true);- db.exec('BEGIN IMMEDIATE');- try {+ return transaction(() => {if (sequence > latest()) throw new Refused('SEQUENCE_GAP', 'upload the missing moves before a snapshot that covers them');const previous = snapshot();if (initialOnly && previous) throw new Refused('BASELINE_EXISTS', 'this board already has a baseline; preserve the local board before choosing which history to keep');@@ -238,9 +262,8 @@ export function createRelayJournal({ directory, boardId, maxBytes = DEFAULT_BYTEdb.prepare('INSERT INTO journal_snapshot (slot,sequence,received_at,payload,sender_kind,sender_user,sender_agent,api_version,page_bytes) VALUES (1,?,?,?,?,?,?,?,?) ON CONFLICT(slot) DO UPDATE SET sequence=excluded.sequence,received_at=excluded.received_at,payload=excluded.payload,sender_kind=excluded.sender_kind,sender_user=excluded.sender_user,sender_agent=excluded.sender_agent,api_version=excluded.api_version,page_bytes=excluded.page_bytes').run(sequence, at.toISOString(), sealed, who.kind, who.userId, who.agent, apiVersion, apiVersion === 2 ? pageBytes : null);db.prepare('DELETE FROM journal_record WHERE sequence <= ?').run(sequence);const result = snapshot();- db.exec('COMMIT');return result;- } catch (error) { db.exec('ROLLBACK'); throw error; }+ });}/** Measure the exact retained-byte change of a replacement and its compacted acknowledged prefix. */@@ -267,5 +290,5 @@ export function createRelayJournal({ directory, boardId, maxBytes = DEFAULT_BYTEclosed = true;}- return { board: id, latest, activity, append, appendNext, recordAt, registrationFor, after, snapshot, saveSnapshot, snapshotDelta, close };+ return { board: id, latest, activity, append, appendNext, recordAt, registrationFor, after, snapshot, saveSnapshot, snapshotDelta, pages, close };}diff --git a/relay/page-store.js b/relay/page-store.jsindex 0ec12882..b2b87f1a 100644--- a/relay/page-store.js+++ b/relay/page-store.js@@ -24,21 +24,22 @@ function principalKey(principal) {}/** Manage one board's objects; callers authorize and hold the shared auth lifecycle lock around every operation. */-export function createRelayPages({ directory, boardId, now = Date.now }) {+export function createRelayPages({ directory, boardId, now = Date.now, database, transaction }) {if (typeof directory !== 'string' || !/^[0-9a-f]{32}$/u.test(boardId ?? '') || typeof now !== 'function') throw new Refused('RELAY_CONFIG', 'Configure private page storage for a persistent board id.');const root = resolve(directory);mkdirSync(root, { recursive: true, mode: 0o700 });const parent = lstatSync(root);if (!parent.isDirectory() || parent.isSymbolicLink() || parent.mode & 0o077) throw new Refused('RELAY_STORAGE', 'Keep relay page storage in a private directory.');- const file = join(root, boardId + '.pages.sqlite');+ const file = join(root, boardId + '.journal.sqlite');try { closeSync(openSync(file, 'wx', 0o600)); } catch (error) { if (error.code !== 'EEXIST') throw error; }const entry = lstatSync(file);if (!entry.isFile() || entry.isSymbolicLink() || entry.mode & 0o077) throw new Refused('RELAY_STORAGE', 'Keep relay page databases as private regular files.');- const db = new DatabaseSync(file);+ const ownsDatabase = database === undefined;+ const db = database ?? new DatabaseSync(file);let closed = false;- try {- db.exec(`PRAGMA busy_timeout=30000; PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE;- CREATE TABLE IF NOT EXISTS page_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);+ /** Initialize only opaque page tables; a borrowed journal connection owns the enclosing transaction. */+ function initialize() {+ db.exec(`CREATE TABLE IF NOT EXISTS page_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);CREATE TABLE IF NOT EXISTS page_object (id TEXT PRIMARY KEY, sequence INTEGER NOT NULL,payload BLOB NOT NULL, children TEXT NOT NULL, created INTEGER NOT NULL);CREATE TABLE IF NOT EXISTS page_upload (id TEXT PRIMARY KEY, principal TEXT NOT NULL,@@ -52,10 +53,21 @@ export function createRelayPages({ directory, boardId, now = Date.now }) {else if (saved.length !== 2 || saved[0].key !== 'board' || saved[0].value !== boardId || saved[1].key !== 'format' || saved[1].value !== '1') {throw new Refused('RELAY_STORAGE', 'Restore this page database with its original board identity and supported format.');}- db.exec('COMMIT');+ }+ try {+ if (ownsDatabase) {+ db.exec('PRAGMA busy_timeout=30000; PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA secure_delete=ON; BEGIN IMMEDIATE');+ initialize(); db.exec('COMMIT');+ } else {+ if (typeof transaction !== 'function') throw new Refused('RELAY_CONFIG', 'Borrow a journal connection with its shared transaction boundary.');+ transaction(initialize);+ }} catch (error) {- try { db.exec('ROLLBACK'); } catch { /* Initialization may have failed before its transaction. */ }- db.close(); throw error;+ if (ownsDatabase) {+ try { db.exec('ROLLBACK'); } catch { /* Initialization may have failed before its transaction. */ }+ db.close();+ }+ throw error;}/** Refuse invalid clocks and use after shutdown without leaking native database messages. */@@ -68,7 +80,9 @@ export function createRelayPages({ directory, boardId, now = Date.now }) {/** Keep staging and collection changes transactional; publication itself is owned by the journal. */function atomic(work) {- clock(); db.exec('BEGIN IMMEDIATE');+ clock();+ if (transaction) return transaction(work);+ db.exec('BEGIN IMMEDIATE');try {const result = work();if (result && typeof result.then === 'function') throw new Refused('RELAY_CONFIG', 'Finish synchronous page storage before awaiting.');@@ -239,6 +253,6 @@ export function createRelayPages({ directory, boardId, now = Date.now }) {}/** Close exactly this store; repeated shutdown is harmless. */- function close() { if (!closed) { db.close(); closed = true; } }+ function close() { if (!closed) { if (ownsDatabase) db.close(); closed = true; } }return { begin, resume, put, publish, pin, get, prune, usage, close, file };}diff --git a/relay/pages-http.js b/relay/pages-http.jsindex baccead8..ac84bff1 100644--- a/relay/pages-http.js+++ b/relay/pages-http.js@@ -3,7 +3,6 @@ import { createHash } from 'node:crypto';import { apiJson, apiRefusal, apiStatus, readApiBody } from '../src/api-http.js';import { Refused } from '../src/refused.js';import { PAGE_BYTES, PAGE_CIPHER_BYTES, PAGE_FANOUT, pageReference } from '../src/sealed-pages.js';-import { createRelayPages } from './page-store.js';const PAGE_BODY_BYTES = 600_000;@@ -34,7 +33,7 @@ export function createRelayPagesHandler({ directory, now, authorize, withJournal/** Open objects only while the caller's shared board lifecycle lock and fresh authorization hold. */function withPages(id, who, work, create = false) {return withJournal(id, journal => {- const pages = createRelayPages({ directory, boardId: id, now });+ const pages = journal.pages({ now });try { return work(journal, pages); } finally { pages.close(); }}, create, who.engineVersion);}diff --git a/relay/storage-quota.js b/relay/storage-quota.jsindex 233c5634..f02fa671 100644--- a/relay/storage-quota.js+++ b/relay/storage-quota.js@@ -7,20 +7,27 @@ import { Refused } from '../src/refused.js';export const ACCOUNT_STORAGE_BYTES = 256_000_000;/** Read only byte counts from a private relay database; never decode or return stored payloads. */-function storedBytes(file, query) {+function storedBytes(file) {let stat;try { stat = lstatSync(file); } catch (error) { if (error.code === 'ENOENT') return 0; throw error; }if (!stat.isFile() || stat.isSymbolicLink() || stat.mode & 0o077) throw new Refused('RELAY_STORAGE', 'Keep account storage in private regular relay databases.');const db = new DatabaseSync(file, { readOnly: true });- try { return db.prepare(query).get().bytes; } finally { db.close(); }+ try {+ const tables = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map(row => row.name));+ let bytes = 0;+ for (const name of ['journal_record', 'journal_snapshot']) {+ if (tables.has(name)) bytes += db.prepare(`SELECT COALESCE(SUM(length(payload)),0) AS bytes FROM ${name}`).get().bytes;+ }+ if (tables.has('page_object')) bytes += db.prepare('SELECT COALESCE(SUM(length(payload)+length(children)),0) AS bytes FROM page_object').get().bytes;+ return bytes;+ } finally { db.close(); }}/** Count committed legacy bytes and every staged or committed opaque page for one linked board. */export function boardStorageBytes(directory, id) {if (!/^[0-9a-f]{32}$/u.test(id ?? '')) throw new Refused('BAD_BOARD', 'Use a persistent board identity for storage accounting.');const root = resolve(directory);- return storedBytes(join(root, id + '.journal.sqlite'), 'SELECT COALESCE((SELECT SUM(length(payload)) FROM journal_record),0)+COALESCE((SELECT SUM(length(payload)) FROM journal_snapshot),0) AS bytes')- + storedBytes(join(root, id + '.pages.sqlite'), 'SELECT COALESCE(SUM(length(payload)+length(children)),0) AS bytes FROM page_object');+ return storedBytes(join(root, id + '.journal.sqlite'));}/** Format the provisional operator-set account budget without printing a credential or repository path. */diff --git a/test/relay-page-store.test.js b/test/relay-page-store.test.jsindex eb53e249..bada977a 100644--- a/test/relay-page-store.test.js+++ b/test/relay-page-store.test.js@@ -3,9 +3,10 @@ import assert from 'node:assert/strict';import { createHash } from 'node:crypto';import { mkdtempSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';-import { join } from 'node:path';+import { dirname, join } from 'node:path';import { test } from 'node:test';import { createRelayPages } from '../relay/page-store.js';+import { createRelayJournal } from '../relay/journal.js';import { Refused } from '../src/refused.js';import { generateBoardKey } from '../src/seal.js';import { PAGE_BYTES, readSealedValue, writeSealedValue } from '../src/sealed-pages.js';@@ -26,6 +27,8 @@ async function fixture(t) {key, boardId, store: stores[0],/** Advance the product lease clock without an elapsed-time performance assertion. */advance(ms) { time += ms; },+ /** Share the fixture clock with a journal borrowing this store's connection. */+ now() { return time; },/** Reopen actual on-disk state after a simulated process restart. */reopen() { box.store.close(); box.store = createRelayPages(options); stores.push(box.store); },/** Upload actual encrypted objects without publishing a head prematurely. */@@ -141,3 +144,22 @@ test('published objects and an existing reader pin survive closing the databasebox.reopen();assert.equal((await box.read(staged.reference, pin)).marker, 'durable encrypted graph');});++test('one database transaction rolls back a head and upload cleanup together [H15,H17,H18]', async t => {+ const box = await fixture(t);+ const staged = await box.stage({ marker: 'atomic publication fixture' }, 0);+ const journal = createRelayJournal({ directory: dirname(box.store.file), boardId: box.boardId, now: () => new Date(box.now()) });+ const pages = journal.pages({ now: box.now });+ try {+ // The upload was staged through another connection; the journal and object tables are in the same file.+ assert.throws(() => pages.publish(staged.upload.id, machine, staged.reference, (reference, bytes) => {+ journal.saveSnapshot(reference.sequence, bytes, machine, { apiVersion: 2, pageBytes: reference.bytes });+ throw new Error('Synthetic failure after the head write.');+ }), /Synthetic failure after the head write/u);+ assert.equal(journal.snapshot(), null, 'no partial head escaped the failed transaction');+ const result = pages.publish(staged.upload.id, machine, staged.reference, (reference, bytes) =>+ journal.saveSnapshot(reference.sequence, bytes, machine, { apiVersion: 2, pageBytes: reference.bytes }));+ assert.equal(result.apiVersion, 2);+ assert.equal(journal.snapshot().sequence, 0);+ } finally { pages.close(); journal.close(); }+});diff --git a/test/relay-pages-http.test.js b/test/relay-pages-http.test.jsindex a0d88fb5..071a5f83 100644--- a/test/relay-pages-http.test.js+++ b/test/relay-pages-http.test.js@@ -6,6 +6,7 @@ import { mkdtempSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { randomBytes } from 'node:crypto';+import { DatabaseSync } from 'node:sqlite';import { test } from 'node:test';import { createRelayAuth } from '../relay/auth.js';import { createGitHubClient } from '../relay/github.js';@@ -79,7 +80,7 @@ async function fixture(t, accountStorageBytes) {const directory = await readSealedDescriptor(state.root, context);return readSealedValue(directory.value.value, { ...context, parents: [state.root.id] });}- return { root, auth, signed, origin, key, call, stage, publish, read };+ return { root, auth, signed, origin, key, call, stage, publish, read, handler };}test('API2 reads legacy heads and publishes opaque pages while old clients explicitly upgrade [H15,H17]', async t => {@@ -150,3 +151,19 @@ test('all page reads reauthorize the board and a readable agent cannot replace aconst after = await box.call('GET', '/api/v2/boards/' + BOARD + '/objects/' + state.document.state.root.id + '?pin=' + state.document.state.read.id, undefined, agent.token);assert.equal(after.status, 401, 'a previously issued read pin never bypasses revocation');});++test('a consistent journal backup carries every opaque page and unlink removes the complete copy [H15,H18]', async t => {+ const box = await fixture(t);+ const staged = await box.stage({ marker: 'BACKUP_HAS_SEALED_HISTORY', text: 'b'.repeat(600_000) });+ assert.equal((await box.publish(staged)).status, 200);+ const backups = box.handler.backup();+ const name = backups.written.find(value => value.startsWith(BOARD + '.'));+ assert.ok(name, 'the ordinary retention backup includes this board');+ const source = new DatabaseSync(join(box.root, 'boards-backups', name), { readOnly: true });+ try {+ assert.ok(source.prepare('SELECT COUNT(*) AS count FROM page_object').get().count > 1);+ assert.equal(source.prepare('SELECT api_version FROM journal_snapshot WHERE slot=1').get().api_version, 2);+ } finally { source.close(); }+ assert.equal((await box.call('DELETE', '/api/v1/boards/' + BOARD)).status, 200);+ assert.equal((await box.call('GET', '/api/v2/boards/' + BOARD + '/state')).status, 404);+});