Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Segmented page storage engine SQL (engine-sql/page-body.ts) on PGLite: every
* Wave B engine member, the manifest generation reads, the flip (happy path,
* revision_changed, retire bound, two preparers), the body CHECK and the page
* triggers on a body_generation-only flip. GC: test/large-pages/segment-gc.test.ts.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { OperationError } from '../src/core/ops/contract.ts';
import { STAGE_BATCH_BYTES } from '../src/core/page-body/limits.ts';
import { chunk, flipInput, hexOf, pageMeta, publishSegmentedPage, segmentText, stageGeneration, stagingInput } from './helpers/page-body-fixtures.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
await engine.executeRaw("INSERT INTO sources (id, name) VALUES ('other', 'other') ON CONFLICT DO NOTHING");
}, 60_000);
afterAll(async () => {
await engine.disconnect();
});
async function pageRow(pageId: number) {
const rows = await engine.executeRaw<Record<string, unknown>>(`SELECT body_storage, compiled_truth, timeline, body_generation::int AS body_generation,
body_bytes::int AS body_bytes, segment_count, truth_lead, timeline_lead, body_flags, content_hash, generation::int AS generation,
knowledge_revision::text AS knowledge_revision, text_projection_revision::text AS text_projection_revision, search_vector::text AS search_vector, title
FROM pages WHERE id = $1`, [pageId]);
return rows[0]!;
}
async function chunkRows(pageId: number) {
return engine.executeRaw<{ chunk_index: number; occurrence_id: number | null; body_generation: number | null; chunk_text: string }>(
`SELECT chunk_index, occurrence_id::int AS occurrence_id, body_generation::int AS body_generation, chunk_text
FROM content_chunks WHERE page_id = $1 ORDER BY chunk_index`, [pageId]);
}
describe('allocateBodyOccurrenceIds', () => {
test('returns distinct increasing ids and nothing for a zero count', async () => {
const ids = await engine.allocateBodyOccurrenceIds(3);
expect(ids.length).toBe(3);
expect(ids[1]).toBeGreaterThan(ids[0]!);
expect(ids[2]).toBeGreaterThan(ids[1]!);
const more = await engine.allocateBodyOccurrenceIds(1);
expect(more[0]).toBeGreaterThan(ids[2]!);
expect(await engine.allocateBodyOccurrenceIds(0)).toEqual([]);
});
});
describe('segment texts', () => {
test('dedupe per source and round-trip bytes exactly', async () => {
const a = segmentText('alpha segment ☃ with a NUL \u0000 byte\n');
const b = segmentText('beta segment\nsecond line\n');
const c = segmentText('gamma segment\n');
expect(await engine.stageSegmentTexts('default', [a, b])).toEqual({ inserted: 2 });
expect(await engine.stageSegmentTexts('default', [a, c])).toEqual({ inserted: 1 });
expect(await engine.stageSegmentTexts('other', [a])).toEqual({ inserted: 1 });
const read = await engine.readSegmentTexts('default', [a.sha256, b.sha256, segmentText('never staged').sha256]);
expect([...read.keys()].sort()).toEqual([hexOf(a.sha256), hexOf(b.sha256)].sort());
expect(Buffer.from(read.get(hexOf(a.sha256))!).equals(Buffer.from(a.body))).toBe(true);
expect(read.get(hexOf(b.sha256))).toBeInstanceOf(Uint8Array);
expect(await engine.readSegmentTexts('default', [])).toEqual(new Map());
});
test('a stage larger than one statement batch inserts every text', async () => {
const segmentBytes = 300 * 1024;
const count = Math.ceil((STAGE_BATCH_BYTES.pglite * 2) / segmentBytes);
const texts = Array.from({ length: count }, (_, i) => segmentText(`${i}:`.padEnd(segmentBytes, 'x')));
expect(await engine.stageSegmentTexts('default', texts)).toEqual({ inserted: count });
const read = await engine.readSegmentTexts('default', texts.map(t => t.sha256));
expect(read.size).toBe(count);
});
});
describe('manifest and flip', () => {
test('a published page reads generation 1 in seq order; staged rows stay invisible until the flip', async () => {
const t = [segmentText('# Intro\none\n'), segmentText('## Two\ntwo\n'), segmentText('## Three\nthree\n')];
const page = await publishSegmentedPage(engine, 'notes/manifest-reads', t);
const live = await engine.readManifest(page.pageId);
expect(live.map(r => r.seq)).toEqual([0, 1, 2]);
expect(live.map(r => r.genFrom)).toEqual([1, 1, 1]);
expect(live.every(r => r.genTo === null && r.stagingId === null)).toBe(true);
expect(hexOf(live[1]!.sha256)).toBe(hexOf(t[1]!.sha256));
expect(live[0]!.entryState).toEqual({ fence: null, depth: 0, heading: ['Intro'] });
expect(live[0]!.outline).toEqual([{ level: 2, text: 'Intro', byte: 0 }]);
expect(live[0]!.dateMin).toBe('2026-01-02');
const replacement = segmentText('## Three, edited\nthree!\n');
const staged = await stageGeneration(engine, { pageId: page.pageId, slug: 'notes/manifest-reads', genFrom: 2, texts: [replacement] });
await engine.executeRaw('UPDATE page_body_manifest SET seq = 2 WHERE staging_id = $1::uuid', [staged.stagingId]);
await engine.executeRaw('UPDATE page_body_manifest SET byte_start = $2 WHERE staging_id = $1::uuid', [staged.stagingId, t[0]!.body.byteLength + t[1]!.body.byteLength]);
expect((await engine.readManifest(page.pageId)).length).toBe(3);
const stagedRows = await engine.readManifest(page.pageId, { stagingId: staged.stagingId });
expect(stagedRows.map(r => r.occurrenceId)).toEqual(staged.occurrences);
const before = await pageRow(page.pageId);
const result = await engine.flipBodyGeneration(flipInput(page.pageId, 1, staged.stagingId, [t[0]!, t[1]!, replacement],
{ closeOccurrences: [page.occurrences[2]!], retireChunkOccurrences: [page.occurrences[2]!] }));
expect(result).toEqual({ retiredChunks: 0, retireDeferred: false });
const gen2 = await engine.readManifest(page.pageId);
expect(gen2.map(r => r.occurrenceId)).toEqual([page.occurrences[0], page.occurrences[1], staged.occurrences[0]]);
const gen1 = await engine.readManifest(page.pageId, { generation: 1 });
expect(gen1.map(r => r.occurrenceId)).toEqual(page.occurrences);
expect(gen1[2]!.genTo).toBe(2);
const tail = await engine.readManifest(page.pageId, { region: 'truth', lastN: 2 });
expect(tail.map(r => r.seq)).toEqual([1, 2]);
expect(await engine.readManifest(page.pageId, { region: 'timeline' })).toEqual([]);
const after = await pageRow(page.pageId);
expect(after.body_generation).toBe(2);
expect(after.compiled_truth).toBeNull();
expect(after.timeline).toBeNull();
expect(after.body_storage).toBe('segmented');
expect(after.segment_count).toBe(3);
expect(after.body_flags).toEqual({ has_fences: false, regions: ['truth'] });
expect(after.generation).toBeGreaterThan(before.generation as number);
expect(after.knowledge_revision).not.toBe(before.knowledge_revision);
expect(String(after.search_vector)).toContain('zephyrwind');
});
test('a flip against a moved generation refuses revision_changed with a read fix and changes nothing', async () => {
const t = [segmentText('# Moved\nbody\n')];
const page = await publishSegmentedPage(engine, 'notes/flip-moved', t);
const staged = await stageGeneration(engine, { pageId: page.pageId, slug: 'notes/flip-moved', genFrom: 2, texts: [segmentText('# Moved\nnew\n')] });
const before = await pageRow(page.pageId);
const err = await engine.flipBodyGeneration(flipInput(page.pageId, 0, staged.stagingId, t)).then(() => null, (e: unknown) => e);
expect(err).toBeInstanceOf(OperationError);
const op = err as OperationError;
expect(op.canonical ?? op.code).toBe('revision_changed');
expect(op.fix?.mcp).toEqual({ tool: 'get_page', arguments: { slug: 'notes/flip-moved', source_id: 'default', body: 'none' } });
expect(JSON.parse(op.detail!)).toEqual({ expected_generation: 0, current_generation: 1 });
expect(await pageRow(page.pageId)).toEqual(before);
expect((await engine.readManifest(page.pageId, { stagingId: staged.stagingId })).length).toBe(1);
const gone = await engine.flipBodyGeneration(flipInput(2_000_000_000, 0, staged.stagingId, t)).then(() => null, (e: unknown) => e);
expect((gone as OperationError).code).toBe('page_identity_changed');
expect(() => engine.flipBodyGeneration(flipInput(page.pageId, 1, staged.stagingId, t, { newGeneration: 3 }))).toThrow(TypeError);
});
test('retirement above retireMaxRows deletes none and reports retireDeferred; within the bound it deletes them', async () => {
const t = [segmentText('# A\na\n'), segmentText('# B\nb\n'), segmentText('# C\nc\n')];
const page = await publishSegmentedPage(engine, 'notes/flip-retire-bound', t, { chunkTexts: true });
const replacement = [segmentText('# A2\na\n'), segmentText('# B2\nb\n'), segmentText('# C2\nc\n')];
const staged = await stageGeneration(engine, { pageId: page.pageId, slug: 'notes/flip-retire-bound', genFrom: 2, texts: replacement, chunkTexts: true });
const deferred = await engine.transaction(tx => tx.flipBodyGeneration(flipInput(page.pageId, 1, staged.stagingId, replacement,
{ closeOccurrences: page.occurrences, retireChunkOccurrences: page.occurrences, retireMaxRows: 2 })));
expect(deferred).toEqual({ retiredChunks: 0, retireDeferred: true });
const rows = await chunkRows(page.pageId);
expect(rows.filter(r => r.body_generation === 1).map(r => r.occurrence_id).sort()).toEqual([...page.occurrences].sort());
expect((await pageRow(page.pageId)).body_generation).toBe(2);
const next = [segmentText('# A3\na\n')];
const staged3 = await stageGeneration(engine, { pageId: page.pageId, slug: 'notes/flip-retire-bound', genFrom: 3, texts: next, chunkTexts: true });
const retired = await engine.flipBodyGeneration(flipInput(page.pageId, 2, staged3.stagingId, next,
{ closeOccurrences: staged.occurrences, retireChunkOccurrences: [...page.occurrences, ...staged.occurrences], retireMaxRows: 6 }));
expect(retired).toEqual({ retiredChunks: 6, retireDeferred: false });
const after = await chunkRows(page.pageId);
expect(after.map(r => r.occurrence_id)).toEqual(staged3.occurrences);
});
test('two preparers at one generation: the loser\'s staged chunk rows are deleted with the winner\'s flip', async () => {
const t = [segmentText('# Race\nbase\n')];
const page = await publishSegmentedPage(engine, 'notes/flip-two-stagings', t, { chunkTexts: true });
const winner = await stageGeneration(engine, { pageId: page.pageId, slug: 'notes/flip-two-stagings', genFrom: 2, texts: [segmentText('# Race\nwinner\n')], chunkTexts: true });
const loser = await stageGeneration(engine, { pageId: page.pageId, slug: 'notes/flip-two-stagings', genFrom: 2, texts: [segmentText('# Race\nloser\n')], chunkTexts: true });
const result = await engine.flipBodyGeneration(flipInput(page.pageId, 1, winner.stagingId, [segmentText('# Race\nwinner\n')],
{ closeOccurrences: page.occurrences, retireChunkOccurrences: page.occurrences }));
expect(result).toEqual({ retiredChunks: 2, retireDeferred: false });
const rows = await chunkRows(page.pageId);
expect(rows.map(r => r.occurrence_id)).toEqual([winner.occurrences[0]!]);
expect((await engine.readManifest(page.pageId)).map(r => r.occurrenceId)).toEqual(winner.occurrences);
});
test('promotion from inline deletes the inline chunk rows and NULLs the body columns', async () => {
await engine.putPage('notes/promote-inline', { type: 'note', title: 'Promote', compiled_truth: 'inline body', timeline: '' });
const [{ id: pageId }] = await engine.executeRaw<{ id: number }>("SELECT id FROM pages WHERE slug = 'notes/promote-inline' AND source_id = 'default'");
await engine.upsertChunks('notes/promote-inline', [{ chunk_index: 0, chunk_text: 'inline body', chunk_source: 'compiled_truth' }]);
const t = [segmentText('# Promoted\nbody\n')];
const staged = await stageGeneration(engine, { pageId, slug: 'notes/promote-inline', genFrom: 1, texts: t, chunkTexts: true });
const result = await engine.flipBodyGeneration(flipInput(pageId, 0, staged.stagingId, t, { retireInlineChunks: true }));
expect(result).toEqual({ retiredChunks: 1, retireDeferred: false });
const rows = await chunkRows(pageId);
expect(rows.map(r => [r.chunk_index, r.body_generation])).toEqual([[1, 1]]);
const page = await pageRow(pageId);
expect([page.body_storage, page.compiled_truth, page.timeline, page.body_generation]).toEqual(['segmented', null, null, 1]);
});
});
describe('stageSegmentChunks and the page shell', () => {
test('chunk indexes continue past the page maximum and carry segment identity and generation', async () => {
const t = [segmentText('# Chunks\nbody\n')];
const page = await publishSegmentedPage(engine, 'notes/stage-chunks', t, { chunkTexts: true });
const [occ] = await engine.allocateBodyOccurrenceIds(1);
const vector = new Float32Array(await embeddingDims());
vector[0] = 1;
const indexes = await engine.stageSegmentChunks(page.pageId, 2, [chunk(occ!, 0, 'first staged'), { ...chunk(occ!, 1, 'second staged'), embedding: vector, model: 'test-model' }]);
expect(indexes).toEqual([1, 2]);
const rows = await engine.executeRaw<Record<string, unknown>>(`SELECT chunk_index, local_ordinal, raw_byte_start::int AS raw_byte_start, raw_byte_end::int AS raw_byte_end,
body_generation::int AS body_generation, embedding IS NULL AS no_vector, embedding_pending_since IS NOT NULL AS pending, embedded_at IS NOT NULL AS embedded,
embedded_text_hash, model FROM content_chunks WHERE page_id = $1 AND occurrence_id = $2 ORDER BY chunk_index`, [page.pageId, occ]);
expect(rows.map(r => [r.chunk_index, r.local_ordinal, r.body_generation, r.no_vector, r.pending, r.embedded])).toEqual([
[1, 0, 2, true, true, false], [2, 1, 2, false, false, true]]);
expect(rows[1]!.model).toBe('test-model');
expect(rows[1]!.embedded_text_hash).not.toBeNull();
expect([rows[0]!.raw_byte_start, rows[0]!.raw_byte_end]).toEqual([0, 'first staged'.length]);
expect(await engine.stageSegmentChunks(page.pageId, 2, [])).toEqual([]);
});
test('createSegmentedPageShell inserts a segmented row at generation 0 with the meta columns', async () => {
const { id } = await engine.createSegmentedPageShell('notes/shell-probe', { ...pageMeta('Shell'), source_path: 'notes/shell-probe.md', ingested_via: 'test' }, { sourceId: 'default' });
const rows = await engine.executeRaw<Record<string, unknown>>(`SELECT body_storage, body_generation::int AS g, body_bytes::int AS b, compiled_truth, timeline,
title, frontmatter, source_path, ingested_via, ingested_at IS NOT NULL AS ingested FROM pages WHERE id = $1`, [id]);
expect(rows[0]).toEqual({ body_storage: 'segmented', g: 0, b: 0, compiled_truth: null, timeline: null, title: 'Shell',
frontmatter: { status: 'probe', nested: { a: 1 } }, source_path: 'notes/shell-probe.md', ingested_via: 'test', ingested: true });
expect(() => engine.createSegmentedPageShell('../bad slug', pageMeta(), { sourceId: 'default' })).toThrow();
});
});
describe('stagings', () => {
test('openBodyStaging records the row; setBodyStagingState moves it and keeps the commit request id', async () => {
const input = stagingInput({ slug: 'notes/staging-probe', declaredBytes: 12, declaredSha256: segmentText('x').sha256 });
await engine.openBodyStaging(input);
const read = async () => (await engine.executeRaw<Record<string, unknown>>(`SELECT state, kind, mode, declared_bytes::int AS declared,
encode(declared_sha256, 'hex') AS sha, commit_request_id::text AS req FROM page_body_stagings WHERE id = $1::uuid`, [input.id]))[0];
expect(await read()).toEqual({ state: 'open', kind: 'put', mode: 'create', declared: 12, sha: hexOf(input.declaredSha256!), req: null });
const req = crypto.randomUUID();
await engine.setBodyStagingState(input.id, 'committed', { commitRequestId: req });
await engine.setBodyStagingState(input.id, 'committed');
expect(await read()).toMatchObject({ state: 'committed', req });
await expect(engine.openBodyStaging({ ...input, id: crypto.randomUUID() })).rejects.toThrow();
});
});
describe('schema guards', () => {
test('the body CHECK refuses an old-shape body write to a segmented page and a NULL body on an inline page', async () => {
const page = await publishSegmentedPage(engine, 'notes/check-probe', [segmentText('# Check\nbody\n')]);
await expect(engine.executeRaw("UPDATE pages SET compiled_truth = 'x' WHERE id = $1", [page.pageId])).rejects.toThrow(/pages_body_storage_chk/);
await engine.putPage('notes/check-inline', { type: 'note', title: 'Inline', compiled_truth: 'inline', timeline: '' });
await expect(engine.executeRaw("UPDATE pages SET compiled_truth = NULL WHERE slug = 'notes/check-inline'")).rejects.toThrow(/pages_body_storage_chk/);
await expect(engine.executeRaw(`INSERT INTO page_versions (page_id, compiled_truth, body_storage) VALUES ($1, NULL, 'inline')`, [page.pageId])).rejects.toThrow(/page_versions_body_chk/);
});
test('a body_generation-only update moves generation and knowledge_revision and clears the projection seal', async () => {
const page = await publishSegmentedPage(engine, 'notes/revision-probe', [segmentText('# Rev\nbody\n')]);
await engine.executeRaw('UPDATE pages SET text_projection_revision = knowledge_revision WHERE id = $1', [page.pageId]);
const before = await pageRow(page.pageId);
await engine.executeRaw('UPDATE pages SET body_generation = body_generation + 1 WHERE id = $1', [page.pageId]);
const after = await pageRow(page.pageId);
expect(after.generation).toBe((before.generation as number) + 1);
expect(after.knowledge_revision).not.toBe(before.knowledge_revision);
expect(after.text_projection_revision).toBeNull();
});
test('the page search vector trigger fires on a timeline_lead-only update', async () => {
const page = await publishSegmentedPage(engine, 'notes/lead-probe', [segmentText('# Lead\nbody\n')]);
await engine.executeRaw("UPDATE pages SET timeline_lead = 'quixotically renamed lead' WHERE id = $1", [page.pageId]);
expect(String((await pageRow(page.pageId)).search_vector)).toContain('quixot');
const [trigger] = await engine.executeRaw<{ cols: string }>(`SELECT string_agg(a.attname, ',' ORDER BY a.attname) AS cols FROM pg_trigger t
JOIN pg_attribute a ON a.attrelid = t.tgrelid AND a.attnum = ANY(t.tgattr::int2[]) WHERE t.tgname = 'trg_pages_search_vector'`);
expect(trigger!.cols).toBe('timeline,timeline_lead,title');
});
test('JSONB manifest and page columns are stored as objects and arrays, not strings', async () => {
const page = await publishSegmentedPage(engine, 'notes/jsonb-probe', [segmentText('# Json\nbody\n')]);
const [types] = await engine.executeRaw<Record<string, string>>(`SELECT jsonb_typeof(m.entry_state) AS entry_state, jsonb_typeof(m.protected_regions) AS protected_regions,
jsonb_typeof(m.outline) AS outline, jsonb_typeof(p.body_flags) AS body_flags
FROM page_body_manifest m JOIN pages p ON p.id = m.page_id WHERE m.page_id = $1`, [page.pageId]);
expect(types).toEqual({ entry_state: 'object', protected_regions: 'array', outline: 'array', body_flags: 'object' });
});
});
async function embeddingDims(): Promise<number> {
const rows = await engine.executeRaw<{ dim: number }>(
`SELECT atttypmod AS dim FROM pg_attribute WHERE attrelid = 'content_chunks'::regclass AND attname = 'embedding' AND attnum > 0`);
return Number(rows[0]!.dim);
}