Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Fence rewrites of segmented bodies (Wave B, spec 6.4 and 11): forget strikes
* and purge drops fence rows of a segmented page without holding its body as
* one string. Only the facts and takes regions the segmenter stored
* (`protected_regions`) are decoded and handed to the transform; a region that
* crosses a forced cut is joined with its neighbours, up to the single-string
* ceiling. A changed live body is staged again through `prepareSegmentedBody`
* from a stream of its segments with the changed ones replaced, so unchanged
* segments keep their rows, texts and chunks, and publishes through the
* generation flip in the caller's transaction. A changed retained generation
* (purge) is rebuilt under its own number. Memory stays O(segment) on every
* path.
*/
import type { BrainEngine } from '../engine.ts';
import { isQuarantined } from '../quarantine.ts';
import { FACTS_FENCE_END } from '../facts-fence.ts';
import { maintenanceAttribution } from '../persistence/attribution.ts';
import { applySegmentedBody, sealSegmentedPage } from '../persistence/segmented-apply.ts';
import { withPageTierKept } from '../trust/fence-append.ts';
import { prepareSegmentedBody, readSegmentedBody, type SegmentedStagingOwner } from './ingest.ts';
import { FENCE_MAX_BYTES, readPageBodyPolicy, stageBatchBytes } from './limits.ts';
import { fenceTooLargeError } from './read-range.ts';
import { pageBodySegmentedError } from './page-shape.ts';
import { readEntryState } from './entry-state.ts';
import { ContentHashStream } from './content-hash-stream.ts';
import type { StoredRegion } from './segmenter.ts';
import type { BodyRegion, JsonObject, ManifestRow } from './types.ts';
import { assertGenerationTiles } from './body-tiling.ts';
import { deleteRewrittenDerivations } from './segment-derive.ts';
/** Rewrites one fence (the text of a stored facts or takes region, markers included), given the body region holding it; called in body order. */
export type FenceTransform = (fence: string, at: { region: BodyRegion }) => Promise<string>;
/** Rows `from..to` of a generation, replaced by `bytes`. */
interface FenceUnit { from: number; to: number; bytes: Uint8Array }
const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
const hex = (bytes: Uint8Array) => Buffer.from(bytes).toString('hex');
const fenceRegions = (row: ManifestRow) => (row.protectedRegions as StoredRegion[]).filter(r => r.kind === 'facts' || r.kind === 'takes');
/** The region runs on into the next segment (it crosses a forced cut there). */
const runsOn = (r: StoredRegion) => r.open_start !== undefined && r.open_end === undefined;
/**
* Every fence unit of `rows` the transform changes, in order. Reads only segments holding a fence. A unit joins the
* segments a fence region runs on through, and a segment that ends inside a facts marker (the next one's entry state
* carries the marker's first bytes) with the next: a fence region of a segment starting inside its opener begins at the
* region's recorded `open_start`, in the segment before, and a facts fence whose closer a cut splits ends after the
* closer's last bytes, which the next segment holds outside any facts region. A fence longer than FENCE_MAX_BYTES,
* counted from its begin marker through its end marker, refuses before its segments are read.
*/
async function changedUnits(engine: BrainEngine, sourceId: string, rows: readonly ManifestRow[], transform: FenceTransform): Promise<FenceUnit[]> {
const units: FenceUnit[] = [];
const joinsNext = (i: number) => i + 1 < rows.length && rows[i + 1]!.region === rows[i]!.region
&& (fenceRegions(rows[i]!).some(runsOn) || !!readEntryState(rows[i + 1]!.entryState)?.withdrawal.carry);
for (let from = 0; from < rows.length; from++) {
if (!fenceRegions(rows[from]!).length && !joinsNext(from)) continue;
let to = from;
while (joinsNext(to)) to++;
const span = rows.slice(from, to + 1);
const spanStart = span[0]!.byteStart;
const extents: Array<{ start: number; end: number; closer: boolean }> = [];
for (const row of span) {
const regions = fenceRegions(row);
const last = extents[extents.length - 1];
const entry = row === span[0] ? undefined : readEntryState(row.entryState)?.withdrawal;
if (entry?.open && entry.carry && last?.end === row.byteStart && !regions.some(r => r.start === 0)) {
last.end += Buffer.byteLength(FACTS_FENCE_END, 'utf8') - Buffer.byteLength(entry.carry, 'utf8');
last.closer = true;
}
for (const r of regions) {
const start = row.byteStart + r.start, end = row.byteStart + r.end;
const tail = extents[extents.length - 1];
if (tail && tail.end === start) tail.end = end;
else extents.push({ start: Math.min(start, r.open_start ?? start), end, closer: false });
}
}
const overBound = extents.find(e => e.end - e.start > FENCE_MAX_BYTES);
if (overBound || (extents[0] && extents[0].start < spanStart)) {
const [page] = await engine.executeRaw<{ slug: string }>('SELECT slug FROM pages WHERE id=$1', [span[0]!.pageId]);
if (overBound) throw fenceTooLargeError(page?.slug ?? String(span[0]!.pageId), sourceId, overBound.end - overBound.start);
throw pageBodySegmentedError({ slug: page?.slug, source_id: sourceId }, 'fence rewrite of a fence whose opener lies before the segments joined for it');
}
const texts = await engine.readSegmentTexts(sourceId, span.map(row => row.sha256));
const raw = Buffer.concat(span.map(row => {
const text = texts.get(hex(row.sha256));
if (!text) throw new Error(`Segment text ${hex(row.sha256)} of page ${row.pageId} is missing.`);
return text;
}));
const parts: Buffer[] = [];
let cursor = 0, changed = false;
for (const e of extents) {
const r = { start: e.start - spanStart, end: e.end - spanStart };
const fence = raw.subarray(r.start, r.end).toString('utf8');
if (e.closer && !fence.endsWith(FACTS_FENCE_END)) throw new Error(`Facts fence at stored byte ${e.start} of page ${span[0]!.pageId} does not end with its end marker across the cut.`);
const next = await transform(fence, { region: span[0]!.region });
parts.push(raw.subarray(cursor, r.start), Buffer.from(next, 'utf8'));
cursor = r.end;
if (next !== fence) changed = true;
}
if (changed) units.push({ from, to, bytes: Buffer.concat([...parts, raw.subarray(cursor)]) });
from = to;
}
return units;
}
/** One region of a generation's stored stream with the changed units replaced, read one segment batch at a time. */
async function* rewrittenRegion(engine: BrainEngine, sourceId: string, rows: readonly ManifestRow[], units: readonly FenceUnit[], region: BodyRegion): AsyncIterable<Uint8Array> {
const unitAt = new Map(units.map(unit => [unit.from, unit]));
const batchBytes = stageBatchBytes(engine.kind);
let batch: ManifestRow[] = [], bytes = 0;
async function* flush(): AsyncIterable<Uint8Array> {
const texts = await engine.readSegmentTexts(sourceId, batch.map(row => row.sha256));
for (const row of batch) {
const text = texts.get(hex(row.sha256));
if (!text) throw new Error(`Segment text ${hex(row.sha256)} of page ${row.pageId} is missing.`);
yield text;
}
batch = []; bytes = 0;
}
for (let i = 0; i < rows.length; i++) {
const row = rows[i]!;
if (row.region !== region) continue;
const unit = unitAt.get(i);
if (unit) {
yield* flush();
yield unit.bytes;
i = unit.to;
continue;
}
if (batch.length && bytes + row.byteLen > batchBytes) yield* flush();
batch.push(row);
bytes += row.byteLen;
}
if (batch.length) yield* flush();
}
const bodyBytesOf = (rows: readonly ManifestRow[], units: readonly FenceUnit[]) => rows.reduce((n, row) => n + row.byteLen, 0)
+ units.reduce((n, unit) => n + unit.bytes.length - rows.slice(unit.from, unit.to + 1).reduce((m, row) => m + row.byteLen, 0), 0);
async function rewriteOwner(engine: BrainEngine): Promise<SegmentedStagingOwner> {
const { principal } = await maintenanceAttribution(engine);
return { kind: 'edit', principalKind: principal.kind, principalId: principal.id, baseRevision: null };
}
/** The replaced rows' segment shas: texts the rewrite stopped referencing from this generation. */
const replacedShas = (rows: readonly ManifestRow[], units: readonly FenceUnit[]) => units.flatMap(unit => rows.slice(unit.from, unit.to + 1).map(row => row.sha256));
/**
* The tier and corpus generation the page's stored vectors were embedded under (its contextual-retrieval stamp), so vector
* reuse admits them as an import of the same body would. A fence edit is not authorship: the stamp stays as it is.
*/
const rewriteTier = (page: { contextual_retrieval_mode: string | null; corpus_generation: string | null }) => page.contextual_retrieval_mode === 'title'
? { tier: 'title' as const, corpusGeneration: page.corpus_generation } : { tier: 'none' as const, corpusGeneration: null };
export interface LiveFenceRewrite {
/** Segment shas the old generation referenced in place of the rewritten units. */
replaced: Uint8Array[];
/** page_segment_derivations rows deleted because they may hold the replaced text (deleteRewrittenDerivations). */
derivations: number;
/** False when the page was left unsealed (its projection runs after commit: `buildSegmentedProjection`). */
sealed: boolean;
}
/**
* Rewrite the fences of a segmented page's live body through `transform`, in
* the caller's transaction: a new generation whose unchanged segments keep
* their rows and chunks, and whose rewritten segments get new occurrences
* (their chunks staged here, the old ones retired by the flip), and the
* derivation rows that may hold the replaced text are deleted. A fence edit
* is not authorship, so the page keeps its trust tier; a soft-deleted page
* stays deleted. Null when no fence changed (nothing is written).
*/
export async function rewriteLiveSegmentedFences(tx: BrainEngine, pageId: number, transform: FenceTransform): Promise<LiveFenceRewrite | null> {
const [page] = await tx.executeRaw<{ id: number; slug: string; source_id: string; title: string; type: string; page_kind: string; frontmatter: JsonObject | null;
body_generation: number | string; body_bytes: number | string; body_flags: JsonObject | null; body_storage: string; deleted_at: string | null; tags: string[];
contextual_retrieval_mode: string | null; corpus_generation: string | null }>(`SELECT p.id,p.slug,p.source_id,p.title,
p.type,p.page_kind,p.frontmatter,p.body_generation,p.body_bytes,p.body_flags,p.body_storage,p.deleted_at,p.contextual_retrieval_mode,p.corpus_generation,
COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id),'[]'::jsonb) AS tags FROM pages p WHERE p.id=$1`, [pageId]);
if (page?.body_storage !== 'segmented') return null;
const sourceId = page.source_id, slug = page.slug, frontmatter = page.frontmatter ?? {};
const generation = Number(page.body_generation);
const rows = await tx.readManifest(pageId, { generation });
await assertGenerationTiles(tx, pageId, generation, rows, Number(page.body_bytes));
const units = await changedUnits(tx, sourceId, rows, transform);
if (!units.length) return null;
const prepared = await prepareSegmentedBody(tx, { sourceId, slug, title: page.title, type: page.type, frontmatter, tags: page.tags,
live: { id: pageId, generation, storage: 'segmented', title: page.title, type: page.type, flags: page.body_flags },
truth: rewrittenRegion(tx, sourceId, rows, units, 'truth'), timeline: rewrittenRegion(tx, sourceId, rows, units, 'timeline'),
hasTimeline: rows.some(row => row.region === 'timeline'), flags: page.body_flags ?? {}, declaredBytes: bodyBytesOf(rows, units),
owner: await rewriteOwner(tx), chunking: { quarantined: isQuarantined(frontmatter), ...rewriteTier(page) } });
const { flipRetireMaxRows } = await readPageBodyPolicy(tx);
const applied = await withPageTierKept(tx, { sourceId, slug }, () => applySegmentedBody(tx, { slug, sourceId, pageId, prepared, promotion: false,
retireMaxRows: flipRetireMaxRows, meta: { title: page.title, type: page.type as never, frontmatter, page_kind: page.page_kind as never } }));
if (page.deleted_at) await tx.executeRaw('UPDATE pages SET deleted_at=$2 WHERE id=$1', [pageId, page.deleted_at]);
if (applied.complete) await sealSegmentedPage(tx, pageId);
const derivations = await deleteRewrittenDerivations(tx, pageId, rows, units);
return { replaced: replacedShas(rows, units), sealed: applied.complete, derivations };
}
const MANIFEST_COLUMNS = `page_id,occurrence_id,seq,region,source_id,sha256,byte_start,byte_len,line_start,line_count,cut_kind,entry_state,
protected_regions,facts_slots,static_proj_bytes,static_proj_lines,outline,date_min,date_max,hash_state`;
/**
* Rebuild retained generation `generation` of a segmented page with its fences
* rewritten (purge of a version), in the caller's transaction. The generation
* gets rows of its own: rows it shared with neighbouring generations keep
* their other generations, and the rebuilt rows (new occurrences, every
* coordinate and hash midstate recomputed) cover exactly this one. Each
* version row of the generation gets the rebuilt content_hash and size.
* Returns the replaced segment shas, or null when no fence changed. A change
* to the page's live generation is refused (the page row is locked, so the
* generation cannot become live under the rebuild).
*/
export async function rewriteSegmentedGeneration(tx: BrainEngine, pageId: number, generation: number, transform: FenceTransform,
version: { title: string; type: string; frontmatter: JsonObject; tags: readonly string[] }): Promise<Uint8Array[] | null> {
const [page] = await tx.executeRaw<{ slug: string; source_id: string; body_flags: JsonObject | null; body_storage: string; body_generation: number | string }>(
'SELECT slug,source_id,body_flags,body_storage,body_generation FROM pages WHERE id=$1 FOR UPDATE', [pageId]);
if (!page) return null;
const sourceId = page.source_id;
const rows = await tx.readManifest(pageId, { generation });
await assertGenerationTiles(tx, pageId, generation, rows);
const units = await changedUnits(tx, sourceId, rows, transform);
if (!units.length) return null;
// Rebuilding rows in place is for retained generations only: the live one changes by a flip (rewriteLiveSegmentedFences),
// which moves the revision and the chunks with it.
if (page.body_storage === 'segmented' && Number(page.body_generation) === generation) {
throw new Error(`rewriteSegmentedGeneration: generation ${generation} is the live body of page ${page.slug}; rewrite it with rewriteLiveSegmentedFences.`);
}
const prepared = await prepareSegmentedBody(tx, { sourceId, slug: page.slug, live: null, ...version,
truth: rewrittenRegion(tx, sourceId, rows, units, 'truth'), timeline: rewrittenRegion(tx, sourceId, rows, units, 'timeline'),
hasTimeline: rows.some(row => row.region === 'timeline'), flags: page.body_flags ?? {}, declaredBytes: bodyBytesOf(rows, units),
owner: await rewriteOwner(tx), chunking: { quarantined: true, corpusGeneration: null, tier: 'none' } });
const staging = prepared.stagingId, next = generation + 1;
// The part of each shared row after this generation moves to a copy (staged, so the live-seq index sees one live row);
// the part before it ends here; rows that began here go. Then the rebuilt rows take [generation, next).
await tx.executeRaw(`INSERT INTO page_body_manifest (${MANIFEST_COLUMNS},gen_from,gen_to,staging_id) SELECT ${MANIFEST_COLUMNS},$3::bigint,gen_to,$4::uuid
FROM page_body_manifest WHERE page_id=$1 AND staging_id IS NULL AND gen_from<=$2 AND (gen_to IS NULL OR gen_to>$3)`, [pageId, generation, next, staging]);
await tx.executeRaw('UPDATE page_body_manifest SET gen_to=$2 WHERE page_id=$1 AND staging_id IS NULL AND gen_from<$2 AND (gen_to IS NULL OR gen_to>$2)', [pageId, generation]);
await tx.executeRaw('DELETE FROM page_body_manifest WHERE page_id=$1 AND staging_id IS NULL AND gen_from=$2', [pageId, generation]);
await tx.stageManifestRows(prepared.createRows!.map(row => ({ ...row, pageId, genFrom: generation })));
await tx.executeRaw(`UPDATE page_body_manifest SET staging_id=NULL, gen_to=CASE WHEN gen_from=$2 THEN $3 ELSE gen_to END
WHERE page_id=$1 AND staging_id=$4::uuid`, [pageId, generation, next, staging]);
await tx.setBodyStagingState(staging, 'committed');
await tx.executeRaw(`UPDATE page_versions SET content_hash=$3,body_bytes=$4 WHERE page_id=$1 AND body_generation=$2 AND body_storage='segmented'`,
[pageId, generation, prepared.body.content_hash, prepared.body.body_bytes]);
return replacedShas(rows, units);
}
/**
* `content_hash` of a segmented page's body at `generation` with its fences
* rewritten (as an import through the purge overlay would hash it), streamed;
* null when no fence changes, so the stored hash stands.
*/
export async function rewrittenContentHash(engine: BrainEngine, pageId: number, sourceId: string, generation: number, transform: FenceTransform,
meta: { title: unknown; type: unknown; frontmatter: Record<string, unknown>; tags: string[] }): Promise<string | null> {
const rows = await engine.readManifest(pageId, { generation });
await assertGenerationTiles(engine, pageId, generation, rows);
const units = await changedUnits(engine, sourceId, rows, transform);
if (!units.length) return null;
const hash = ContentHashStream.begin(meta);
for await (const bytes of rewrittenRegion(engine, sourceId, rows, units, 'truth')) hash.pushTruth(utf8.decode(bytes));
for await (const bytes of rewrittenRegion(engine, sourceId, rows, units, 'timeline')) hash.pushTimeline(utf8.decode(bytes));
return hash.finish(meta);
}
/**
* Whether a segmented body (live, or `generation`) contains any of `needles`,
* case-insensitively, within one region: each segment is lower-cased on its
* own and searched with the tail of the previous one prepended (a needle split
* by a cut is found). `'deadline'` when `deadline` (performance.now()) passed.
*/
export async function segmentedBodyContains(engine: BrainEngine, pageId: number, sourceId: string, needles: readonly string[],
opts: { generation?: number; deadline?: number } = {}): Promise<boolean | 'deadline'> {
const lowered = needles.map(n => n.toLowerCase()).filter(Boolean);
if (!lowered.length) return false;
const keep = Math.max(...lowered.map(n => n.length)) - 1;
let carry = '', region: BodyRegion | null = null;
for await (const part of readSegmentedBody(engine, pageId, sourceId, opts.generation === undefined ? {} : { generation: opts.generation })) {
if (opts.deadline !== undefined && performance.now() > opts.deadline) return 'deadline';
if (part.region !== region) { carry = ''; region = part.region; }
const text = carry + utf8.decode(part.bytes).toLowerCase();
if (lowered.some(n => text.includes(n))) return true;
carry = keep > 0 ? text.slice(-keep) : '';
}
return false;
}