Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
37.6 KB
Raw Blame
/**
 * Segmented page storage (Wave B, spec 7.1-7.3): ranged reads of a segmented
 * body, one segment text at a time, never the whole body as one string.
 *
 * Two views, chosen by space and audience:
 *
 * - The read projection (space 'projected', either audience; space 'stored'
 *   for a remote caller): the text projectPageText gives the joined body
 *   (page-window/document.ts), the one text Wave A windows, outlines, chunks
 *   and evidence are cut from, the same for local and remote callers.
 *   Projected offsets are its UTF-8 bytes: truth segments, the low-tier block
 *   after the truth, TIMELINE_SEPARATOR and the timeline segments when the
 *   timeline shows any text. Each segment is projected whole from its stored
 *   regions (segment-projection.ts) after the withdrawal overlay, then cut.
 * - The trusted body (space 'stored', audience 'trusted'): the stored stream
 *   after the withdrawal overlay, what readPageSnapshot gives a trusted reader
 *   of an inline page (export, derived passes, a local full read). It has no
 *   coordinates of its own: `projectedStart` repeats `storedStart`.
 *
 * Offsets are located from manifest sums: a page's rows are one light query,
 * static projection sizes add up, and only segments with facts slots are read
 * to add their rendered facts tables and to load the page's fence overlay
 * (`projectedLayout`, cached on the handle). A stored-space range selects the
 * rows it overlaps; a token, a rendered facts table or a block the overlay
 * rewrote is emitted whole or not at all.
 *
 * Offsets can drift without a revision change (a hold or a tier change moves
 * the projection, and a withdrawal block outside a facts region changes a
 * segment's length the sums do not see): a read can then repeat or skip text,
 * never show text the projection hides.
 */
import { createHash } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import { TIMELINE_SEPARATOR } from '../page-window/document.ts';
import { sanitizeRemoteBody } from '../remote-body.ts';
import { credentialSafeProjection } from '../credential-projection.ts';
import { loadFenceChunkOverlay, markFenceChunk, renderFenceLowTier, type FenceChunkOverlay, type FenceLowTierRow } from '../eligibility/fence-overlay.ts';
import { partialFenceRuns, withdrawalFenceBlocks } from '../facts/withdrawal-overlay.ts';
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END } from '../facts-fence.ts';
import { assertRowsTile, generationBodyBytes, LIVE_READ_ATTEMPTS } from './body-tiling.ts';
import { readEntryState } from './entry-state.ts';
import { FENCE_MAX_BYTES, SEGMENT_MAX_BYTES } from './limits.ts';
import { ledgerApplies, readBodyLedger, overlaySegmentText, type BodyLedger } from './segment-ledger.ts';
import { regionTrust, SEGMENTER_VERSION, type FactsSlot, type OutlineHeading, type StoredRegion } from './segmenter.ts';
import { overlaidPieces, projectionStaleError, projectSegment, type ProjectedPiece, type SegmentProjection } from './segment-projection.ts';
import type { BodyRegion, JsonObject } from './types.ts';
import { opError } from '../ops/contract.ts';
import { REVISION_BACKFILL_PENDING } from '../page-state/types.ts';

export type BodySpace = 'stored' | 'projected';
export type BodyAudience = 'trusted' | 'remote';

export type { BodyLedger } from './segment-ledger.ts';

/** One manifest row of the read generation, without its JSON metadata. */
export interface BodyRow {
  seq: number;
  occurrenceId: number;
  region: BodyRegion;
  sha256: Uint8Array;
  byteStart: number;
  byteLen: number;
  lineStart: number;
  lineCount: number;
  staticProjBytes: number;
  staticProjLines: number;
  outlineCount: number;
  hasFacts: boolean;
  hasDate: boolean;
}

/**
 * What a caller authorized before it reads a segmented body: the page's knowledge revision and, when its snapshot says,
 * the body generation. Every visibility, deletion, metadata or body change advances the revision, so a pinned read
 * refuses (revision_changed) rather than read a body or a page state nobody authorized; `admit` re-runs the parts of
 * the authorization the revision does not cover (a trust floor, a re-derivation hold) and throws to refuse.
 */
export interface BodyPin {
  revision: string;
  generation?: number;
  admit?: () => Promise<void>;
}

/** A segmented body opened for reading: the page, the generation's rows and the ledger. Reuse it across reads of one request. */
export interface SegmentedBody {
  pageId: number;
  sourceId: string;
  slug: string;
  generation: number;
  bodyBytes: number;
  bodyLines: number;
  truthBytes: number;
  flags: JsonObject;
  rows: BodyRow[];
  ledger: BodyLedger;
  /** The page was segmented by another segmenter version: every segment's regions are recomputed. */
  stale: boolean;
  layout?: ProjectedLayout;
  /** Set when the read follows an authorization: confirmBodyPin checks it again before the text is returned. */
  pin?: BodyPin;
  /** GST-2891: a read for a caller whose fact reads are world-only: its fence overlays omit the rows held from it. */
  hideHeld?: boolean;
}

/** Where every row of the read projection sits (`projectedLayout`). */
export interface ProjectedLayout {
  /** Document byte of each row's projection, its bytes and its newlines; -1 for timeline rows when the timeline shows nothing. */
  rowStart: number[];
  rowBytes: number[];
  rowNewlines: number[];
  /** Rendered facts table bytes per slot, for rows with facts slots. */
  rowFacts: Array<number[] | null>;
  /** The low-tier block after the truth ('' when none) and where it starts. */
  lowTier: string;
  lowTierStart: number;
  /** Where the timeline text starts after TIMELINE_SEPARATOR; -1 when the page shows no timeline. */
  timelineByte: number;
  totalBytes: number;
  newlines: number;
  overlay: FenceChunkOverlay | undefined;
}

export interface BodyRangeRequest {
  pageId: number;
  /** Default: the live generation. */
  generation?: number;
  space: BodySpace;
  /** Bytes in that space, [start, end); `end` may be Infinity. */
  start: number;
  end: number;
  audience: BodyAudience;
  /** Rows of one region only (the low-tier block counts as truth; TIMELINE_SEPARATOR as neither). */
  region?: BodyRegion;
  /** An opened body (openSegmentedBody), reused across reads. */
  body?: SegmentedBody;
  /** The page's ledger when the caller read a snapshot; else loaded. */
  ledger?: BodyLedger;
  /** The authorization this read follows (openSegmentedBody). */
  pin?: BodyPin;
}

export interface BodyRangePiece {
  bytes: Buffer;
  /** Stored-stream byte the piece stands at (for a token or a facts table, its region's start). */
  storedStart: number;
  /** Byte of the piece in the read projection (the trusted body: equal to storedStart). */
  projectedStart: number;
}

const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
const HEAVY_BATCH = 64;

/**
 * @internal Manifest rows a read visits (`visits`): each row a range read, a block walk or a layout pass looks at, for
 * the linearity test. A pass that filters the whole manifest per block visits rows x blocks; one walk visits each row
 * a bounded number of times. Counts, not time.
 */
export const bodyRowWork = { visits: 0 };
const toJson = (v: unknown) => typeof v === 'string' ? JSON.parse(v) as unknown : v;
const bytesOf = (v: unknown): Uint8Array => { const b = v as Uint8Array; return new Uint8Array(b.buffer, b.byteOffset, b.byteLength); };
const newlines = (s: string) => { let n = 0; for (let i = s.indexOf('\n'); i !== -1; i = s.indexOf('\n', i + 1)) n++; return n; };

/** The page row a pin was taken at: same revision and, when pinned, the same body generation. */
async function checkPin(pin: BodyPin, page: Record<string, unknown> | undefined, pageId: number): Promise<void> {
  const revision = page?.revision == null ? REVISION_BACKFILL_PENDING : String(page.revision);
  if (!page || revision !== pin.revision || (pin.generation !== undefined && Number(page.body_generation) !== pin.generation)) {
    const slug = page?.slug == null ? String(pageId) : String(page.slug);
    throw opError('revision_changed', `Page ${slug} changed after this read was authorized; nothing from the read was returned.`,
      "Read the page again: get_page body: 'none' returns its current revision, then read the window or outline you wanted.",
      { legacy_error: 'invalid_params', detail: JSON.stringify({ page_id: pageId }) });
  }
  await pin.admit?.();
}

/** A generation's manifest rows, each carrying the generation read (`generation` null: the page's live one at the statement). */
function manifestRows(engine: BrainEngine, pageId: number, generation: number | null): Promise<Record<string, unknown>[]> {
  return engine.executeRaw<Record<string, unknown>>(`SELECT m.seq, m.occurrence_id, m.region, m.sha256, m.byte_start, m.byte_len,
      m.line_start, m.line_count, m.static_proj_bytes, m.static_proj_lines, jsonb_array_length(m.outline) AS outline_count,
      jsonb_array_length(m.facts_slots) > 0 AS has_facts, m.date_min IS NOT NULL AS has_date, g.generation
    FROM (SELECT COALESCE($2::bigint, (SELECT body_generation FROM pages WHERE id=$1)) AS generation) g
    JOIN page_body_manifest m ON m.page_id=$1 AND m.staging_id IS NULL AND m.gen_from<=g.generation AND (m.gen_to IS NULL OR m.gen_to>g.generation)
    ORDER BY m.seq`, [pageId, generation]);
}

/**
 * Open a segmented body. With `pin` the read is the authorized one: the page must still be at the pinned revision (and
 * generation), the caller's admission is re-run, and the rows read are that generation's.
 *
 * The page row, the manifest rows and (given `expect`, the slug and source the caller already holds) the ledger are
 * read at once: rows read for another generation than the page row names (a body published between the statements)
 * are read again, and a ledger read under a name the page row does not carry is read again under its own. A live read
 * whose generation a publish retired before its rows were read (they no longer tile it) opens the new live generation,
 * up to LIVE_READ_ATTEMPTS times (GBRA-287).
 */
export async function openSegmentedBody(engine: BrainEngine, opts: { pageId: number; generation?: number; ledger?: BodyLedger; pin?: BodyPin;
  expect?: { slug: string; sourceId: string }; hideHeld?: boolean }, attempt = 1): Promise<SegmentedBody> {
  const requested = opts.pin?.generation ?? opts.generation ?? null;
  const [[page], read, expected] = await Promise.all([
    engine.executeRaw<Record<string, unknown>>(`SELECT id, source_id, slug, body_storage, body_generation, body_bytes, body_lines, truth_bytes,
      body_flags, segmenter_version, knowledge_revision::text AS revision FROM pages WHERE id=$1`, [opts.pageId]),
    manifestRows(engine, opts.pageId, requested),
    opts.ledger || !opts.expect ? undefined : readBodyLedger(engine, opts.expect.sourceId, opts.expect.slug),
  ]);
  if (opts.pin) await checkPin(opts.pin, page, opts.pageId);
  const generation = requested ?? Number(page?.body_generation);
  // An earlier generation (a segmented version) keeps its manifest rows after the page was demoted to inline storage.
  if (!page || (page.body_storage !== 'segmented' && generation === Number(page.body_generation))) throw new Error(`Page ${opts.pageId} is not a segmented page.`);
  const rows = requested === null && Number(read[0]?.generation) !== generation ? await manifestRows(engine, opts.pageId, generation) : read;
  const slug = String(page.slug);
  const sourceId = String(page.source_id);
  const bodyRows = rows.map(r => ({
    seq: Number(r.seq), occurrenceId: Number(r.occurrence_id), region: r.region as BodyRegion, sha256: bytesOf(r.sha256),
    byteStart: Number(r.byte_start), byteLen: Number(r.byte_len), lineStart: Number(r.line_start), lineCount: Number(r.line_count), staticProjBytes: Number(r.static_proj_bytes),
    staticProjLines: Number(r.static_proj_lines), outlineCount: Number(r.outline_count), hasFacts: r.has_facts === true, hasDate: r.has_date === true,
  }));
  try {
    assertRowsTile(bodyRows, { slug, sourceId, total: await generationBodyBytes(engine, { id: opts.pageId, body_generation: page.body_generation, body_bytes: page.body_bytes }, generation) });
  } catch (error) {
    if (requested !== null || attempt >= LIVE_READ_ATTEMPTS) throw error;
    const [now] = await engine.executeRaw<{ body_generation: unknown }>('SELECT body_generation FROM pages WHERE id=$1', [opts.pageId]);
    if (!now || Number(now.body_generation) === generation) throw error;
    return openSegmentedBody(engine, opts, attempt + 1);
  }
  const ledger = opts.ledger ?? (expected && opts.expect!.slug === slug && opts.expect!.sourceId === sourceId ? expected : await readBodyLedger(engine, sourceId, slug));
  return {
    pageId: opts.pageId, sourceId, slug, generation, bodyBytes: Number(page.body_bytes ?? 0), bodyLines: Number(page.body_lines ?? 0), truthBytes: Number(page.truth_bytes ?? 0),
    flags: (toJson(page.body_flags) ?? {}) as JsonObject, ledger, stale: Number(page.segmenter_version) !== SEGMENTER_VERSION, ...(opts.pin ? { pin: opts.pin } : {}),
    ...(opts.hideHeld ? { hideHeld: true } : {}), rows: bodyRows,
  };
}

/** After a pinned read, before its text is returned: the page is still at the pin and still admits the caller. */
export async function confirmBodyPin(engine: BrainEngine, body: SegmentedBody): Promise<void> {
  if (!body.pin) return;
  const [page] = await engine.executeRaw<Record<string, unknown>>('SELECT slug, body_generation, knowledge_revision::text AS revision FROM pages WHERE id=$1', [body.pageId]);
  await checkPin(body.pin, page, body.pageId);
}

interface LoadedSegment {
  row: BodyRow;
  bytes: Uint8Array;
  entryState: unknown;
  protectedRegions: StoredRegion[];
  /** The text's sha256 is not the row's. */
  mismatch: boolean;
}

/** The rows' texts and JSON metadata, one text at a time (metadata in batches, read alongside the batch's first text). */
async function* loadSegments(engine: BrainEngine, body: SegmentedBody, rows: readonly BodyRow[], stop?: () => void): AsyncIterable<LoadedSegment> {
  const readText = (row: BodyRow) => { stop?.(); return engine.readSegmentTexts(body.sourceId, [row.sha256]); };
  for (let at = 0; at < rows.length; at += HEAVY_BATCH) {
    const batch = rows.slice(at, at + HEAVY_BATCH);
    const [first, metaRows] = await Promise.all([readText(batch[0]!), engine.executeRaw<Record<string, unknown>>(`SELECT occurrence_id, entry_state, protected_regions
      FROM page_body_manifest WHERE page_id=$1 AND occurrence_id=ANY($2::bigint[]) AND staging_id IS NULL AND gen_from<=$3 AND (gen_to IS NULL OR gen_to>$3)`,
    [body.pageId, batch.map(r => r.occurrenceId), body.generation])]);
    const meta = new Map(metaRows.map(m => [Number(m.occurrence_id), m]));
    for (const [i, row] of batch.entries()) {
      const m = meta.get(row.occurrenceId);
      const bytes = (i === 0 ? first : await readText(row)).get(Buffer.from(row.sha256).toString('hex'));
      if (!m || bytes === undefined) throw projectionStaleError(row.byteStart, 'its manifest row or text is missing');
      yield {
        row, bytes, entryState: toJson(m.entry_state), protectedRegions: (toJson(m.protected_regions) ?? []) as StoredRegion[],
        mismatch: !createHash('sha256').update(bytes).digest().equals(Buffer.from(row.sha256)),
      };
    }
  }
}

function overlaidText(engine: BrainEngine, body: SegmentedBody, seg: LoadedSegment, raw: string, view: 'read' | 'trusted'): Promise<string> {
  return overlaySegmentText(engine, body.ledger, { sourceId: body.sourceId, byteStart: seg.row.byteStart, entryState: seg.entryState }, raw, view);
}

/** The fence overlay for a segment: the page's (layout), plus the rows of fence blocks outside its facts slots. */
async function segmentOverlay(engine: BrainEngine, body: SegmentedBody, seg: LoadedSegment, text: string, base: FenceChunkOverlay | undefined) {
  if (seg.row.hasFacts || !text.includes('gbrain:facts:begin')) return base;
  const extra = await loadFenceChunkOverlay(engine, { sourceId: body.sourceId, slug: body.slug, compiled_truth: fenceBlockText(text) }, { hideHeld: body.hideHeld });
  if (!extra) return base;
  if (!base) return extra;
  return { omit: new Set([...base.omit, ...extra.omit]), demote: new Map([...base.demote, ...extra.demote]),
    unconfirmed: new Set([...(base.unconfirmed ?? []), ...(extra.unconfirmed ?? [])]) };
}

function fenceBlockText(text: string): string {
  return withdrawalFenceBlocks(text).map(b => text.slice(b.start, b.end)).join('\n');
}

async function projectLoaded(engine: BrainEngine, body: SegmentedBody, seg: LoadedSegment, overlay: FenceChunkOverlay | undefined): Promise<SegmentProjection> {
  const raw = utf8.decode(seg.bytes);
  const text = await overlaidText(engine, body, seg, raw, 'read');
  return projectSegment({ region: seg.row.region, bytes: seg.bytes, byteStart: seg.row.byteStart, lineStart: seg.row.lineStart, entryState: seg.entryState,
    protectedRegions: seg.protectedRegions, fenceTrust: regionTrust(body.flags, seg.row.region) },
  { text, fenceOverlay: await segmentOverlay(engine, body, seg, text, overlay), stale: body.stale || seg.mismatch });
}

interface RowSummary { bytes: number; newlines: number; facts: number[]; lowTier: FenceLowTierRow[]; blank: boolean }

function summarize(p: SegmentProjection): RowSummary {
  let bytes = 0, lines = 0, blank = true;
  for (const piece of p.pieces) {
    bytes += Buffer.byteLength(piece.text, 'utf8');
    lines += newlines(piece.text);
    if (blank && piece.text.trim()) blank = false;
  }
  return { bytes, newlines: lines, facts: p.factsBytes, lowTier: p.lowTier, blank };
}

/**
 * Where each row sits in the read projection. Reads only the segments with facts slots (twice when the page's fence
 * overlay is not empty: its world rows decide the overlay), plus, when no stored sum says so, timeline segments until
 * one shows text.
 */
export async function projectedLayout(engine: BrainEngine, body: SegmentedBody): Promise<ProjectedLayout> {
  if (body.layout) return body.layout;
  const factsRows = body.rows.filter(r => r.hasFacts);
  const summaries = new Map<number, RowSummary>();
  const blocks: string[] = [];
  for await (const seg of loadSegments(engine, body, factsRows)) {
    const text = await overlaidText(engine, body, seg, utf8.decode(seg.bytes), 'read');
    blocks.push(fenceBlockText(text));
    summaries.set(seg.row.seq, summarize(await projectLoaded(engine, body, seg, undefined)));
  }
  const overlay = blocks.length ? await loadFenceChunkOverlay(engine, { sourceId: body.sourceId, slug: body.slug, compiled_truth: blocks.join('\n') }, { hideHeld: body.hideHeld }) : undefined;
  if (overlay) for await (const seg of loadSegments(engine, body, factsRows)) summaries.set(seg.row.seq, summarize(await projectLoaded(engine, body, seg, overlay)));

  const n = body.rows.length;
  const layout: ProjectedLayout = { rowStart: new Array(n).fill(-1), rowBytes: new Array(n).fill(0), rowNewlines: new Array(n).fill(0),
    rowFacts: new Array(n).fill(null), lowTier: '', lowTierStart: 0, timelineByte: -1, totalBytes: 0, newlines: 0, overlay };
  let pos = 0, lines = 0;
  const place = (i: number) => {
    const row = body.rows[i]!;
    const s = summaries.get(row.seq);
    layout.rowStart[i] = pos;
    layout.rowBytes[i] = s?.bytes ?? row.staticProjBytes;
    layout.rowNewlines[i] = s?.newlines ?? row.staticProjLines;
    layout.rowFacts[i] = s?.facts ?? null;
    pos += layout.rowBytes[i]!;
    lines += layout.rowNewlines[i]!;
  };
  body.rows.forEach((row, i) => { if (row.region === 'truth') place(i); });
  const low = (region: BodyRegion) => body.rows.filter(r => r.region === region).flatMap(r => summaries.get(r.seq)?.lowTier ?? []);
  layout.lowTier = [...renderFenceLowTier(low('truth')), ...renderFenceLowTier(low('timeline'))]
    .map(({ tier, unconfirmed, body: text }) => '\n\n' + markFenceChunk(tier, credentialSafeProjection(sanitizeRemoteBody(text)), unconfirmed)).join('');
  layout.lowTierStart = pos;
  pos += Buffer.byteLength(layout.lowTier, 'utf8');
  lines += newlines(layout.lowTier);
  if (await timelineVisible(engine, body, summaries)) {
    pos += Buffer.byteLength(TIMELINE_SEPARATOR, 'utf8');
    lines += newlines(TIMELINE_SEPARATOR);
    layout.timelineByte = pos;
    body.rows.forEach((row, i) => { if (row.region === 'timeline') place(i); });
  }
  layout.totalBytes = pos;
  layout.newlines = lines;
  body.layout = layout;
  return layout;
}

/** projectPageText shows the timeline only when its projection holds non-whitespace text. */
async function timelineVisible(engine: BrainEngine, body: SegmentedBody, summaries: Map<number, RowSummary>): Promise<boolean> {
  for (const row of body.rows) {
    if (row.region !== 'timeline') continue;
    const known = summaries.get(row.seq);
    if (known) { if (!known.blank) return true; continue; }
    if (row.staticProjBytes === 0) continue;
    if (row.hasDate || row.outlineCount > 0) return true;
    for await (const seg of loadSegments(engine, body, [row])) if (!summarize(await projectLoaded(engine, body, seg, undefined)).blank) return true;
  }
  return false;
}

/** Projected totals for `body: 'none'` and window metadata: bytes, lines (as window.ts totalLines counts them), timeline start. */
export async function projectedTotals(engine: BrainEngine, body: SegmentedBody): Promise<{ total_bytes: number; total_lines: number; timeline_byte_start: number }> {
  const layout = await projectedLayout(engine, body);
  let last = '';
  if (layout.totalBytes > 0) {
    for await (const piece of readBodyRange(engine, { pageId: body.pageId, space: 'projected', start: layout.totalBytes - 1, end: layout.totalBytes, audience: 'remote', body })) last = piece.bytes.toString('latin1');
  }
  const lines = layout.totalBytes === 0 ? 0 : layout.newlines + (last.endsWith('\n') ? 0 : 1);
  return { total_bytes: layout.totalBytes, total_lines: lines, timeline_byte_start: layout.timelineByte };
}

export type ProjectedUnit = { kind: 'row'; index: number; start: number } | { kind: 'text'; text: string; start: number; storedStart: number; region: BodyRegion | null };
type Unit = ProjectedUnit;

/** The read projection's parts in document order: truth rows, the low-tier block, the separator, timeline rows. */
export function projectedUnits(body: SegmentedBody, layout: ProjectedLayout): Unit[] {
  const out: Unit[] = [];
  body.rows.forEach((row, i) => { if (row.region === 'truth') out.push({ kind: 'row', index: i, start: layout.rowStart[i]! }); });
  if (layout.lowTier) out.push({ kind: 'text', text: layout.lowTier, start: layout.lowTierStart, storedStart: body.truthBytes, region: 'truth' });
  if (layout.timelineByte >= 0) {
    out.push({ kind: 'text', text: TIMELINE_SEPARATOR, start: layout.timelineByte - Buffer.byteLength(TIMELINE_SEPARATOR, 'utf8'), storedStart: body.truthBytes, region: null });
    body.rows.forEach((row, i) => { if (row.region === 'timeline') out.push({ kind: 'row', index: i, start: layout.rowStart[i]! }); });
  }
  return out;
}

/**
 * Whether text the read projection hides or rewrites starts before `end` (read-view bytes, or stored bytes): a row whose
 * projection is not its stored bytes or that holds facts, the low-tier block, or (stored) a timeline the view omits.
 * Stored offsets past such text would measure it for a remote caller (get_page edit_target, edit_page `within`).
 */
export async function hiddenBefore(engine: BrainEngine, body: SegmentedBody, end: number, space: 'projected' | 'stored'): Promise<boolean> {
  const layout = await projectedLayout(engine, body);
  if (space === 'stored' && layout.timelineByte < 0 && body.bodyBytes > body.truthBytes && end > body.truthBytes) return true;
  return projectedUnits(body, layout).some(u => (space === 'projected' ? u.start : u.kind === 'row' ? body.rows[u.index]!.byteStart : u.storedStart) < end
    && (u.kind === 'text' ? u.text !== TIMELINE_SEPARATOR : layout.rowBytes[u.index] !== body.rows[u.index]!.byteLen || body.rows[u.index]!.hasFacts));
}

const unitBytes = (u: Unit, layout: ProjectedLayout) => u.kind === 'row' ? layout.rowBytes[u.index]! : Buffer.byteLength(u.text, 'utf8');

/** One segment of the trusted body: its row, its stored bytes and entry state, and its text after the withdrawal overlay in pieces. */
export interface TrustedSegment {
  row: BodyRow;
  raw: Uint8Array;
  entryState: unknown;
  pieces: ProjectedPiece[];
}

/** The longest run of segments joined across forced cuts that a pass holds as one string. */
export const BLOCK_MAX_BYTES = 8 * SEGMENT_MAX_BYTES;

/** `page_body_segmented`: a run a derived pass must read whole is over BLOCK_MAX_BYTES. */
export function blockTooLargeError(slug: string, sourceId: string, bytes: number) {
  return opError('page_body_segmented',
    `Page ${slug} (source ${sourceId}) has a ${bytes}-byte run a derived pass must read whole (a line or fence longer than a segment), over its ${BLOCK_MAX_BYTES}-byte bound. Nothing was derived from it.`,
    'Break the long line or fence into shorter lines and write the page again.');
}

/** `page_body_segmented`: a facts fence a withdrawal must read whole is longer than FENCE_MAX_BYTES (limits.ts). */
export function fenceTooLargeError(slug: string, sourceId: string, bytes: number) {
  return opError('page_body_segmented',
    `Page ${slug} (source ${sourceId}) has a facts fence of at least ${bytes} bytes, over the ${FENCE_MAX_BYTES}-byte bound a withdrawal reads a fence whole within. Nothing was read from it.`,
    'Split the facts fence into fences under the bound and write the page again.');
}

const MARKER_CARRY_MAX = Math.max(FACTS_FENCE_BEGIN.length, FACTS_FENCE_END.length) - 1;
/** The end of `text` that begins a facts marker without completing it: the bytes of a marker a forced cut splits. */
export function markerCarry(text: string): string {
  for (let k = Math.min(text.length, MARKER_CARRY_MAX); k > 0; k--) {
    const tail = text.slice(-k);
    if ((k < FACTS_FENCE_BEGIN.length && FACTS_FENCE_BEGIN.startsWith(tail)) || (k < FACTS_FENCE_END.length && FACTS_FENCE_END.startsWith(tail))) return tail;
  }
  return '';
}

/**
 * The stored text of the facts fence a forced cut splits that begins at stored byte `openStart` of `region` (its slots'
 * `open_start`): read from the live segments one at a time, from the one holding its opener (one that records no slot
 * of it when the cut fell inside the opener) through its end marker, or to the region's end for a fence that never
 * closes. A fence longer than FENCE_MAX_BYTES, counted from its begin marker, refuses before more is read.
 */
export async function readStoredFence(engine: Pick<BrainEngine, 'readManifest' | 'readSegmentTexts'>, page: { id: number; slug: string; sourceId: string }, region: BodyRegion, openStart: number): Promise<string> {
  const end = Buffer.from(FACTS_FENCE_END, 'utf8');
  const parts: Buffer[] = [];
  let bytes = 0;
  for (const row of await engine.readManifest(page.id, { region })) {
    if (row.byteStart + row.byteLen <= openStart) continue;
    const sha = Buffer.from(row.sha256).toString('hex');
    const text = (await engine.readSegmentTexts(page.sourceId, [row.sha256])).get(sha);
    if (!text) throw new Error(`Segment text ${sha} of page ${page.id} is missing.`);
    const part = Buffer.from(text).subarray(Math.max(0, openStart - row.byteStart));
    const seam = Buffer.concat([...parts.slice(-1).map(p => p.subarray(Math.max(0, p.length - end.length + 1))), part]);
    const at = seam.indexOf(end);
    const seamStart = bytes - (seam.length - part.length);
    parts.push(part);
    bytes += part.length;
    const fenceBytes = at === -1 ? bytes : seamStart + at + end.length;
    if (fenceBytes > FENCE_MAX_BYTES) throw fenceTooLargeError(page.slug, page.sourceId, fenceBytes);
    if (at !== -1) return Buffer.concat(parts).subarray(0, fenceBytes).toString('utf8');
  }
  return Buffer.concat(parts).toString('utf8');
}

/**
 * The trusted body of `rows` (rows of `body`, in order), one segment at a time: what readBodyRange's trusted stored read
 * cuts. Under a ledger that applies, a facts fence block cannot be overlaid in pieces, so the segments a block spans,
 * and a segment that ends inside a facts marker (a marker a forced cut splits) with the next one, are joined, overlaid
 * whole and yielded as one segment covering their rows, as withdrawal discovery and rewriteLiveSegmentedFences join
 * them. A fence longer than FENCE_MAX_BYTES, counted from its begin marker through its end marker, refuses with
 * `page_body_segmented`; a read that starts inside a block (a range after its start) refuses with `page_projection_stale`.
 * `stop` is called before each segment text is read; it throws to end the read there.
 */
export async function* trustedSegments(engine: BrainEngine, body: SegmentedBody, rows: readonly BodyRow[], stop?: () => void): AsyncIterable<TrustedSegment> {
  const emit = async (segs: LoadedSegment[]): Promise<TrustedSegment> => {
    const first = segs[0]!;
    const bytes = segs.length === 1 ? first.bytes : Buffer.concat(segs.map(seg => seg.bytes));
    const row = segs.length === 1 ? first.row : { ...first.row, byteLen: bytes.length, lineCount: segs.reduce((n, seg) => n + seg.row.lineCount, 0) };
    const raw = utf8.decode(bytes);
    return { row, raw: bytes, entryState: first.entryState, pieces: overlaidPieces(raw, await overlaidText(engine, body, { ...first, row }, raw, 'trusted')) };
  };
  if (!ledgerApplies(body.ledger)) {
    for await (const seg of loadSegments(engine, body, rows, stop)) yield await emit([seg]);
    return;
  }
  const bound = (bytes: number) => { if (bytes > FENCE_MAX_BYTES) throw fenceTooLargeError(body.slug, body.sourceId, bytes); };
  let held: LoadedSegment[] = [];
  // After the held segments: whether a fence block is open, the marker bytes they end with, and where the open block begins.
  let open = false, carry = '', openAt = 0;
  for await (const seg of loadSegments(engine, body, rows, stop)) {
    if (held.length && !open && !carry) { yield await emit(held); held = []; }
    if (!held.length) { open = readEntryState(seg.entryState)?.withdrawal.open === true; carry = ''; openAt = seg.row.byteStart; }
    held.push(seg);
    const text = carry + utf8.decode(seg.bytes);
    const base = seg.row.byteStart - Buffer.byteLength(carry, 'utf8');
    const { head, tail } = partialFenceRuns(text, open);
    if (open && head > 0) bound(base + Buffer.byteLength(text.slice(0, head), 'utf8') - openAt);
    const carried = open && head === 0;
    open = tail < text.length;
    if (open && !carried) openAt = base + Buffer.byteLength(text.slice(0, tail), 'utf8');
    if (open) bound(seg.row.byteStart + seg.row.byteLen - openAt);
    carry = markerCarry(text);
  }
  if (held.length) yield await emit(held);
}

export async function* readBodyRange(engine: BrainEngine, req: BodyRangeRequest): AsyncIterable<BodyRangePiece> {
  if (!(req.start >= 0) || !(req.end >= req.start) || !Number.isInteger(req.start) || (!Number.isInteger(req.end) && req.end !== Infinity)) {
    throw new Error(`readBodyRange: [${req.start}, ${req.end}) is not a byte range.`);
  }
  if (req.end === req.start) return;
  const body = req.body ?? await openSegmentedBody(engine, req);
  if (req.space === 'projected') return yield* projectedRange(engine, body, req);
  bodyRowWork.visits += body.rows.length;
  const indices = body.rows.flatMap((r, i) => (!req.region || r.region === req.region) && r.byteStart < req.end && r.byteStart + r.byteLen > req.start ? [i] : []);
  const rows = indices.map(i => body.rows[i]!);
  if (req.audience === 'trusted') {
    for await (const seg of trustedSegments(engine, body, rows)) {
      for (const piece of seg.pieces) {
        const cut = cutStored(piece, seg.row.byteStart, req.start, req.end);
        if (cut) yield { bytes: cut.bytes, storedStart: cut.storedStart, projectedStart: cut.storedStart };
      }
    }
    return;
  }
  const layout = await projectedLayout(engine, body);
  let k = 0;
  for await (const seg of loadSegments(engine, body, rows)) {
    let doc = layout.rowStart[indices[k++]!]!;
    if (doc < 0) continue;
    for (const piece of (await projectLoaded(engine, body, seg, layout.overlay)).pieces) {
      const cut = cutStored(piece, seg.row.byteStart, req.start, req.end);
      if (cut) yield { bytes: cut.bytes, storedStart: cut.storedStart, projectedStart: doc + cut.offset };
      doc += Buffer.byteLength(piece.text, 'utf8');
    }
  }
}

/** The part of a piece inside stored range [start, end): linear text byte for byte, anything else whole. */
function cutStored(piece: ProjectedPiece, base: number, start: number, end: number): { bytes: Buffer; storedStart: number; offset: number } | null {
  const s = base + piece.rawStart;
  const e = base + piece.rawEnd;
  if (!(e > start && s < end) && !(s === e && s >= start && s < end)) return null;
  const bytes = Buffer.from(piece.text, 'utf8');
  if (!piece.linear) return { bytes, storedStart: s, offset: 0 };
  const from = Math.max(start, s) - s;
  return { bytes: bytes.subarray(from, Math.min(end, e) - s), storedStart: s + from, offset: from };
}

async function* projectedRange(engine: BrainEngine, body: SegmentedBody, req: BodyRangeRequest): AsyncIterable<BodyRangePiece> {
  const layout = await projectedLayout(engine, body);
  const units = projectedUnits(body, layout);
  bodyRowWork.visits += units.length;
  const wanted = units.filter(u => {
    const region = u.kind === 'row' ? body.rows[u.index]!.region : u.region;
    return (!req.region || region === req.region) && u.start < req.end && u.start + unitBytes(u, layout) > req.start;
  });
  const rows = wanted.flatMap(u => u.kind === 'row' ? [body.rows[u.index]!] : []);
  const segments = loadSegments(engine, body, rows)[Symbol.asyncIterator]();
  for (const u of wanted) {
    if (u.kind === 'text') {
      const bytes = Buffer.from(u.text, 'utf8');
      const from = Math.max(req.start, u.start) - u.start;
      yield { bytes: bytes.subarray(from, Math.min(req.end, u.start + bytes.length) - u.start), storedStart: u.storedStart, projectedStart: u.start + from };
      continue;
    }
    const next = await segments.next();
    if (next.done) return;
    let doc = u.start;
    for (const piece of (await projectLoaded(engine, body, next.value, layout.overlay)).pieces) {
      const bytes = Buffer.from(piece.text, 'utf8');
      const from = Math.max(req.start, doc) - doc;
      const to = Math.min(req.end, doc + bytes.length) - doc;
      if (to > from) {
        const storedStart = next.value.row.byteStart + piece.rawStart + (piece.linear ? from : 0);
        yield { bytes: bytes.subarray(from, to), storedStart, projectedStart: doc + from };
      }
      doc += bytes.length;
      if (doc >= req.end) break;
    }
  }
}

/** The projected byte where 1-based line `line` starts (totalBytes past the last line); reads at most one segment. */
export async function projectedLineStart(engine: BrainEngine, body: SegmentedBody, line: number): Promise<number> {
  const layout = await projectedLayout(engine, body);
  let want = line - 1;
  if (want <= 0) return 0;
  for (const u of projectedUnits(body, layout)) {
    const count = u.kind === 'row' ? layout.rowNewlines[u.index]! : newlines(u.text);
    if (want > count) { want -= count; continue; }
    let text = '';
    if (u.kind === 'text') text = u.text;
    else for await (const piece of readBodyRange(engine, { pageId: body.pageId, space: 'projected', start: u.start, end: u.start + layout.rowBytes[u.index]!, audience: 'remote', body })) text += piece.bytes.toString('utf8');
    let at = -1;
    for (let k = 0; k < want; k++) { at = text.indexOf('\n', at + 1); if (at === -1) return u.start + Buffer.byteLength(text, 'utf8'); }
    return u.start + Buffer.byteLength(text.slice(0, at + 1), 'utf8');
  }
  return layout.totalBytes;
}

export interface BodyOutlineEntry { heading: string; level: number; byte_start: number; region: BodyRegion }

/**
 * The stored outline (headings of each segment's static projection outside code fences and protected regions, at
 * most OUTLINE_MAX_HEADINGS per segment), paged by entry index: reads the metadata of the rows the page covers, no
 * segment text. Offsets are read-projection bytes; line numbers are not stored.
 */
export async function projectedOutline(engine: BrainEngine, body: SegmentedBody, from: number, limit: number):
  Promise<{ entries: BodyOutlineEntry[]; total_entries: number; from_index: number; next_from_index?: number }> {
  const layout = await projectedLayout(engine, body);
  const visible = body.rows.map((row, i) => ({ row, i })).filter(({ i }) => layout.rowStart[i]! >= 0)
    .sort((a, b) => layout.rowStart[a.i]! - layout.rowStart[b.i]!);
  const total = visible.reduce((n, v) => n + v.row.outlineCount, 0);
  const entries: BodyOutlineEntry[] = [];
  const needed: typeof visible = [];
  let index = 0;
  for (const v of visible) {
    if (index + v.row.outlineCount > from && index < from + limit) needed.push(v);
    index += v.row.outlineCount;
  }
  const meta = needed.length ? new Map((await engine.executeRaw<Record<string, unknown>>(`SELECT occurrence_id, facts_slots, outline FROM page_body_manifest
    WHERE page_id=$1 AND occurrence_id=ANY($2::bigint[]) AND staging_id IS NULL AND gen_from<=$3 AND (gen_to IS NULL OR gen_to>$3)`,
  [body.pageId, needed.map(v => v.row.occurrenceId), body.generation])).map(m => [Number(m.occurrence_id), m])) : new Map();
  index = 0;
  let next = 0;
  for (const v of visible) {
    const m = needed.includes(v) ? meta.get(v.row.occurrenceId) : undefined;
    if (!m) { index += v.row.outlineCount; continue; }
    const slots = (toJson(m.facts_slots) ?? []) as FactsSlot[];
    const facts = layout.rowFacts[v.i] ?? [];
    for (const h of (toJson(m.outline) ?? []) as OutlineHeading[]) {
      if (index >= from && entries.length < limit) {
        let byte = layout.rowStart[v.i]! + h.proj_byte;
        slots.forEach((slot, k) => { if (slot.proj_byte <= h.proj_byte) byte += facts[k] ?? 0; });
        entries.push({ heading: h.text, level: h.level, byte_start: byte, region: v.row.region });
        next = index + 1;
      }
      index++;
    }
  }
  return { entries, total_entries: total, from_index: from, ...(next < total && entries.length ? { next_from_index: next } : {}) };
}