Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
29.8 KB
Raw Blame
/**
 * Segmented page ingest (Wave B, spec 3.1-3.3): turns a page body (the stored
 * stream of its truth region, then its timeline region) into staged segment
 * texts, a diff against the live manifest and staged manifest and chunk rows,
 * all outside the publication transaction. Memory is O(segment): the regions
 * arrive as byte streams, texts are flushed in STAGE_BATCH_BYTES statements and
 * chunking reads the staged texts back one batch at a time.
 *
 * Diff (spec 3.3 step 3): a new segment whose `(sha256, entry_state)` matches a
 * live occurrence keeps that occurrence id; it keeps its row too when every
 * coordinate (seq, offsets, hash midstate) is unchanged, else it gets a new
 * row with the same id (shifted). Other segments get fresh ids. Live
 * occurrences left unmatched retire. `content_hash` resumes from the hash
 * midstate of the last row of the unchanged prefix (1.4).
 */
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import { stableJson } from '../persistence/digest.ts';
import { faultPoint } from '../persistence/fault-points.ts';
import { withCoordinatedWrite } from '../persistence/context.ts';
import { readManagedMode } from '../persistence/managed-mode.ts';
import { maintenanceAttribution, principalAttribution } from '../persistence/attribution.ts';
import type { Principal } from '../persistence/model.ts';
import { stepFenceWalk, type FenceWalk } from '../fence-scan.ts';
import { BODY_LEAD_BYTES, BODY_STAGING_HARD_MS, BODY_STAGING_IDLE_MS, PROJECTION_LOCK_HOLD_BYTES, SEGMENTED_CREATE_SEAL_MAX_BYTES, stageBatchBytes } from './limits.ts';
import type { BodyRegion, BodyStagingKind, BodyStorage, JsonObject, ManifestRow, ManifestRowInput, SegmentChunkInput,
  SegmentedBodyColumns, SegmentTextInput } from './types.ts';
import { regionTrust, segmentStream, SEGMENTER_VERSION, type StoredRegion } from './segmenter.ts';
import { initialEntryState, readEntryState, timelineEntryState, type EntryState } from './entry-state.ts';
import { ContentHashStream } from './content-hash-stream.ts';
import { prepareSegmentChunks } from './segment-chunks.ts';
import { reuseSegmentChunkVectors } from '../import-chunk-reuse.ts';
import { newFenceChunkBudget, type FenceChunkBudget } from '../markdown-chunks.ts';
import { loadFenceChunkOverlay } from '../eligibility/fence-overlay.ts';
import { withdrawalFenceBlocks } from '../facts/withdrawal-overlay.ts';
import { overlaySegmentText, readBodyLedger, type BodyLedger } from './segment-ledger.ts';
import { readTiledManifest } from './body-tiling.ts';
import { hasStaleChunks } from './projection.ts';

const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });

/** The live page a segmented write replaces (null: create). */
export interface SegmentedLivePage { id: number; generation: number; storage: BodyStorage; title: string; type: string; flags: JsonObject | null }

/** Who stages, for the `page_body_stagings` row (a GC root while open). */
export interface SegmentedStagingOwner { kind: BodyStagingKind; principalKind: string; principalId: string; baseRevision: string | null }

export interface SegmentedBodyInput {
  sourceId: string;
  slug: string;
  live: SegmentedLivePage | null;
  title: string;
  type: string;
  frontmatter: JsonObject;
  tags: readonly string[];
  /** The stored stream of each region (sanitized, valid UTF-8). */
  truth: AsyncIterable<Uint8Array>;
  timeline: AsyncIterable<Uint8Array>;
  /** The timeline region is not empty (the truth then ends at `region_end`, not `eof`). */
  hasTimeline: boolean;
  /** Page-global facts known at EOF (pass 1): `fence_trust`, `timeline_split`, ... */
  flags: JsonObject;
  /** Body bytes as declared by the source (exact for strings, the file size for streams). */
  declaredBytes: number;
  owner: SegmentedStagingOwner;
  /** Chunking inputs for a replace (a create chunks after publication). */
  chunking: SegmentChunkingContext;
}

export interface SegmentChunkingContext {
  /** Quarantined pages keep no chunks, as inline pages. */
  quarantined: boolean;
  corpusGeneration: string | null;
  tier: 'title' | 'none';
}

export interface PreparedSegmentedBody {
  stagingId: string;
  /** The live body generation the diff read (0 for create). */
  expectedGeneration: number;
  newGeneration: number;
  /** Create only: every manifest row, written in the publication transaction once the page row exists. */
  createRows: ManifestRowInput[] | null;
  /**
   * Create within SEGMENTED_CREATE_SEAL_MAX_BYTES (or quarantined): the chunk rows, prepared for `slug` and inserted by
   * the publication, which then seals. Null: the create publishes unsealed and its projection runs after commit.
   */
  createChunks: { slug: string; chunks: SegmentChunkInput[] } | null;
  closeOccurrences: number[];
  retireChunkOccurrences: number[];
  /** Occurrences of the new generation that have no chunk rows after staging (a create without `createChunks`: all, chunked after publication). */
  unchunked: number;
  /**
   * The live generation stores chunk rows it does not hold (an earlier write's deferred retirement), so the publication
   * leaves the page unsealed. Read here, not in the publication transaction: the probe walks every occurrence of the page.
   */
  priorStale: boolean;
  body: SegmentedBodyColumns;
  /** No manifest row changed: the write changes metadata only. */
  bodyUnchanged: boolean;
  rows: { texts: number; manifest: number; chunks: number };
  /** `chunk_index` of every chunk row this preparation staged; the flip drops any other row staged under `newGeneration` (a dead preparation's). */
  stagedChunkIndexes: number[];
}

/**
 * Runs writes to the guarded segment tables (texts, manifest rows, sweeps) of
 * `sourceId` outside a publication transaction. On a managed brain they run in
 * a coordinated write for that source, as the writer guard requires
 * (writer-guard-schema.ts); the caller is the source's owner preparing or
 * collecting its own write (`principal`, else the local maintenance writer).
 * On any other brain they run directly.
 */
export async function segmentWriteScope(engine: BrainEngine, sourceId: string, principal?: Principal):
  Promise<<T>(fn: (exec: BrainEngine) => Promise<T>) => Promise<T>> {
  if (!await readManagedMode(engine)) return fn => fn(engine);
  const attribution = principal ? principalAttribution(principal) : await maintenanceAttribution(engine);
  return fn => engine.transaction(tx => withCoordinatedWrite(tx, [sourceId], () => fn(tx), attribution));
}

/** A region of a string as a byte stream, sliced on code-point boundaries. */
export async function* stringRegion(text: string, sliceChars = 256 * 1024): AsyncIterable<Uint8Array> {
  const encoder = new TextEncoder();
  for (let at = 0; at < text.length;) {
    let end = Math.min(text.length, at + sliceChars);
    const last = text.charCodeAt(end - 1);
    if (end < text.length && last >= 0xd800 && last <= 0xdbff) end--;
    yield encoder.encode(text.slice(at, end));
    at = end;
  }
}

/** The longest UTF-8 prefix of `bytes` within `max` bytes that ends on a code-point boundary, decoded. */
export function utf8Prefix(bytes: Uint8Array, max: number): string {
  let end = Math.min(bytes.length, max);
  if (end < bytes.length) while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end--;
  return utf8.decode(bytes.subarray(0, end));
}

/** scanMarkdownCode's trust of a stored region: 'void' when a fence is left open or a longer opener sits inside one. */
export function regionFenceTrust(text: string): 'trusted' | 'void' {
  const walk: FenceWalk = { opener: null, mispaired: false };
  for (let at = 0; at <= text.length;) {
    let end = text.indexOf('\n', at);
    if (end < 0) end = text.length;
    for (const line of text.slice(at, end).split('\r')) stepFenceWalk(walk, line);
    at = end + 1;
  }
  return walk.opener || walk.mispaired ? 'void' : 'trusted';
}

/** `body_flags` of a page whose stored regions are strings (pass 1's fence trust, for the in-memory path). */
export function stringBodyFlags(truth: string, timeline: string): JsonObject {
  return { fence_trust: regionFenceTrust(truth), timeline_fence_trust: regionFenceTrust(timeline) };
}

const hex = (bytes: Uint8Array) => Buffer.from(bytes).toString('hex');
const sameBytes = (a: Uint8Array, b: Uint8Array) => Buffer.from(a).equals(Buffer.from(b));

/** An occurrence is its bytes, entry state and the trust its stored regions were computed under: a trust flip rechunks the region. */
export const occurrenceKey = (sha256: Uint8Array, entryState: unknown, trust: 'trusted' | 'void') => `${hex(sha256)}|${stableJson(entryState)}|${trust}`;

/** Every coordinate of a live row equals the row the new segment would get, so the row stays. */
export function rowUnchanged(live: ManifestRow, next: ManifestRowInput): boolean {
  return live.seq === next.seq && live.region === next.region && Number(live.byteStart) === next.byteStart
    && live.byteLen === next.byteLen && Number(live.lineStart) === next.lineStart && live.cutKind === next.cutKind
    && sameBytes(live.hashState, next.hashState) && stableJson(live.protectedRegions) === stableJson(next.protectedRegions)
    && stableJson(live.factsSlots) === stableJson(next.factsSlots) && stableJson(live.outline) === stableJson(next.outline)
    && live.staticProjBytes === next.staticProjBytes && live.staticProjLines === next.staticProjLines
    && live.dateMin === next.dateMin && live.dateMax === next.dateMax;
}

async function sourceIncarnation(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string): Promise<string> {
  const [row] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [sourceId]);
  if (!row) throw new Error(`Source ${sourceId} not found.`);
  return row.incarnation;
}

/** Open the staging row every segmented write records (kind, principal, base); returns its id. */
export async function openSegmentedStaging(engine: BrainEngine, input: Pick<SegmentedBodyInput, 'sourceId' | 'slug' | 'live' | 'declaredBytes' | 'owner'>): Promise<string> {
  const id = randomUUID();
  const now = Date.now();
  await engine.openBodyStaging({
    id, kind: input.owner.kind, principalKind: input.owner.principalKind, principalId: input.owner.principalId, clientRequestId: randomUUID(),
    sourceId: input.sourceId, sourceIncarnation: await sourceIncarnation(engine, input.sourceId), slug: input.slug,
    pageId: input.live?.id ?? null, mode: input.live ? 'replace' : 'create', baseRevision: input.owner.baseRevision,
    baseGeneration: input.live?.generation ?? null, declaredBytes: input.declaredBytes, declaredSha256: null, reservedBytes: 0,
    idleExpiresAt: new Date(now + BODY_STAGING_IDLE_MS), hardExpiresAt: new Date(now + BODY_STAGING_HARD_MS),
  });
  return id;
}

/**
 * A full collection after each staging batch. A batch leaves garbage several times its payload (driver parameter
 * text, protocol buffers, decoded segment texts, chunk strings), and JSC sizes its collection threshold to the heap,
 * which PGLite's WASM memory makes large, so batches piled up uncollected: a 100 MiB import peaked ~1.1 GB on PGLite
 * without this and ~0.75 GB with it (scripts/bench-large-pages.ts). One collection of a small live heap per 4 MiB.
 */
function collectBatchGarbage(): void {
  Bun.gc(true);
}

/**
 * Pass 2 for one write: segment both regions, hash, stage texts, diff against
 * the live manifest, stage the changed manifest rows and (replace) the chunks
 * of every occurrence the new generation lacks. A failure aborts the staging.
 */
export async function prepareSegmentedBody(engine: BrainEngine, input: SegmentedBodyInput): Promise<PreparedSegmentedBody> {
  const stagingId = await openSegmentedStaging(engine, input);
  try {
    const prepared = await stageBody(engine, input, stagingId);
    await engine.setBodyStagingState(stagingId, 'prepared');
    return prepared;
  } catch (error) {
    await engine.setBodyStagingState(stagingId, 'aborted').catch(() => undefined);
    throw error;
  }
}

async function stageBody(engine: BrainEngine, input: SegmentedBodyInput, stagingId: string): Promise<PreparedSegmentedBody> {
  const live = input.live?.storage === 'segmented' ? input.live : null;
  const liveRows = live ? await engine.readManifest(live.id) : [];
  const byKey = new Map<string, ManifestRow[]>();
  for (const row of liveRows) {
    const key = occurrenceKey(row.sha256, row.entryState, regionTrust(live!.flags, row.region));
    const same = byKey.get(key);
    if (same) same.push(row); else byKey.set(key, [row]);
  }
  const writes = await segmentWriteScope(engine, input.sourceId, { kind: input.owner.principalKind as Principal['kind'], id: input.owner.principalId });
  const batchBytes = stageBatchBytes(engine.kind);
  const expectedGeneration = input.live?.generation ?? 0;
  const newGeneration = expectedGeneration + 1;
  const pageId = input.live?.id ?? 0;
  let prefixIntact = !!live && live.title === input.title && live.type === input.type;
  let hasher: ContentHashStream | null = null;
  const resume = (index: number) => index === 0 ? ContentHashStream.begin({ title: input.title, type: input.type })
    : ContentHashStream.fromState(liveRows[index - 1]!.hashState);
  const pending: SegmentTextInput[] = [];
  let pendingBytes = 0;
  const flushTexts = async () => {
    if (!pending.length) return;
    const batch = pending.splice(0);
    await writes(exec => exec.stageSegmentTexts(input.sourceId, batch, { stagingId }));
    collectBatchGarbage();
    pendingBytes = 0;
    if (engine.kind === 'pglite') await new Promise(resolve => setImmediate(resolve));
  };
  const rows: ManifestRowInput[] = [];
  const used = new Set<ManifestRow>();
  const matched: Array<ManifestRow | null> = [];
  let texts = 0, byteStart = 0, lineStart = 0, truthBytes = 0;
  const leads: Record<BodyRegion, string> = { truth: '', timeline: '' };
  let hasProtected = false;
  let truthExit: EntryState = initialEntryState({ region: 'truth' });
  for (const region of ['truth', 'timeline'] as const) {
    const stream = region === 'truth' ? input.truth : input.timeline;
    const entry = region === 'truth' ? truthExit : timelineEntryState(truthExit);
    const end = region === 'truth' && input.hasTimeline ? 'region_end' : 'eof';
    let first = true;
    for await (const segment of segmentStream(stream, entry, { byteStart, lineStart, fenceTrust: regionTrust(input.flags, region), end })) {
      const seq = rows.length;
      if (first) { leads[region] = utf8Prefix(segment.bytes, BODY_LEAD_BYTES); first = false; }
      if (region === 'truth') truthExit = segment.exitState;
      const text = utf8.decode(segment.bytes);
      const key = occurrenceKey(segment.sha256, segment.entryState, segment.fenceTrust);
      const prior = liveRows[seq];
      const prefixRow = prefixIntact && prior && prior.region === region && Number(prior.byteStart) === byteStart
        && occurrenceKey(prior.sha256, prior.entryState, regionTrust(live!.flags, prior.region)) === key ? prior : null;
      let hashState: Uint8Array;
      if (prefixRow) hashState = prefixRow.hashState;
      else {
        hasher ??= resume(prefixIntact ? seq : 0);
        prefixIntact = false;
        if (region === 'truth') hasher.pushTruth(text); else hasher.pushTimeline(text);
        hashState = hasher.exportState();
      }
      const row: ManifestRowInput = {
        pageId, occurrenceId: 0, genFrom: newGeneration, seq, region, sourceId: input.sourceId, sha256: segment.sha256,
        byteStart, byteLen: segment.bytes.length, lineStart, lineCount: segment.lineCount, cutKind: segment.cutKind,
        entryState: segment.entryState as unknown as JsonObject, protectedRegions: segment.protectedRegions, factsSlots: segment.factsSlots,
        staticProjBytes: segment.staticProjBytes, staticProjLines: segment.staticProjLines, outline: segment.outline,
        dateMin: segment.dateMin, dateMax: segment.dateMax, hashState, stagingId,
      };
      const candidates = byKey.get(key)?.filter(candidate => !used.has(candidate)) ?? [];
      const match = candidates.find(candidate => candidate.seq === seq) ?? candidates[0] ?? null;
      if (match) used.add(match);
      matched.push(match);
      if (!match) {
        pending.push({ sha256: segment.sha256, body: segment.bytes, lineCount: segment.lineCount });
        pendingBytes += segment.bytes.length;
        texts++;
        if (pendingBytes >= batchBytes) await flushTexts();
      }
      if (segment.protectedRegions.length) hasProtected = true;
      rows.push(row);
      byteStart += segment.bytes.length;
      lineStart += segment.lineCount;
      if (region === 'truth') truthBytes += segment.bytes.length;
    }
  }
  await flushTexts();
  await faultPoint('segmented:texts_staged', { sourceId: input.sourceId });
  const contentHash = (hasher ?? resume(prefixIntact ? rows.length : 0)).finish({ frontmatter: input.frontmatter, tags: [...input.tags] });

  const fresh = rows.filter((_, i) => !matched[i]);
  const ids = fresh.length ? await engine.allocateBodyOccurrenceIds(fresh.length) : [];
  fresh.forEach((row, i) => { row.occurrenceId = ids[i]!; });
  const changed: ManifestRowInput[] = [];
  const closeOccurrences: number[] = [];
  rows.forEach((row, i) => {
    const match = matched[i];
    if (!match) { changed.push(row); return; }
    row.occurrenceId = Number(match.occurrenceId);
    if (input.live && rowUnchanged(match, row)) return;
    changed.push(row);
    closeOccurrences.push(row.occurrenceId);
  });
  const retireChunkOccurrences = liveRows.filter(row => !used.has(row)).map(row => Number(row.occurrenceId));
  closeOccurrences.push(...retireChunkOccurrences);
  const body: SegmentedBodyColumns = {
    content_hash: contentHash, body_bytes: byteStart, truth_bytes: truthBytes, body_lines: lineStart, segment_count: rows.length,
    segmenter_version: SEGMENTER_VERSION, truth_lead: leads.truth, timeline_lead: leads.timeline,
    body_flags: { ...input.flags, has_protected: hasProtected },
  };
  if (!input.live) {
    const createChunks = input.chunking.quarantined || byteStart <= SEGMENTED_CREATE_SEAL_MAX_BYTES ? { slug: input.slug, chunks: [] as SegmentChunkInput[] } : null;
    if (createChunks) await stageOccurrenceChunks(engine, { pageId: 0, sourceId: input.sourceId, slug: input.slug, generation: newGeneration, occurrences: rows,
      title: input.title, frontmatter: input.frontmatter, flags: body.body_flags, chunking: input.chunking, retiredOccurrences: [], hold: createChunks.chunks });
    return { stagingId, expectedGeneration, newGeneration, createRows: rows, createChunks, closeOccurrences: [], retireChunkOccurrences: [],
      unchunked: createChunks ? 0 : rows.length, priorStale: false, body, bodyUnchanged: false, rows: { texts, manifest: rows.length, chunks: createChunks?.chunks.length ?? 0 }, stagedChunkIndexes: [] };
  }
  if (changed.length) await writes(exec => exec.stageManifestRows(changed));
  await faultPoint('segmented:manifest_staged', { sourceId: input.sourceId });
  const chunkable = await occurrencesWithoutChunks(engine, input.live.id, expectedGeneration, rows);
  const chunks = await stageOccurrenceChunks(engine, { pageId: input.live.id, sourceId: input.sourceId, slug: input.slug, generation: newGeneration,
    occurrences: chunkable, title: input.title, frontmatter: input.frontmatter, flags: body.body_flags, chunking: input.chunking,
    retiredOccurrences: retireChunkOccurrences, truthLead: body.truth_lead });
  await faultPoint('segmented:chunks_staged', { sourceId: input.sourceId });
  const kept = new Set(rows.filter((_, i) => matched[i]).map(row => row.occurrenceId));
  retireChunkOccurrences.push(...chunkable.filter(row => kept.has(row.occurrenceId)).map(row => row.occurrenceId));
  const priorStale = !!live && await hasStaleChunks(engine, live.id, expectedGeneration, liveRows.map(row => Number(row.occurrenceId)));
  return { stagingId, expectedGeneration, newGeneration, createRows: null, createChunks: null, closeOccurrences, retireChunkOccurrences,
    unchunked: input.chunking.quarantined ? 0 : chunkable.length - chunks.occurrences, priorStale, body,
    bodyUnchanged: changed.length === 0 && retireChunkOccurrences.length === 0, rows: { texts, manifest: changed.length, chunks: chunks.rows },
    stagedChunkIndexes: chunks.indexes };
}

/**
 * The rows of `rows` whose occurrence has no live chunk row at `generation`: new
 * occurrences, and kept ones a crashed or deferred projection never chunked. One
 * index probe per occurrence (content_chunks_occurrence_generation_idx), never a
 * scan of the page's chunk rows.
 */
export async function occurrencesWithoutChunks(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number, generation: number,
  rows: readonly ManifestRowInput[]): Promise<ManifestRowInput[]> {
  if (!rows.length) return [];
  const chunked = new Set((await engine.executeRaw<{ occurrence_id: number }>(`SELECT o AS occurrence_id FROM unnest($2::bigint[]) AS o
    WHERE EXISTS (SELECT 1 FROM content_chunks c
      -- chunk-visibility-ok: write path; the generation predicate is explicit, at the generation the caller diffed against
      WHERE c.page_id=$1 AND c.occurrence_id=o AND c.body_generation<=$3::bigint)`,
  [pageId, rows.map(row => row.occurrenceId), generation])).map(row => Number(row.occurrence_id)));
  return rows.filter(row => !chunked.has(row.occurrenceId));
}

/**
 * @internal Work stageOccurrenceChunks does, for the projection tests: segment text batches chunked (`batches`) and
 * chunk rows prepared (`chunks`), whoever stages them (a replace's preparation, a create's, or a projection), and the
 * most segment bytes one projection transaction staged under the page lock (`heldBytes`, PROJECTION_LOCK_HOLD_BYTES).
 */
export const segmentChunkWork = { batches: 0, chunks: 0, heldBytes: 0 };

/**
 * Chunk occurrences one text batch at a time (texts read back from
 * `page_segment_texts`), reuse vectors from the retired occurrences' rows, and
 * stage the chunk rows under `generation`. Shared by a replace's preparation
 * and the post-publication projection.
 */
export async function stageOccurrenceChunks(engine: BrainEngine, input: { pageId: number; sourceId: string; slug: string; generation: number;
  occurrences: readonly ManifestRowInput[]; title: string; frontmatter: JsonObject; flags: JsonObject; chunking: SegmentChunkingContext;
  retiredOccurrences: readonly number[];
  /** The truth_lead the write stores, for the vector reuse onto a page stored under per-chunk synopsis; omitted, the stored one. */
  truthLead?: string;
  /** Projection: each batch inserts under the page guard, at most PROJECTION_LOCK_HOLD_BYTES of occurrences per transaction, after re-reading which
   *  occurrences still lack rows, so two projections never both chunk one; a page whose generation moved past `generation` stops the projection before it inserts. */
  guarded?: boolean;
  /** A create's preparation: the chunks are collected here (its page row does not exist yet) and inserted by the publication. */
  hold?: SegmentChunkInput[];
  /** Guarded: runs in each insert transaction before its insert (the projection renews its claim on the page's queue row);
   *  false when the caller lost its claim: that transaction inserts nothing and the staging stops. */
  renew?: (tx: BrainEngine) => Promise<boolean>;
  /** The page's fence-chunk budget when one page is chunked over several calls (the re-chunk, page-body/rechunk.ts); default a new one. */
  fenceBudget?: FenceChunkBudget }): Promise<{ rows: number; occurrences: number; indexes: number[] }> {
  if (input.chunking.quarantined || !input.occurrences.length) return { rows: 0, occurrences: 0, indexes: [] };
  const batchBytes = stageBatchBytes(engine.kind);
  const fenceBudget = input.fenceBudget ?? newFenceChunkBudget();
  let ledger: BodyLedger | undefined;
  let rows = 0, done = 0;
  const indexes: number[] = [];
  for (let start = 0; start < input.occurrences.length;) {
    let end = start, bytes = 0;
    while (end < input.occurrences.length && (end === start || bytes + input.occurrences[end]!.byteLen <= batchBytes)) bytes += input.occurrences[end++]!.byteLen;
    const batch = input.occurrences.slice(start, end);
    start = end;
    if (input.guarded) await faultPoint('segmented:projection_batch', { sourceId: input.sourceId, slug: input.slug });
    const texts = await engine.readSegmentTexts(input.sourceId, batch.map(row => row.sha256));
    ledger ??= await readBodyLedger(engine, input.sourceId, input.slug);
    const segments = [];
    for (const row of batch) {
      const bytes = texts.get(hex(row.sha256));
      if (bytes === undefined) throw new Error(`Segment text ${hex(row.sha256)} of page ${input.pageId} is missing.`);
      const entryState = readEntryState(row.entryState);
      if (!entryState) throw new Error(`Manifest row ${row.occurrenceId} of page ${input.pageId} has an entry state of another version; resegment the page.`);
      const fenceTrust = regionTrust(input.flags, row.region);
      const text = await overlaySegmentText(engine, ledger, { sourceId: input.sourceId, byteStart: row.byteStart, entryState }, utf8.decode(bytes), 'read');
      segments.push({ row, fenceTrust, text, segment: { region: row.region, bytes, byteStart: row.byteStart, entryState, fenceTrust, protectedRegions: row.protectedRegions as StoredRegion[] } });
    }
    const factsTexts = segments.flatMap(({ text }) => text.includes('gbrain:facts:begin') ? withdrawalFenceBlocks(text).map(block => text.slice(block.start, block.end)) : []);
    const overlay = factsTexts.length ? await loadFenceChunkOverlay(engine, { sourceId: input.sourceId, slug: input.slug, compiled_truth: factsTexts.join('\n') }) : undefined;
    const chunks: SegmentChunkInput[] = [];
    for (const { row, fenceTrust, segment, text } of segments) {
      for (const chunk of await prepareSegmentChunks(segment, { fenceTrust, frontmatter: input.frontmatter, overlay, fenceBudget, text })) {
        chunks.push({ ...chunk, occurrence_id: row.occurrenceId });
      }
    }
    if (chunks.length && input.retiredOccurrences.length) await reuseSegmentChunkVectors(engine, { pageId: input.pageId,
      retiredOccurrences: input.retiredOccurrences, chunks, title: input.title, corpusGeneration: input.chunking.corpusGeneration, tier: input.chunking.tier,
      truthLead: input.truthLead });
    segmentChunkWork.batches++;
    segmentChunkWork.chunks += chunks.length;
    if (input.hold) input.hold.push(...chunks);
    else if (chunks.length && !input.guarded) indexes.push(...await engine.stageSegmentChunks(input.pageId, input.generation, chunks));
    let moved = false, lost = false;
    if (input.guarded) {
      const byOccurrence = new Map<number, SegmentChunkInput[]>();
      for (const chunk of chunks) {
        const same = byOccurrence.get(chunk.occurrence_id);
        if (same) same.push(chunk); else byOccurrence.set(chunk.occurrence_id, [chunk]);
      }
      for (let at = 0; at < batch.length && !moved && !lost;) {
        let stop = at, held = 0;
        while (stop < batch.length && (stop === at || held + batch[stop]!.byteLen <= PROJECTION_LOCK_HOLD_BYTES)) held += batch[stop++]!.byteLen;
        const part = batch.slice(at, stop).filter(row => byOccurrence.has(row.occurrenceId));
        at = stop;
        if (part.length) rows += await engine.transaction(async tx => {
          await tx.lockPageKeys([{ sourceId: input.sourceId, slug: input.slug }]);
          await faultPoint('segmented:projection_insert', { sourceId: input.sourceId, slug: input.slug });
          const [page] = await tx.executeRaw<{ body_generation: number | string }>('SELECT body_generation FROM pages WHERE id=$1', [input.pageId]);
          if (Number(page?.body_generation) !== input.generation) { moved = true; return 0; }
          if (input.renew && !await input.renew(tx)) { lost = true; return 0; }
          const open = await occurrencesWithoutChunks(tx, input.pageId, input.generation, part);
          const still = open.flatMap(row => byOccurrence.get(row.occurrenceId)!);
          if (still.length) await tx.stageSegmentChunks(input.pageId, input.generation, still);
          segmentChunkWork.heldBytes = Math.max(segmentChunkWork.heldBytes, open.reduce((sum, row) => sum + row.byteLen, 0));
          return still.length;
        });
      }
    } else rows += chunks.length;
    if (moved || lost) break;
    done += batch.length;
    collectBatchGarbage();
    if (engine.kind === 'pglite') await new Promise(resolve => setImmediate(resolve));
  }
  return { rows, occurrences: done, indexes };
}

/** The stored body of `pageId` at `generation` (default live), region by region, one segment at a time. */
export async function* readSegmentedBody(engine: BrainEngine, pageId: number, sourceId: string,
  opts: { generation?: number; bodyBytes?: number } = {}): AsyncIterable<{ region: BodyRegion; bytes: Uint8Array }> {
  const { rows } = await readTiledManifest(engine, pageId, opts.generation, opts.bodyBytes);
  const batchBytes = stageBatchBytes(engine.kind);
  for (let start = 0; start < rows.length;) {
    let end = start, bytes = 0;
    while (end < rows.length && (end === start || bytes + rows[end]!.byteLen <= batchBytes)) bytes += rows[end++]!.byteLen;
    const batch = rows.slice(start, end);
    start = end;
    const texts = await engine.readSegmentTexts(sourceId, batch.map(row => row.sha256));
    for (const row of batch) {
      const text = texts.get(hex(row.sha256));
      if (text === undefined) throw new Error(`Segment text ${hex(row.sha256)} of page ${pageId} is missing.`);
      yield { region: row.region, bytes: text };
    }
  }
}

/**
 * The whole body of a segmented page as strings, at `opts.generation` (default live; `opts.bodyBytes` its known size).
 * Only for pages within the single-string ceiling (callers check `body_bytes`).
 */
export async function materializeSegmentedBody(engine: BrainEngine, pageId: number, sourceId: string,
  opts: { generation?: number; bodyBytes?: number } = {}): Promise<{ compiled_truth: string; timeline: string }> {
  const parts: Record<BodyRegion, string[]> = { truth: [], timeline: [] };
  for await (const part of readSegmentedBody(engine, pageId, sourceId, opts)) parts[part.region].push(utf8.decode(part.bytes));
  return { compiled_truth: parts.truth.join(''), timeline: parts.timeline.join('') };
}