Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.6 KB
Raw Blame
import type { BrainEngine } from './engine.ts';
import { overlayWithdrawalBody } from './facts/withdrawal-overlay.ts';
import type { PageWithdrawal } from './page-state/types.ts';
import { rowToPage } from './utils.ts';
import { EXPORT_PAYLOAD_LIMIT } from './export-stage.ts';
import { isSegmentedPage } from './page-body/page-shape.ts';
import { openSegmentedBody, readBodyRange, type BodyLedger } from './page-body/read-range.ts';
import { TIMELINE_SEPARATOR } from './page-window/document.ts';
import type { Page } from './types.ts';
import { serializeMarkdown, slugStampedFrontmatter } from './markdown.ts';

const EXPORT_COLLECT_BYTES = 1024 * 1024;

/** Withdrawals are entity-scoped: `subject` is the entity slug, '*' applies to every page. */
export type ExportWithdrawal = PageWithdrawal & { subject: string };

export async function readExportWithdrawals(tx: BrainEngine, sourceId: string): Promise<ExportWithdrawal[]> {
  const [size] = await tx.executeRaw<{ bytes: number }>(`SELECT COALESCE(sum(octet_length(
    jsonb_build_object('visibility',visibility,'fact_hash',fact_hash,'withdrawn_at',withdrawn_at,'subject',subject)::text)+1),0)+2 AS bytes
    FROM fact_withdrawals WHERE source_id=$1`, [sourceId]);
  if (Number(size.bytes) > EXPORT_PAYLOAD_LIMIT) throw new Error('Export withdrawal ledger capacity exceeded. No destination output was published.');
  // Purge tombstones ride the same overlay and drop their rows from the export.
  return tx.executeRaw<ExportWithdrawal>(`SELECT visibility,fact_hash,withdrawn_at,subject,false AS purged
    FROM fact_withdrawals WHERE source_id=$1
    UNION ALL SELECT visibility,fact_hash,purged_at,subject,true FROM fact_purges WHERE source_id=$1
    ORDER BY visibility,fact_hash,subject`, [sourceId]);
}

export type ExportSnapshot = { page: Page; tags: string[] };

export async function readExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot> {
  return (await readExportPages(tx, [key], withdrawals))[0]!;
}

/**
 * The export snapshots of `keys`, in key order, in at most two statements however many keys there are (#7617: a
 * per-page read made a large source's conversion plan one round trip per page): the rows with their tags, then one
 * fence normalization for every inline page whose withdrawal overlay applies, by overlayCanonicalBodies' rules and SQL.
 * Rows are looked up by primary key (#7618: a join on the keys let the planner scan the whole source for each key, so a
 * per-page export went quadratic), and each row must still carry its key's source and slug.
 */
export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id: string; source_id: string; slug: string }>, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot[]> {
  if (!keys.length) return [];
  const rows = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (
    SELECT * FROM pages WHERE id = ANY($1::bigint[])
  ) SELECT p.*,
    COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags
    FROM export_page p WHERE p.deleted_at IS NULL`,
  [keys.map(key => key.id)]);
  const byId = new Map(rows.map(row => [String(row.id), row]));
  const snapshots = keys.map(key => {
    const row = byId.get(String(key.id));
    if (!row || row.source_id !== key.source_id || row.slug !== key.slug) throw new Error('Export snapshot unexpectedly lost a selected page.');
    return { page: rowToPage(row), tags: row.export_tags as string[] };
  });
  const fenced = snapshots.flatMap(({ page }) => {
    if (isSegmentedPage(page) || !(page.compiled_truth.includes('gbrain:facts:begin') || page.timeline.includes('gbrain:facts:begin'))) return [];
    const applicable = applicableWithdrawals(page.slug, withdrawals);
    return applicable.length ? [{ page, applicable }] : [];
  });
  if (!fenced.length) return snapshots;
  const normalized = await tx.executeRaw<{ i: number; body: string | null; timeline: string | null }>(`SELECT t.i::int AS i,
    (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
      FROM unnest(string_to_array(t.body,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS body,
    (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
      FROM unnest(string_to_array(t.timeline,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS timeline
    FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS t(body, timeline, i)`,
  [fenced.map(({ page }) => page.compiled_truth), fenced.map(({ page }) => page.timeline)]);
  for (const row of normalized) {
    const { page, applicable } = fenced[Number(row.i) - 1]!;
    page.compiled_truth = overlayWithdrawalBody(page.compiled_truth, row.body ?? '', applicable);
    page.timeline = overlayWithdrawalBody(page.timeline, row.timeline ?? '', applicable);
  }
  return snapshots;
}

/**
 * The frontmatter and header fields of a page's exported file. #5966: the slug stamp rule lives in
 * slugStampedFrontmatter (markdown.ts), shared with every render; an escaped file always carries its slug.
 */
function exportHeader(snapshot: ExportSnapshot, escaped: boolean) {
  const { page } = snapshot;
  const frontmatter = escaped ? { ...(page.frontmatter ?? {}), slug: page.slug } : slugStampedFrontmatter(page.slug, page.frontmatter ?? {});
  return { frontmatter, meta: { type: page.type, title: page.title, tags: snapshot.tags } };
}

/** The exported Markdown of an inline page read by readExportPage, so its withdrawal overlay is applied. */
export function inlineExportMarkdown(snapshot: ExportSnapshot, escaped = false): string {
  const { frontmatter, meta } = exportHeader(snapshot, escaped);
  return serializeMarkdown(frontmatter, snapshot.page.compiled_truth, snapshot.page.timeline, meta);
}

/** The `head` segmentedExportMarkdown starts a segmented page's file with. */
export function segmentedExportHead(snapshot: ExportSnapshot, escaped = false): string {
  const { frontmatter, meta } = exportHeader(snapshot, escaped);
  return serializeMarkdown(frontmatter, '', '', meta).slice(0, -1);
}

/** The ledger an export overlays one page's body with: the source's withdrawals that apply to the page. */
export function exportBodyLedger(slug: string, withdrawals: ExportWithdrawal[]): BodyLedger {
  return { withdrawals: applicableWithdrawals(slug, withdrawals) };
}

function applicableWithdrawals(slug: string, withdrawals: ExportWithdrawal[]): PageWithdrawal[] {
  return withdrawals.filter(w => w.subject === '*' || w.subject === slug)
    .map(({ visibility, fact_hash, withdrawn_at, purged }) => ({ visibility, fact_hash, withdrawn_at, ...(purged ? { purged: true } : {}) }));
}

/**
 * The Markdown file of a segmented page, piece by piece: `head` (serializeMarkdown's frontmatter and blank line), the
 * truth, TIMELINE_SEPARATOR before the first timeline byte, the timeline and the final newline. With `revision` (the
 * snapshot `head` was rendered from), a page that moved since refuses `revision_changed` (GBRA-287: the head and the body
 * are one revision's). Each segment is read
 * once (readBodyRange, trusted stored body) under `ledger` (export: `exportBodyLedger`, the overlay readExportPage gives
 * an inline page; a restore: the page snapshot's), so the bytes are those serializeMarkdown gives the same page read inline. A full collection every EXPORT_COLLECT_BYTES read keeps the
 * reads' garbage (protocol buffers, decoded texts) from piling up as in the streamed import (ingest.ts): a 127 MiB page
 * peaked ~700 MiB above a 2 MiB one without it, 8-55 MiB with it.
 */
export async function* segmentedExportMarkdown(tx: BrainEngine, page: Pick<Page, 'id'>, ledger: BodyLedger, head: string, revision?: string): AsyncIterable<Uint8Array | string> {
  const body = await openSegmentedBody(tx, { pageId: page.id, ledger, ...(revision !== undefined ? { pin: { revision } } : {}) });
  let uncollected = 0;
  const collect = (bytes: number) => {
    uncollected += bytes;
    if (uncollected < EXPORT_COLLECT_BYTES) return;
    Bun.gc(true);
    uncollected = 0;
  };
  yield head;
  for await (const piece of readBodyRange(tx, { pageId: page.id, space: 'stored', start: 0, end: Infinity, audience: 'trusted', region: 'truth', body })) {
    yield piece.bytes;
    collect(piece.bytes.length);
  }
  let separated = false;
  for await (const piece of readBodyRange(tx, { pageId: page.id, space: 'stored', start: 0, end: Infinity, audience: 'trusted', region: 'timeline', body })) {
    if (!piece.bytes.length) continue;
    if (!separated) yield TIMELINE_SEPARATOR;
    separated = true;
    yield piece.bytes;
    collect(piece.bytes.length);
  }
  yield '\n';
}