Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.7 KB
Raw Blame
/**
 * Large pages (Wave A): how `gbrain get` renders get_page, locally and over
 * the thin client.
 *
 * A whole page prints as its markdown, as it always has. A partial response
 * (body none / outline / window, a section, grep blocks) prints only what it carries, and says on
 * stderr that it is partial and how to read on, so `gbrain get > file` never
 * writes part of a page as if it were all of it.
 *
 * The thin client asks for the whole page. A host that cannot send it in one
 * response refuses with page_too_large_for_inline; the client then pages the
 * round-trip markdown (view: 'markdown') to its end, checks the total size and
 * sha256 the host reported, and prints it window by window, or fails without
 * printing. The windows wait in a 0600 temp file until the copy verifies, so
 * a page of any size prints with one window in memory.
 *
 * A segmented page (Wave B) is never returned whole by get_page. Locally the
 * refusal is page_body_segmented and `gbrain get` streams the page instead:
 * the frontmatter head, the truth segments, TIMELINE_SEPARATOR and the
 * timeline segments, one segment at a time. `--json` puts the page in one
 * string, so above PAGE_MAX_BYTES it refuses, locally and over the thin
 * client, with a fix that drops `--json`.
 */
import { createHash, randomBytes } from 'node:crypto';
import { open, rm, type FileHandle } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../core/engine.ts';
import { serializeMarkdown } from '../core/markdown.ts';
import { opError } from '../core/ops/contract.ts';
import { transcriptFence, unquoteTranscriptText } from '../core/search/transcript-quote.ts';
import { guestWarningLine } from '../core/guests/marker.ts';
import { readFix } from '../core/ops/op-fix.ts';
import { PAGE_MAX_BYTES } from '../core/page-size-policy.ts';
import { isSegmentedPage } from '../core/page-body/page-shape.ts';
import { VALID_TIME_NOT_IN_FORCE_EXIT_CODE } from '../core/exit-codes.ts';
import { assertValidTimeEcho } from '../core/valid-time.ts';
import { setCliExitVerdict } from '../core/cli-force-exit.ts';

/** Window size the thin client asks for while paging a whole page. */
export const REMOTE_PAGE_WINDOW_BYTES = 512 * 1024;
const MAX_RESTARTS = 3;

type GetPageResult = Record<string, any>;

function partialNote(r: GetPageResult): string {
  const size = r.size ?? {};
  const total = typeof size.total_bytes === 'number' ? `${size.total_bytes} bytes` : 'unknown size';
  if (r.window && typeof r.window.byte_start === 'number') {
    const next = r.next_cursor ? ` Read on: gbrain get ${r.slug} --cursor ${r.next_cursor}` : '';
    return `[gbrain] partial page: bytes ${r.window.byte_start}-${r.window.byte_end} of ${total} (lines ${r.window.line_start}-${r.window.line_end}).${next}`;
  }
  return `[gbrain] partial page (${r.body}, ${total}): no page body was printed. Whole page: gbrain get ${r.slug} --body full`;
}

function renderOutline(r: GetPageResult): string {
  const o = r.outline;
  const lines = [`# ${r.title ?? r.slug} — ${r.size?.total_bytes ?? '?'} bytes, ${r.size?.total_lines ?? '?'} lines, revision ${r.revision}`];
  for (const e of o?.entries ?? []) {
    const label = e.heading !== undefined ? `${'#'.repeat(e.level ?? 1)} ${e.heading ?? '(before first heading)'}` : e.first_line ?? '';
    const dates = e.date_min ? ` [${e.date_min}${e.date_max && e.date_max !== e.date_min ? `..${e.date_max}` : ''}]` : '';
    lines.push(`L${e.line_start}-${e.line_end}\t${e.byte_end - e.byte_start}B\t${label}${dates}`);
  }
  if (o?.next_from_index !== undefined) lines.push(`… ${o.total_entries - o.next_from_index} more entries: --outline-from ${o.next_from_index}`);
  return lines.join('\n') + '\n';
}

/** A section or grep read (#7180): its text on stdout, where it sits in the page on stderr. */
function renderSlice(r: GetPageResult): string {
  const more = r.truncated ? ' Truncated: some of it was not printed.' : '';
  if (r.section) {
    const s = r.section;
    const next = r.next?.arguments?.lines ? ` Read on: gbrain get ${r.slug} --lines '${JSON.stringify(r.next.arguments.lines)}'` : '';
    process.stderr.write(`[gbrain] section ${(s.heading_path ?? [s.heading]).join(' > ')}: lines ${s.line_start}-${s.line_end}, ${r.chars} chars.${more}${next}\n`);
    return s.text.endsWith('\n') ? s.text : `${s.text}\n`;
  }
  const g = r.grep;
  process.stderr.write(`[gbrain] grep ${JSON.stringify(g.term)}: ${g.total_matches} matching paragraph${g.total_matches === 1 ? '' : 's'}${g.omitted_matches ? `, ${g.returned_matches} printed` : ''}, ${r.chars} chars.${more}\n`);
  return g.blocks.map((b: GetPageResult) => `-- lines ${b.line_start}-${b.line_end} (match at ${b.match_starts.join(', ')})\n${b.text}\n`).join('');
}

/** The stderr line for a result's valid_time: an as_of answer's status, edition, history and doubt, or a plain read's status today. */
export function validTimeLine(vt: GetPageResult): string {
  if (vt.history === undefined) {
    return `[gbrain] valid_time: ${vt.status} as of ${vt.as_of} (valid_from ${vt.valid_from ?? '-'}, valid_until ${vt.valid_until ?? '-'}).`;
  }
  const edition = vt.chosen_is_current ? 'current page' : vt.chosen_version_id != null ? `version ${vt.chosen_version_id}` : 'none in force';
  const date = vt.valid_from ?? vt.earliest_known_valid_from ?? '-';
  return `[gbrain] valid_time as_of ${vt.as_of}: ${vt.status}; edition ${edition} (valid_from ${date}${vt.valid_until ? `, valid_until ${vt.valid_until}` : ''}); history ${vt.history}${vt.uncertain ? `; uncertain: ${vt.why}` : ''}.`;
}

/** True for an as_of answer with no edition in force: stdout gets the status and why, never a document. */
export function asOfNotInForce(r: GetPageResult | null | undefined): boolean {
  return !!r?.valid_time && r.valid_time.history !== undefined && r.valid_time.status !== 'in_force';
}

/**
 * `gbrain get --as-of` (GBRA-217): an edition read in windows prints whole, each window fetched with the previous
 * window's `next` (slug, source, date and the pinned edition); stderr carries the valid_time header once. False when the
 * result is not a windowed as_of answer (formatGetPage renders it).
 */
export async function printAsOfWindows(call: PagedCall, first: GetPageResult, params: Record<string, unknown>, write: (text: string) => Promise<void>): Promise<boolean> {
  if (params.as_of === undefined || params.cursor !== undefined || params.json === true || first.valid_time?.status !== 'in_force' || !first.window || !first.next) return false;
  process.stderr.write(validTimeLine(first.valid_time) + '\n');
  for (let r = first; ;) {
    if (!r.window) throw new Error(`The edition of ${String(first.slug)} in force on ${String(params.as_of)} is no longer retained (${r.valid_time?.why ?? 'edition_pruned'}); the rest of it was not printed.`);
    await write(r.window.text);
    if (!r.next) return true;
    r = await call(r.next.arguments as Record<string, unknown>);
  }
}

/**
 * After a `gbrain get --as-of` call, locally or over the thin client (`versions` set): a host reply that does not echo
 * `valid_time.as_of` fails closed (`host_upgrade_required`, on every window too), the exit status is set, and a windowed
 * edition is paged. True when the edition was printed here.
 */
export async function finishAsOfRead(call: PagedCall, result: GetPageResult, params: Record<string, unknown>, write: (text: string) => Promise<void>,
  versions?: { client: string; host: string | null }): Promise<boolean> {
  if (params.as_of === undefined) return false;
  if (versions) assertValidTimeEcho(params, result, versions);
  const checked: PagedCall = versions ? async p => { const r = await call(p); assertValidTimeEcho(params, r, versions); return r; } : call;
  const exit = asOfExitCode(result);
  if (exit) setCliExitVerdict(exit);
  return printAsOfWindows(checked, result, params, write);
}

/** The exit status of an as_of answer: 15 when no edition is in force, 1 when the edition's text has gone, 0 otherwise. */
export function asOfExitCode(r: GetPageResult | null | undefined): number {
  if (!r?.valid_time || r.valid_time.history === undefined) return 0;
  if (r.valid_time.status !== 'in_force') return VALID_TIME_NOT_IN_FORCE_EXIT_CODE;
  return r.compiled_truth === undefined && !r.window ? 1 : 0;
}

/** stdout text for one get_page result; partial responses add a stderr note. A guest page's text starts with its warning line (GBRA-504). */
export function formatGetPage(r: GetPageResult): string {
  if (r.guest) {
    const text = formatGetPage({ ...r, guest: undefined });
    return text && `${guestWarningLine(r.guest)}\n${text}`;
  }
  if (r.valid_time) process.stderr.write(validTimeLine(r.valid_time) + '\n');
  if (asOfNotInForce(r)) return `status: ${r.valid_time.status}\nwhy: ${r.valid_time.why ?? 'none'}\n`;
  if (r.valid_time?.history !== undefined && r.compiled_truth === undefined && !r.window) return '';
  if (r.error === 'ambiguous_slug') {
    return `Ambiguous slug. Did you mean:\n${r.candidates.map((c: string) => `  ${c}`).join('\n')}\n`;
  }
  if (typeof r.body !== 'string') {
    if (typeof r.content === 'string' && r.revision) process.stderr.write(`[gbrain] revision ${r.revision}: put or delete this page with --expected-revision ${r.revision}.\n`);
    return serializeMarkdown(r.frontmatter || {}, r.compiled_truth || '', r.timeline || '', { type: r.type, title: r.title, tags: r.tags || [] });
  }
  if (r.section || r.grep) return renderSlice(r);
  process.stderr.write(partialNote(r) + '\n');
  if (r.window) return r.window.text;
  if (r.outline) return renderOutline(r);
  return `revision: ${r.revision}\nsize: ${r.size?.total_bytes ?? '?'} bytes, ${r.size?.total_lines ?? '?'} lines\n` +
    (r.chunks ? `chunks: ${r.chunks.total} (${r.chunks.embedded} embedded)\n` : '');
}

/** The canonical code of a tool error: a thin-client RemoteMcpError's detail, or an in-process OperationError (a legacy wire value moves the code to canonical). */
export function remoteErrorCode(e: unknown): string | undefined {
  const err = e as { detail?: { code?: string; canonical_code?: string }; canonical?: string; code?: unknown } | null;
  if (err?.detail && typeof err.detail === 'object') return err.detail.canonical_code ?? err.detail.code;
  return err?.canonical ?? (typeof err?.code === 'string' ? err.code : undefined);
}

/** True when a get_page call named no body mode, selector, round-trip flag or as_of date: the caller wants the page as it is now. */
export function wantsWholePage(params: Record<string, unknown>): boolean {
  return ['body', 'lines', 'bytes', 'around_chunk_id', 'cursor', 'include_content', 'content_only', 'section', 'grep', 'as_of'].every(k => params[k] === undefined);
}

/**
 * @internal Work the thin client's paged read does, for the memory test: windows written to the spool (`spooled`)
 * and read back from it to print (`replayed`). Counts, not time.
 */
export const pagedReadWork = { spooled: 0, replayed: 0 };

type PagedCall = (params: Record<string, unknown>) => Promise<GetPageResult>;
/** `fence`: the host quoted the windows (#8042, GST-687), so the page prints inside one quote fence (#8377). */
interface SpooledPage { revision: string; windows: number; bytes: number; fence: { open: string; close: string } | null; parts(): AsyncIterable<Buffer> }

/**
 * Page a whole page's round-trip markdown through windows into a 0600 temp file, then hand `use` the verified copy;
 * `parts()` reads it back window by window as the windows arrived. Restarts from the top when the page changes
 * mid-read; throws rather than hand over a short or unverified copy. The temp file is removed whatever happens.
 */
async function withSpooledPage<T>(call: PagedCall, params: Record<string, unknown>, use: (page: SpooledPage) => Promise<T>): Promise<T> {
  const path = join(tmpdir(), `gbrain-page-${process.pid}-${randomBytes(8).toString('hex')}.md`);
  const spool = await open(path, 'wx+', 0o600);
  try {
    const { lengths, revision, fence } = await spoolWholePage(call, params, spool);
    const bytes = lengths.reduce((n, length) => n + length, 0);
    return await use({ revision, windows: lengths.length, bytes, fence, async *parts() {
      for (let i = 0, at = 0; i < lengths.length; at += lengths[i]!, i++) {
        const part = Buffer.allocUnsafe(lengths[i]!);
        for (let filled = 0; filled < part.length;) {
          const { bytesRead } = await spool.read(part, filled, part.length - filled, at + filled);
          if (bytesRead === 0) throw new Error(`The spooled copy of ${String(params.slug)} was truncated; the rest of the page was not printed.`);
          filled += bytesRead;
        }
        pagedReadWork.replayed++;
        yield part;
      }
    } });
  } finally {
    await spool.close().catch(() => undefined);
    await rm(path, { force: true });
  }
}

/** The paging loop of withSpooledPage: the byte length of each window written to `spool`, once size and sha256 verify. */
async function spoolWholePage(call: PagedCall, params: Record<string, unknown>, spool: FileHandle): Promise<{ lengths: number[]; revision: string; fence: SpooledPage['fence'] }> {
  const base = { slug: params.slug, ...(params.source_id !== undefined ? { source_id: params.source_id } : {}), body: 'window', view: 'markdown' };
  for (let attempt = 0; attempt <= MAX_RESTARTS; attempt++) {
    await spool.truncate(0);
    const lengths: number[] = [];
    const hash = createHash('sha256');
    let bytes = 0;
    let quoted = false;
    let longest = 0;
    let run = 0;
    let r = await call({ ...base, max_bytes: REMOTE_PAGE_WINDOW_BYTES });
    const { revision, size } = r;
    let changed = false;
    for (;;) {
      if (!r.window || typeof r.window.byte_start !== 'number' || !size || typeof size.sha256 !== 'string') {
        throw new Error(`get_page did not return a markdown window for ${String(params.slug)}; the page was not printed.`);
      }
      // #8042: a host quotes each window of a chat page; size and sha256 cover the page's own bytes.
      const text = r.quoted === 'transcript' ? unquoteTranscriptText(r.window.text, String(r.slug)) : r.window.text;
      quoted ||= r.quoted === 'transcript';
      for (const ch of text) longest = Math.max(longest, run = ch === '`' ? run + 1 : 0);
      const part = Buffer.from(text, 'utf8');
      hash.update(part);
      for (let written = 0; written < part.length;) written += (await spool.write(part, written, part.length - written, bytes + written)).bytesWritten;
      lengths.push(part.length);
      pagedReadWork.spooled++;
      bytes += part.length;
      if (!r.next_cursor) break;
      try {
        r = await call({ ...base, cursor: r.next_cursor });
      } catch (e) {
        if (remoteErrorCode(e) !== 'revision_changed') throw e;
        changed = true;
        break;
      }
      if (r.revision !== revision) { changed = true; break; }
    }
    if (changed) continue;
    if (bytes !== size.total_bytes || hash.digest('hex') !== size.sha256) {
      throw new Error(`The paged copy of ${String(params.slug)} does not match the size and sha256 the host reported; the page was not printed.`);
    }
    return { lengths, revision, fence: quoted ? transcriptFence(String(r.slug), longest) : null };
  }
  throw new Error(`${String(params.slug)} kept changing while it was being read (${MAX_RESTARTS + 1} attempts); the page was not printed. Try again.`);
}

/** `--json` holds the whole page in one string: refused above PAGE_MAX_BYTES, with a fix that streams it instead. */
function jsonTooLargeError(slug: string, sourceId: string | undefined, bytes: number) {
  return opError('page_too_large_for_inline', `Page ${slug} is ${bytes} bytes, over the ${PAGE_MAX_BYTES}-byte limit of one --json document.`,
    'Run gbrain get without --json: it prints the page whole, window by window or segment by segment.',
    { fix: readFix('Prints the page whole without holding it as one string.', { argv: ['gbrain', 'get', ...(sourceId ? ['--source', sourceId] : []), '--', slug] }) });
}

/**
 * A whole page's round-trip markdown as one string, paged and verified as `printPagedPage` does it: for `--json`,
 * which refuses a page over PAGE_MAX_BYTES before this holds it.
 */
export async function readWholePageMarkdown(call: PagedCall, params: Record<string, unknown>): Promise<{ markdown: string; revision: string; windows: number; fence: SpooledPage['fence'] }> {
  return withSpooledPage(call, params, async page => {
    const parts: Buffer[] = [];
    for await (const part of page.parts()) parts.push(part);
    return { markdown: Buffer.concat(parts, page.bytes).toString('utf8'), revision: page.revision, windows: page.windows, fence: page.fence };
  });
}

/** The thin client's whole-page read after a page_too_large_for_inline refusal: page, verify, then print each window (or one --json document). */
export async function printPagedPage(call: PagedCall, params: Record<string, unknown>, write: (text: string) => Promise<void>): Promise<void> {
  if (params.json === true) {
    const paged = await readWholePageMarkdown(async p => {
      const r = await call(p);
      if (r.size?.total_bytes > PAGE_MAX_BYTES) throw jsonTooLargeError(String(params.slug), params.source_id as string | undefined, r.size.total_bytes);
      return r;
    }, params);
    const content = paged.fence ? `${paged.fence.open}${paged.markdown}${paged.fence.close}` : paged.markdown;
    await write(JSON.stringify({ slug: params.slug, revision: paged.revision, content, windows: paged.windows }) + '\n');
    return;
  }
  await withSpooledPage(call, params, async page => {
    if (page.fence) await write(page.fence.open);
    for await (const part of page.parts()) await write(part.toString('utf8'));
    if (page.fence) await write(page.fence.close);
  });
}

/** What the local get_page call returns when `streamSegmentedGet` printed the page itself. */
export const SEGMENTED_STREAMED = Symbol('segmented page streamed');

/**
 * Local `gbrain get` of a segmented page, after get_page refused it whole (page_body_segmented, whose fix names the
 * resolved slug and source): stream the trusted markdown view to `write`, one segment at a time, as the inline
 * page would print. False when `error` is not that refusal of a whole-page read.
 */
export async function streamSegmentedGet(engine: BrainEngine, params: Record<string, unknown>, error: unknown, write: (text: string) => Promise<void>): Promise<boolean> {
  const ref = (error as { fix?: { mcp?: { arguments?: Record<string, unknown> } } } | null)?.fix?.mcp?.arguments;
  if (remoteErrorCode(error) !== 'page_body_segmented' || !wantsWholePage(params) || typeof ref?.slug !== 'string' || typeof ref.source_id !== 'string') return false;
  const snapshot = await engine.readPageSnapshot(ref.slug, { sourceId: ref.source_id, includeDeleted: params.include_deleted === true });
  if (!snapshot || !isSegmentedPage(snapshot.page)) return false;
  const { page, tags } = snapshot;
  const json = params.json === true;
  if (json && Number(page.body_bytes) > PAGE_MAX_BYTES) throw jsonTooLargeError(page.slug, page.source_id, Number(page.body_bytes));
  const [{ openSegmentedBody }, { markdownHead, segmentedMarkdownDocument }] = await Promise.all([import('../core/page-body/read-range.ts'), import('../core/page-window/segmented.ts')]);
  const body = await openSegmentedBody(engine, { pageId: page.id, ledger: { withdrawals: snapshot.withdrawals, ...(snapshot.globalPurges ? { globalPurges: snapshot.globalPurges } : {}) },
    pin: { revision: snapshot.revision, generation: snapshot.bodyGeneration } });
  const doc = await segmentedMarkdownDocument(engine, body, markdownHead(page, tags), 'trusted');
  const parts: string[] = [];
  for await (const bytes of doc.stream()) {
    if (json) parts.push(bytes.toString('utf8'));
    else await write(bytes.toString('utf8'));
  }
  if (json) await write(JSON.stringify({ slug: page.slug, source_id: page.source_id, revision: snapshot.revision, content: parts.join('') }) + '\n');
  return true;
}