Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.7 KB
Raw Blame
/**
 * Large pages A7: no silent head-only extraction.
 *
 * Page extractors read a bounded slice of the body (facts: 8,000 chars,
 * atoms: `cycle.extract_atoms.max_input_chars`). Reading the head every time
 * means the text appended to a long conversation page is never extracted.
 * A per-page watermark per pipeline fixes that: the UTF-8 byte offset of the
 * body the last successful extraction covered, the sha256 of the body up to
 * that offset, and the pipeline version. When the current body still starts
 * with exactly that prefix, the next extraction reads only the appended tail
 * (its newest `cap` characters when the tail itself is longer). Otherwise it
 * reads the head as before and records the durable receipt `input_truncated`
 * in `page_extraction_state`, which doctor's `extraction_input_truncated`
 * check reports. A full read clears the receipt; a tail read keeps it, since
 * the text skipped earlier is still unread.
 *
 * A segmented page's compiled truth is planned the same way without holding it
 * (planSegmentedExtraction): one pass over its segments (readBodyRange) hashes
 * the body and the watermark prefix and keeps only the head and the newest
 * window of the appended tail.
 */
import { createHash } from 'node:crypto';
import type { BrainEngine } from './engine.ts';
import { truncateUtf8 } from './text-safe.ts';
import { openSegmentedBody, readBodyRange } from './page-body/read-range.ts';
import { readGarbageCollector, segmentedPassWork } from './page-body/segment-blocks.ts';

export type ExtractionPipeline = 'facts_backstop' | 'extract_atoms' | 'conversation_facts';

/** Bump a pipeline's version when its extracted body changes shape: every stored watermark then falls back to the head once. */
export const EXTRACTION_PIPELINE_VERSION: Readonly<Record<ExtractionPipeline, number>> = {
  facts_backstop: 1, extract_atoms: 1, conversation_facts: 1,
};

export type HeadFallbackReason = 'no_watermark' | 'prefix_changed' | 'version_changed' | 'no_new_text';

/** The durable receipt of an extraction that did not read all of its input. */
export interface InputTruncated {
  /**
   * PR 4 (Wave B incremental derivation, transcripts): `no_messages`, a segmented conversation page reads as a transcript
   * format over its whole text yet no message parsed under it; the run changed nothing. `detection_ambiguous`, no format is
   * detected over a segmented conversation page yet the formats in `grammars` parse messages in it: the page is held, nothing
   * extracted or collected, until it changes or the operator chooses its format. `grammar_chosen`, the operator's format
   * (`gbrain extract-conversation-facts --grammar`) for page version `version_token`; not a truncation.
   */
  kind: 'head' | 'tail_window' | 'skipped_too_large' | 'no_messages' | 'detection_ambiguous' | 'grammar_chosen';
  pipeline: ExtractionPipeline;
  /** Characters the extractor read (0 for a skipped page). */
  processed_chars: number;
  /** Characters of the input the cap applied to: the body for `head`, the appended tail for `tail_window`; absent for an unread input. */
  total_chars?: number;
  reason?: HeadFallbackReason;
  /** `skipped_too_large`: the input's UTF-8 size, the cap it exceeded and where the input came from. */
  total_bytes?: number;
  cap_bytes?: number;
  origin?: 'page' | 'sidecar';
  /**
   * conversation_facts on a page over the window cap: `no_prior_facts` (the newest window was extracted; the page had
   * no conversation facts a clear could lose) or `prior_facts_kept` (skipped: the page's `prior_facts` conversation
   * facts may lie outside the window, so nothing was cleared).
   */
  window_rule?: 'no_prior_facts' | 'prior_facts_kept';
  prior_facts?: number;
  /** PR 4 (Wave B incremental derivation, transcripts): `detection_ambiguous`: the messages each format that parses any parses over the page. */
  grammars?: Record<string, number>;
  /** PR 4 (Wave B incremental derivation, transcripts): `grammar_chosen`: the chosen format (null: none, the page is not a transcript). */
  grammar?: string | null;
  /** PR 4 (Wave B incremental derivation, transcripts): `grammar_chosen`, `detection_ambiguous`: the page version the choice or the hold is for. */
  version_token?: string;
}

export interface StoredWatermark { pipelineVersion: number; bytes: number; sha256: string }

export interface ExtractionPlan {
  pipeline: ExtractionPipeline;
  /** full: the body fits the cap; tail: past the watermark; head: no usable watermark. */
  mode: 'full' | 'tail' | 'head';
  /** Exactly what the extractor reads, at most `cap` characters. Empty only for a tail with nothing appended. */
  text: string;
  truncated: InputTruncated | null;
  /** The watermark to store once the extraction succeeded: the whole current body. */
  next: { bytes: number; sha256: string };
}

const sha256 = (bytes: Uint8Array): string => createHash('sha256').update(bytes).digest('hex');

/** The newest `cap` characters of `text`, never starting inside a surrogate pair, at a line start when one is near. */
function newestWindow(text: string, cap: number): string {
  let start = text.length - cap;
  const code = text.charCodeAt(start);
  if (code >= 0xdc00 && code <= 0xdfff) start++;
  const newline = text.indexOf('\n', start);
  if (newline >= 0 && newline - start < Math.floor(cap / 10)) start = newline + 1;
  return text.slice(start);
}

/**
 * Decide what one extraction reads. `emptyTail` says what a matching
 * watermark with nothing appended means: `skip` (nothing new to extract) or
 * `head` (the caller re-reads on purpose, e.g. after the cap rose).
 */
export function planExtraction(body: string, stored: StoredWatermark | null,
  opts: { cap: number; pipeline: ExtractionPipeline; emptyTail: 'skip' | 'head' }): ExtractionPlan {
  const { cap, pipeline } = opts;
  const buf = Buffer.from(body, 'utf8');
  const next = { bytes: buf.length, sha256: sha256(buf) };
  if (body.length <= cap) return { pipeline, mode: 'full', text: body, truncated: null, next };
  const head = (reason: HeadFallbackReason): ExtractionPlan => ({ pipeline, mode: 'head', text: truncateUtf8(body, cap), next,
    truncated: { kind: 'head', pipeline, processed_chars: Math.min(cap, body.length), total_chars: body.length, reason } });
  if (!stored) return head('no_watermark');
  if (stored.pipelineVersion !== EXTRACTION_PIPELINE_VERSION[pipeline]) return head('version_changed');
  const offset = stored.bytes;
  const onBoundary = offset === buf.length || (offset < buf.length && (buf[offset]! & 0xc0) !== 0x80);
  if (offset > buf.length || !onBoundary || sha256(buf.subarray(0, offset)) !== stored.sha256) return head('prefix_changed');
  const tail = buf.subarray(offset).toString('utf8');
  if (!tail.trim()) return opts.emptyTail === 'skip' ? { pipeline, mode: 'tail', text: '', truncated: null, next } : head('no_new_text');
  if (tail.length <= cap) return { pipeline, mode: 'tail', text: tail, truncated: null, next };
  const window = newestWindow(tail, cap);
  return { pipeline, mode: 'tail', text: window, next,
    truncated: { kind: 'tail_window', pipeline, processed_chars: window.length, total_chars: tail.length } };
}

/**
 * planExtraction over a segmented page's compiled truth (the trusted stored text readPageSnapshot gives an inline
 * page), read segment by segment: the same plan, holding at most the head, the newest 2 x `cap` characters of the
 * tail and one segment. O(page) work, O(segment) memory.
 */
export async function planSegmentedExtraction(engine: BrainEngine, pageId: number, stored: StoredWatermark | null,
  opts: { cap: number; pipeline: ExtractionPipeline; emptyTail: 'skip' | 'head' }): Promise<ExtractionPlan> {
  const { cap, pipeline } = opts;
  const body = await openSegmentedBody(engine, { pageId });
  const offset = stored && stored.pipelineVersion === EXTRACTION_PIPELINE_VERSION[pipeline] ? stored.bytes : null;
  const whole = createHash('sha256');
  const prefix = createHash('sha256');
  const decoder = new TextDecoder('utf-8', { ignoreBOM: true });
  const tailDecoder = new TextDecoder('utf-8', { ignoreBOM: true });
  let bytes = 0, chars = 0, tailChars = 0, head = '', tail = '', tailText = false;
  let boundary: number | undefined;
  const collect = readGarbageCollector();
  for await (const piece of readBodyRange(engine, { pageId, space: 'stored', start: 0, end: Infinity, audience: 'trusted', region: 'truth', body })) {
    const buf = piece.bytes;
    collect(buf.length);
    segmentedPassWork.bytes += buf.length;
    whole.update(buf);
    const text = decoder.decode(buf, { stream: true });
    chars += text.length;
    if (head.length <= cap) head += text.slice(0, cap + 1 - head.length);
    if (offset !== null) {
      const split = Math.max(0, Math.min(buf.length, offset - bytes));
      if (split > 0) prefix.update(buf.subarray(0, split));
      if (offset >= bytes && offset < bytes + buf.length) boundary = buf[offset - bytes];
      const after = tailDecoder.decode(buf.subarray(split), { stream: true });
      tailChars += after.length;
      tailText ||= after.trim().length > 0;
      segmentedPassWork.carry += tail.length + after.length;
      tail = (tail + after).slice(-2 * cap);
    }
    bytes += buf.length;
  }
  const next = { bytes, sha256: whole.digest('hex') };
  if (chars <= cap) return { pipeline, mode: 'full', text: head, truncated: null, next };
  const headPlan = (reason: HeadFallbackReason): ExtractionPlan => ({ pipeline, mode: 'head', text: truncateUtf8(head, cap), next,
    truncated: { kind: 'head', pipeline, processed_chars: cap, total_chars: chars, reason } });
  if (!stored) return headPlan('no_watermark');
  if (offset === null) return headPlan('version_changed');
  const onBoundary = offset === bytes || (offset < bytes && boundary !== undefined && (boundary & 0xc0) !== 0x80);
  if (offset > bytes || !onBoundary || prefix.digest('hex') !== stored.sha256) return headPlan('prefix_changed');
  if (!tailText) return opts.emptyTail === 'skip' ? { pipeline, mode: 'tail', text: '', truncated: null, next } : headPlan('no_new_text');
  if (tailChars <= cap) return { pipeline, mode: 'tail', text: tail, truncated: null, next };
  const window = newestWindow(tail, cap);
  return { pipeline, mode: 'tail', text: window, next, truncated: { kind: 'tail_window', pipeline, processed_chars: window.length, total_chars: tailChars } };
}

let warnedStorage = false;
function warnStorageOnce(action: string, error: unknown): void {
  if (warnedStorage) return;
  warnedStorage = true;
  console.error(`[extraction-watermark] could not ${action} (${error instanceof Error ? error.message : String(error)}); `
    + 'large pages fall back to head-only extraction until gbrain apply-migrations brings the schema to head.');
}

/** The page's id and stored watermark for `pipeline`; `pageId` is null when no live page has that slug. Fail-soft to no watermark. */
export async function readPageWatermark(engine: BrainEngine, sourceId: string, slug: string,
  pipeline: ExtractionPipeline): Promise<{ pageId: number | null; stored: StoredWatermark | null }> {
  try {
    const [row] = await engine.executeRaw<{ id: number; pipeline_version: number | null; watermark_bytes: string | number | null; watermark_sha256: string | null }>(
      `SELECT p.id, s.pipeline_version, s.watermark_bytes, s.watermark_sha256
         FROM pages p LEFT JOIN page_extraction_state s ON s.page_id = p.id AND s.pipeline = $3
        WHERE p.source_id = $1 AND p.slug = $2 AND p.deleted_at IS NULL`, [sourceId, slug, pipeline]);
    if (!row) return { pageId: null, stored: null };
    const stored = row.pipeline_version !== null && row.watermark_bytes !== null && row.watermark_sha256
      ? { pipelineVersion: Number(row.pipeline_version), bytes: Number(row.watermark_bytes), sha256: row.watermark_sha256 } : null;
    return { pageId: Number(row.id), stored };
  } catch (error) {
    warnStorageOnce('read the extraction watermark', error);
    return { pageId: null, stored: null };
  }
}

/**
 * Store a successful extraction: the watermark moves to the end of the body
 * it planned against. A full read clears the receipt, a truncated read
 * replaces it, and a complete tail read keeps whatever gap an earlier read left.
 */
export async function commitExtractionPlan(engine: BrainEngine, pageId: number, plan: ExtractionPlan): Promise<void> {
  const receipt = plan.truncated ? JSON.stringify(plan.truncated) : null;
  try {
    await engine.executeRaw(
      `INSERT INTO page_extraction_state (page_id, pipeline, pipeline_version, watermark_bytes, watermark_sha256, input_truncated, updated_at)
       VALUES ($1, $2, $3, $4, $5, $6::text::jsonb, now())
       ON CONFLICT (page_id, pipeline) DO UPDATE SET pipeline_version = EXCLUDED.pipeline_version,
         watermark_bytes = EXCLUDED.watermark_bytes, watermark_sha256 = EXCLUDED.watermark_sha256,
         input_truncated = CASE WHEN $7 THEN page_extraction_state.input_truncated ELSE EXCLUDED.input_truncated END,
         updated_at = now()`,
      [pageId, plan.pipeline, EXTRACTION_PIPELINE_VERSION[plan.pipeline], plan.next.bytes, plan.next.sha256, receipt,
        plan.mode === 'tail' && !plan.truncated]);
  } catch (error) {
    warnStorageOnce('store the extraction watermark', error);
  }
}

/** Record (receipt) or clear (null) the partial-input receipt of a pipeline that keeps no watermark: the conversation-facts size skip. */
/** PR 4 (Wave B incremental derivation, transcripts): the page's receipt for `pipeline`, or null (none, or unreadable). */
export async function readExtractionReceipt(engine: BrainEngine, pageId: number, pipeline: ExtractionPipeline): Promise<InputTruncated | null> {
  try {
    const [row] = await engine.executeRaw<{ receipt: InputTruncated | string | null }>(
      'SELECT input_truncated AS receipt FROM page_extraction_state WHERE page_id = $1 AND pipeline = $2', [pageId, pipeline]);
    const receipt = row?.receipt ?? null;
    return typeof receipt === 'string' ? JSON.parse(receipt) as InputTruncated : receipt;
  } catch (error) {
    warnStorageOnce('read the extraction receipt', error);
    return null;
  }
}

export async function setExtractionReceipt(engine: BrainEngine, pageId: number, pipeline: ExtractionPipeline,
  receipt: InputTruncated | null): Promise<void> {
  try {
    if (!receipt) {
      await engine.executeRaw('DELETE FROM page_extraction_state WHERE page_id = $1 AND pipeline = $2', [pageId, pipeline]);
      return;
    }
    await engine.executeRaw(
      `INSERT INTO page_extraction_state (page_id, pipeline, pipeline_version, input_truncated, updated_at)
       VALUES ($1, $2, $3, $4::text::jsonb, now())
       ON CONFLICT (page_id, pipeline) DO UPDATE SET input_truncated = EXCLUDED.input_truncated, updated_at = now()`,
      [pageId, pipeline, EXTRACTION_PIPELINE_VERSION[pipeline], JSON.stringify(receipt)]);
  } catch (error) {
    warnStorageOnce('store the extraction receipt', error);
  }
}