Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.9 KB
Raw Blame
/**
 * Lenient raw-row extraction of facts/takes fences (#6188).
 *
 * The repair rules and the validator gates work on this view, never on the
 * strict parsers' output and never on their warning strings (which embed row
 * text). For each fence a section holds it returns the markers (located
 * outside markdown code, like every fence reader), the before-region (begin
 * marker to end marker, or to the end of the section when the end marker is
 * missing), the header and its canonical column map (positional when no
 * header is recognizable), rows before the header, and every row's cells
 * with their source spans. A row's identity is its ordinal among the fence's
 * data rows, so it survives renumbering and duplicate claims.
 *
 * `accepted` replays the strict parser on the same region, so a row is
 * accepted exactly when `parseFactsFence` / `parseTakesFence` would return it.
 */
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END, factsHeaderLayout, factsRowProblem, type FactsFenceColumn } from '../facts-fence.ts';
import { TAKES_FENCE_BEGIN, TAKES_FENCE_END } from '../takes-fence.ts';
import { indexOfOutsideCode, scanMarkdownCode, type MarkdownCodeMap } from '../fence-scan.ts';
import { decodeFenceCell, isSeparatorRow } from '../fence-shared.ts';
import { BASE_WIDTH, canonicalColumn, cellValid, COLUMNS, MIN_CELLS, NAMED_COLUMNS, parseRowNum, TOLERATED_TRAILING } from './schema.ts';
import type { FenceIssue, FenceKind, FenceReason, FenceSection } from './types.ts';

export const MARKERS: Record<FenceKind, { begin: string; end: string }> = {
  facts: { begin: FACTS_FENCE_BEGIN, end: FACTS_FENCE_END },
  takes: { begin: TAKES_FENCE_BEGIN, end: TAKES_FENCE_END },
};
/** The two-dash takes markers authors write from memory (#3769). */
const NEAR_MISS = { begin: /^<!--\s*gbrain:takes:begin\s*-->$/, end: /^<!--\s*gbrain:takes:end\s*-->$/ };

export interface RawCell {
  /** Cell text as `parseRowCells` reads it: trimmed, `\|` unescaped, `<br>` decoded. */
  text: string;
  /** Trimmed source text of the cell, escapes kept. */
  raw: string;
  /** Offsets of `raw` in the section. */
  start: number;
  end: number;
}

export interface RawRow {
  /** 1-based line in the section. */
  line: number;
  /** Offsets of the trimmed line in the section. */
  start: number;
  end: number;
  cells: RawCell[];
  /** The trimmed line ends with the pipe `parseRowCells` strips. */
  closed: boolean;
  /** Ordinal among the fence's data rows (rows before the header first); -1 for header and separator lines. */
  occurrence: number;
  /** Sits before the header (every row of a fence with no recognizable header). */
  beforeHeader: boolean;
  /** Cell for each canonical column the row fills. */
  byColumn: Map<string, RawCell>;
  /** Cells that map to no column. */
  extra: RawCell[];
  /** Row-shape problem that makes column positions untrustworthy. */
  shape: 'ok' | 'short_row' | 'extra_cells';
  /** The strict parser returns this row. */
  accepted: boolean;
}

export interface RawMarker {
  start: number;
  end: number;
  line: number;
  /** A two-dash takes marker rather than the canonical three-dash form. */
  nearMiss: boolean;
}

export interface RawFence {
  kind: FenceKind;
  section: FenceSection;
  /** The fence the strict parser reads (later ones are repeats). */
  primary: boolean;
  begin: RawMarker;
  end: RawMarker | null;
  /** The before-region: just after the begin marker to the end marker, or to the section end. */
  regionStart: number;
  regionEnd: number;
  header: RawRow | null;
  /** The strict parser recognizes `header` (its cells include `claim` and `kind`). */
  strictHeader: boolean;
  /** Canonical column per header cell; COLUMNS[kind] when there is no header. */
  columns: Array<string | null>;
  /** The header is not the strict canonical layout (alias-only, out of order, or missing whole columns), so a header rewrite is needed. */
  needsRewrite: boolean;
  separators: RawRow[];
  /** Data rows in order (header and separators excluded). */
  rows: RawRow[];
  issues: FenceIssue[];
}

export interface RawSection {
  section: FenceSection;
  text: string;
  fences: RawFence[];
  /** Marker problems that belong to no extractable fence. */
  issues: FenceIssue[];
}

/** Extract every facts and takes fence of one section. */
export function extractRawRows(text: string, section: FenceSection = 'body'): RawSection {
  const out: RawSection = { section, text, fences: [], issues: [] };
  if (!text.includes('gbrain:facts:') && !text.includes('gbrain:takes:')) return out;
  const code = scanMarkdownCode(text);
  const lines = new LineIndex(text);
  for (const kind of ['facts', 'takes'] as const) extractKind(out, kind, code, lines);
  return out;
}

/** The primary fence of a kind, or null. */
export function primaryFence(raw: RawSection, kind: FenceKind): RawFence | null {
  return raw.fences.find(f => f.kind === kind && f.primary) ?? null;
}

class LineIndex {
  private readonly starts: number[] = [0];
  constructor(readonly text: string) {
    for (let i = text.indexOf('\n'); i !== -1; i = text.indexOf('\n', i + 1)) this.starts.push(i + 1);
  }
  /** 1-based line holding offset `at`. */
  lineOf(at: number): number {
    let lo = 0;
    let hi = this.starts.length - 1;
    while (lo < hi) {
      const mid = (lo + hi + 1) >> 1;
      if (this.starts[mid]! <= at) lo = mid; else hi = mid - 1;
    }
    return lo + 1;
  }
  /** Trimmed bounds of the line holding `at`. */
  trimmedLine(at: number): { start: number; end: number } {
    const lineStart = this.starts[this.lineOf(at) - 1]!;
    const nl = this.text.indexOf('\n', at);
    const lineEnd = nl === -1 ? this.text.length : nl;
    const raw = this.text.slice(lineStart, lineEnd);
    const lead = raw.length - raw.trimStart().length;
    return { start: lineStart + lead, end: lineStart + raw.trimEnd().length };
  }
}

function issue(kind: FenceKind, section: FenceSection, reason: FenceReason, line: number | null, row: number | null = null, column: string | null = null): FenceIssue {
  return { fence: kind, section, row, column, line, reason };
}

function countOutsideCode(text: string, needle: string, code: MarkdownCodeMap): number[] {
  const at: number[] = [];
  for (let i = indexOfOutsideCode(text, needle, 0, code); i !== -1; i = indexOfOutsideCode(text, needle, i + needle.length, code)) at.push(i);
  return at;
}

function extractKind(out: RawSection, kind: FenceKind, code: MarkdownCodeMap, lines: LineIndex): void {
  const { text, section } = out;
  const { begin, end } = MARKERS[kind];
  const begins = countOutsideCode(text, begin, code);
  const ends = countOutsideCode(text, end, code);
  const first = begins.length ? marker(begins[0]!, begin.length, lines, false) : kind === 'takes' ? nearMissBegin(out, code, lines) : null;
  if (ends.length && (!first || ends[0]! < first.start)) {
    out.issues.push(issue(kind, section, 'missing_begin', lines.lineOf(ends[0]!)));
  }
  if (!first) return;
  let cursor: RawMarker | null = first;
  while (cursor) {
    const fence = buildFence(out, kind, cursor, code, lines, out.fences.every(f => f.kind !== kind));
    out.fences.push(fence);
    if (!fence.end) break;
    const next = indexOfOutsideCode(text, begin, fence.end.end, code);
    cursor = next === -1 ? null : marker(next, begin.length, lines, false);
  }
  const primary = out.fences.find(f => f.kind === kind && f.primary)!;
  const repeats = [begin, end].flatMap(m => countOutsideCode(text, m, code).slice(1));
  if (repeats.length) primary.issues.unshift(issue(kind, section, 'repeated_marker', lines.lineOf(Math.min(...repeats))));
}

function marker(at: number, length: number, lines: LineIndex, nearMiss: boolean): RawMarker {
  return { start: at, end: at + length, line: lines.lineOf(at), nearMiss };
}

/** A two-dash takes begin marker standing alone on its line, when no canonical begin exists. */
function nearMissBegin(out: RawSection, code: MarkdownCodeMap, lines: LineIndex): RawMarker | null {
  if (!out.text.includes('gbrain:takes:begin')) return null;
  const at = indexOfOutsideCode(out.text, 'gbrain:takes:begin', 0, code);
  if (at === -1) return null;
  const bounds = lines.trimmedLine(at);
  if (NEAR_MISS.begin.test(out.text.slice(bounds.start, bounds.end))) {
    return { ...bounds, line: lines.lineOf(at), nearMiss: true };
  }
  out.issues.push(issue('takes', out.section, 'marker_near_miss', lines.lineOf(at)));
  return null;
}

function findEnd(text: string, kind: FenceKind, from: number, code: MarkdownCodeMap, lines: LineIndex): RawMarker | null {
  const end = MARKERS[kind].end;
  const at = indexOfOutsideCode(text, end, from, code);
  if (at !== -1) return marker(at, end.length, lines, false);
  if (kind !== 'takes') return null;
  for (let i = indexOfOutsideCode(text, 'gbrain:takes:end', from, code); i !== -1; i = indexOfOutsideCode(text, 'gbrain:takes:end', i + 1, code)) {
    const bounds = lines.trimmedLine(i);
    if (NEAR_MISS.end.test(text.slice(bounds.start, bounds.end))) return { ...bounds, line: lines.lineOf(i), nearMiss: true };
  }
  return null;
}

function buildFence(out: RawSection, kind: FenceKind, begin: RawMarker, code: MarkdownCodeMap, lines: LineIndex, primary: boolean): RawFence {
  const end = findEnd(out.text, kind, begin.end, code, lines);
  const fence: RawFence = {
    kind, section: out.section, primary, begin, end,
    regionStart: begin.end, regionEnd: end ? end.start : out.text.length,
    header: null, strictHeader: false, columns: [...COLUMNS[kind]], needsRewrite: false,
    separators: [], rows: [], issues: [],
  };
  const pipeLines = regionRows(out.text, fence.regionStart, fence.regionEnd, lines);
  classifyLines(fence, pipeLines);
  markAccepted(fence, pipeLines);
  return fence;
}

/** Every pipe-table line of a region, parsed with source spans. */
function regionRows(text: string, start: number, end: number, lines: LineIndex): RawRow[] {
  const rows: RawRow[] = [];
  let lineNo = lines.lineOf(start);
  for (let at = start; at <= end; lineNo++) {
    const nl = text.indexOf('\n', at);
    const lineEnd = nl === -1 || nl > end ? end : nl;
    const row = parseRowSpans(text, at, lineEnd, lineNo);
    if (row) rows.push(row);
    if (lineEnd === end) break;
    at = lineEnd + 1;
  }
  return rows;
}

/** `parseRowCells` with offsets: null unless the trimmed line starts with `|` and has a second pipe. */
export function parseRowSpans(text: string, lineStart: number, lineEnd: number, line: number): RawRow | null {
  const rawLine = text.slice(lineStart, lineEnd);
  const trimmed = rawLine.trim();
  if (!trimmed.startsWith('|') || !trimmed.includes('|', 1)) return null;
  const start = lineStart + (rawLine.length - rawLine.trimStart().length);
  const end = start + trimmed.length;
  const closed = trimmed.length >= 2 && trimmed.endsWith('|');
  const cells: RawCell[] = [];
  const innerEnd = closed ? end - 1 : end;
  let cellStart = start + 1;
  for (let i = start + 1; i <= innerEnd; i++) {
    if (i < innerEnd && text[i] === '\\' && text[i + 1] === '|' && i + 1 < innerEnd) { i++; continue; }
    if (i < innerEnd && text[i] !== '|') continue;
    cells.push(cellAt(text, cellStart, i));
    cellStart = i + 1;
  }
  return { line, start, end, cells, closed, occurrence: -1, beforeHeader: false, byColumn: new Map(), extra: [], shape: 'ok', accepted: false };
}

function cellAt(text: string, from: number, to: number): RawCell {
  const slice = text.slice(from, to);
  const raw = slice.trim();
  const lead = slice.length - slice.trimStart().length;
  const start = raw ? from + lead : from + Math.min(1, slice.length);
  return { raw, start, end: start + raw.length, text: decodeFenceCell(raw.replace(/\\\|/g, '|')) };
}

function strictHeaderCells(row: RawRow): boolean {
  const lower = row.cells.map(c => c.text.toLowerCase());
  return lower.includes('claim') && lower.includes('kind');
}

function aliasHeaderCells(kind: FenceKind, row: RawRow): boolean {
  const mapped = row.cells.map(c => canonicalColumn(kind, c.text));
  return mapped.every(c => c !== null) && mapped.includes('claim') && mapped.includes('kind');
}

function classifyLines(fence: RawFence, pipeLines: RawRow[]): void {
  const { kind } = fence;
  const headerAt = pipeLines.findIndex(r => !isSeparatorRow(r.cells.map(c => c.text)) && (strictHeaderCells(r) || aliasHeaderCells(kind, r)));
  if (headerAt !== -1) setHeader(fence, pipeLines[headerAt]!);
  pipeLines.forEach((row, i) => {
    if (i === headerAt) return;
    const separator = isSeparatorRow(row.cells.map(c => c.text));
    row.beforeHeader = headerAt === -1 || i < headerAt;
    if (row.beforeHeader && pipeLines.length && headerAt !== -1) {
      fence.issues.push(issue(kind, fence.section, 'row_before_header', row.line, rowNumOf(fence, row)));
    }
    if (separator) { fence.separators.push(row); return; }
    row.occurrence = fence.rows.length;
    mapRow(fence, row);
    fence.rows.push(row);
  });
  if (headerAt === -1 && fence.rows.length) fence.issues.push(issue(kind, fence.section, 'no_header', fence.begin.line));
}

function setHeader(fence: RawFence, header: RawRow): void {
  const { kind } = fence;
  fence.header = header;
  fence.strictHeader = strictHeaderCells(header);
  const seen = new Set<string>();
  fence.columns = header.cells.map(cell => {
    const column = canonicalColumn(kind, cell.text);
    if (!column || seen.has(column)) return null;
    seen.add(column);
    return column;
  });
  // A header shorter than the narrowest row the parser reads leaves whole columns absent;
  // a facts header cell with no canonical name (or a repeated one) is never read by position (#6385).
  fence.needsRewrite = !fence.strictHeader || !canonicalOrder(kind, fence.columns) || fence.columns.length < MIN_CELLS[kind]
    || (kind === 'facts' && fence.columns.some(c => c === null));
}

/** The header puts every column the strict parser reads by position at its canonical position. */
function canonicalOrder(kind: FenceKind, columns: Array<string | null>): boolean {
  const canonical = COLUMNS[kind];
  const named = NAMED_COLUMNS[kind];
  return columns.every((column, i) => {
    if (kind === 'takes' && i >= 7) return column === null || named.has(column);
    if (i >= canonical.length) return true;
    return column === canonical[i];
  });
}

/** Row number in the `#` column, or null. */
export function rowNumOf(fence: RawFence, row: RawRow): number | null {
  const at = fence.columns.indexOf('#');
  const cell = at === -1 ? undefined : row.cells[at];
  return cell ? parseRowNum(cell.text) : null;
}

function mapRow(fence: RawFence, row: RawRow): void {
  const { kind } = fence;
  const positional = !fence.header || !fence.needsRewrite;
  row.cells.forEach((cell, j) => {
    const column = positional ? positionalColumn(fence, j) : fence.columns[j] ?? null;
    if (column && !row.byColumn.has(column)) row.byColumn.set(column, cell);
    else row.extra.push(cell);
  });
  if (row.beforeHeader) return;
  row.shape = rowShape(fence, row);
  if (row.shape !== 'ok') fence.issues.push(issue(kind, fence.section, row.shape, row.line, rowNumOf(fence, row)));
}

/** The column the strict parser reads at index `j` (takes resolution columns by header name). */
function positionalColumn(fence: RawFence, j: number): string | null {
  const canonical = COLUMNS[fence.kind];
  if (fence.kind === 'takes' && j >= 7) return fence.header ? (fence.columns[j] ?? null) : null;
  return canonical[j] ?? null;
}

function rowShape(fence: RawFence, row: RawRow): RawRow['shape'] {
  const { kind } = fence;
  const headerLen = fence.columns.length;
  if (!fence.needsRewrite) {
    if (row.cells.length < MIN_CELLS[kind]) return 'short_row';
    // Facts rows of 14 cells are the typed layout even under a narrow header.
    const layoutMax = kind === 'facts' && row.cells.length === 14 ? 14 : Math.max(headerLen, BASE_WIDTH[kind]);
    const allValid = [...row.byColumn].every(([column, cell]) => cellValid(kind, column, cell.text));
    // A facts cell past the canonical layout has no column to mean (#6385).
    if (kind === 'facts' && row.extra.some(cell => cell.text.trim())) return 'extra_cells';
    return row.cells.length > layoutMax && !allValid ? 'extra_cells' : 'ok';
  }
  if (row.cells.length > headerLen) return 'extra_cells';
  const missing = fence.columns.slice(row.cells.length).filter((c): c is string => c !== null);
  return missing.every(c => TOLERATED_TRAILING[kind].has(c)) ? 'ok' : 'short_row';
}

/** Cells the strict parsers reject a row for (takes holders only warn); facts by header column (#6385), takes by position. */
const FACTS_CHECKED: readonly FactsFenceColumn[] = ['kind', 'visibility', 'notability', 'confidence', 'claim_value', 'attributed_to'];
const TAKES_CHECKED: ReadonlyArray<[string, number]> = [['kind', 2], ['weight', 4]];

/** Replay the strict parser over the region: which rows it returns. */
function markAccepted(fence: RawFence, pipeLines: RawRow[]): void {
  if (!fence.primary || !fence.end || fence.begin.nearMiss || fence.end.nearMiss) return;
  const { kind } = fence;
  const seen = new Set<number>();
  let sawHeader = false;
  let layout: FactsFenceColumn[] = [];
  let named = 0;
  for (const row of pipeLines) {
    if (!sawHeader) {
      sawHeader = strictHeaderCells(row);
      if (sawHeader && kind === 'facts') {
        const header = factsHeaderLayout(row.cells.map(c => c.text));
        if ('problem' in header) return;
        ({ layout, named } = header);
      }
      continue;
    }
    const texts = row.cells.map(c => c.text);
    if (isSeparatorRow(texts)) continue;
    if (kind === 'facts') {
      if (factsRowProblem(layout, named, texts)) continue;
      const cell = (column: FactsFenceColumn) => texts[layout.indexOf(column)] ?? '';
      const rowNum = parseRowNum(cell('#'));
      if (rowNum === null || seen.has(rowNum)) continue;
      seen.add(rowNum);
      row.accepted = FACTS_CHECKED.every(column => cellValid('facts', column, cell(column)));
      continue;
    }
    if (texts.length < MIN_CELLS[kind]) continue;
    const rowNum = parseRowNum(texts[0]!);
    if (rowNum === null || seen.has(rowNum)) continue;
    seen.add(rowNum);
    row.accepted = TAKES_CHECKED.every(([column, at]) => cellValid(kind, column, texts[at] ?? ''));
  }
}