Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.5 KB
Raw Blame
/**
 * extract_atoms response parsing: the typed parse outcome, the bounded `[`
 * anchor scan and the atom shape gate. Peeled out of extract-atoms.ts, which
 * re-exports the public names.
 */
import { matchingCloseBracket, stripReasoningBlocks } from '../llm-json.ts';
import { ATOM_TYPES } from './extract-atoms-schema.ts';

export interface ExtractedAtom {
  title: string;
  atom_type: typeof ATOM_TYPES[number];
  body: string;
  source_quote?: string;
  lesson?: string;
  /**
   * 1-3 kebab-case topic labels for concept clustering. Consumed by
   * synthesize_concepts (groups atoms by `frontmatter.concepts`; only
   * labels shared by >=2 atoms materialize a concept page, so the prompt
   * biases reuse-over-coinage). #2123.
   */
  concepts?: string[];
  virality_score?: number;
  emotional_register?: string;
}

/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */
const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/;

/**
 * gbrain#4148 — typed parse outcome. Malformed model output and a legitimate
 * zero-yield extraction both used to collapse into `[]`, so malformed output
 * was tombstoned as success (the page never retried, its atoms silently
 * lost). `ok: false` means the response was not parseable as an atoms array
 * AT ALL — a content-deterministic failure class the caller counts toward a
 * bounded tombstone; `ok: true, atoms: []` means the model genuinely
 * extracted nothing.
 */
export type AtomsParseOutcome =
  | { ok: true; atoms: ExtractedAtom[] }
  | { ok: false; reason: string; stopped?: true };

/**
 * #6260: the gateway stop reason decides how far a response is trusted. A
 * response stopped by the output cap, a refusal or a content filter is never
 * parsed; `other` and `tool_calls` (some local providers report a normal end
 * as `other`) are parsed, but their zero-yield is never taken as a
 * completion. Both come back `ok: false, stopped: true`, so the caller counts
 * them like malformed output (the bounded streak; managed: a failure receipt,
 * which never retires earlier atoms). A missing reason (legacy chat seams)
 * reads as a normal end.
 */
export function parseAtomsOutcome(raw: string, stopReason?: string): AtomsParseOutcome {
  if (stopReason === 'length' || stopReason === 'refusal' || stopReason === 'content_filter') {
    return { ok: false, stopped: true, reason: `output stopped before the end (stopReason=${stopReason})` };
  }
  const outcome = parseAtomsText(raw);
  if ((stopReason === 'other' || stopReason === 'tool_calls') && outcome.ok && outcome.atoms.length === 0) {
    return { ok: false, stopped: true, reason: `an empty answer under stopReason=${stopReason} is not taken as complete` };
  }
  return outcome;
}

function parseAtomsText(raw: string): AtomsParseOutcome {
  const direct = parseAtomsOutcomeInner(raw);
  if (direct.ok) return direct;
  // Same reasoning-block hazard as the facts extractor: `indexOf('[')` below
  // finds a bracket inside <think> when the model drafts its array while
  // reasoning, so the parse fails and the page is halted. Ladder, not a
  // pre-filter: raw first, stripped only on failure — and the ORIGINAL
  // outcome is returned when the retry also fails, so error reasons are
  // unchanged for non-reasoning models.
  const stripped = stripReasoningBlocks(raw);
  if (stripped && stripped !== raw.trim()) {
    const retry = parseAtomsOutcomeInner(stripped);
    if (retry.ok) return retry;
  }
  return direct;
}

/**
 * Bound on how many `[` offsets the anchor scan will try. Completions are
 * already capped by `max_output_tokens`, so this is a belt-and-braces guard
 * against a pathological bracket-dense response, not a functional limit —
 * every failing candidate fails at ~offset 0 of its own slice, so the scan is
 * cheap. Reached-the-cap behaves exactly like found-nothing: the FIRST
 * offset's outcome is returned, i.e. today's behaviour.
 */
const MAX_ARRAY_ANCHOR_CANDIDATES = 64;

/**
 * Parse the JSON array anchored at ONE `[` offset: whole-slice parse, then a
 * trim-back to the array's own closing `]` so trailing prose cannot hijack it
 * (the first complete array wins). Split out of parseAtomsOutcomeInner so the
 * anchor scan can try successive offsets without duplicating the reason
 * strings — those are asserted by tests and ride the drain's `last_error`.
 */
function parseArrayAtOffset(
  cleaned: string,
  start: number,
): { ok: true; parsed: unknown[] } | { ok: false; reason: string } {
  const slice = cleaned.slice(start);
  let parsed: unknown;
  try {
    parsed = JSON.parse(slice);
  } catch {
    // Trim back to the array's own closing bracket to drop trailing prose.
    const arrayEnd = matchingCloseBracket(slice);
    if (arrayEnd === -1) return { ok: false, reason: 'unterminated JSON array' };
    try {
      parsed = JSON.parse(slice.slice(0, arrayEnd + 1));
    } catch {
      return { ok: false, reason: 'unparseable JSON array' };
    }
  }
  if (!Array.isArray(parsed)) return { ok: false, reason: 'JSON value is not an array' };
  return { ok: true, parsed };
}

function parseAtomsOutcomeInner(raw: string): AtomsParseOutcome {
  // Strip markdown code fences if the LLM wrapped JSON in them.
  let cleaned = raw.trim();
  const fenceMatch = cleaned.match(/```(?:json)?\s*([\s\S]*?)```/);
  if (fenceMatch) cleaned = fenceMatch[1].trim();

  const firstStart = cleaned.indexOf('[');
  if (firstStart === -1) return { ok: false, reason: 'no JSON array in response' };

  // ANCHOR SCAN. Pre-fix this committed to `indexOf('[')` — the FIRST bracket
  // anywhere in the response. Any bracket in a preamble hijacked the anchor,
  // and a brain whose house style mandates inline `[Source: …]` citations and
  // `[[wikilink]]` backlinks (or whose transcripts carry `[user]` / `[tool: …]`
  // role markers) makes the model echo one while narrating, so a response
  // carrying a perfectly good array was reported `unparseable JSON array`.
  //
  // Acceptance requires the candidate to parse AND to yield >= 1 atom-shaped
  // element (`atomsFromParsedArray` is the single source of truth for
  // "atom-shaped"). Parseability alone is NOT enough: a zero-yield result is
  // exactly what TOMBSTONES an item (#2144), so only ONE parseable shape may
  // produce it — the literal `[]` the #4948 prompt asks for when nothing is
  // extractable, accepted at ANY offset (a model that echoes a `[Source: …]`
  // citation before obeying must not lose its honest `[]` to this gate and
  // burn three strikes into a tombstone + halt). A NON-empty array whose
  // elements all fail the shape gate is malformed output: it rides the
  // failure streak like every other parse failure instead of tombstoning the
  // item forever on the first try.
  //
  // #6260: the honest `[]` counts only at top level. An empty array that sits
  // inside an earlier `[` that is still open at its offset (a clipped
  // response, or complete JSON broken by a missing comma) is an atom's own
  // `"concepts": []`, not the model saying "nothing here", so it must not
  // turn malformed output into a zero-yield stamp.
  let firstAttempt: ReturnType<typeof parseArrayAtOffset> | null = null;
  let sawEmptyArray = false;
  let candidates = 0;
  const priorStarts: number[] = [];
  const enclosed = (at: number): boolean => priorStarts.some((s) => {
    const end = matchingCloseBracket(cleaned, s);
    return end === -1 || end > at;
  });
  for (
    let start = firstStart;
    start !== -1 && candidates < MAX_ARRAY_ANCHOR_CANDIDATES;
    start = cleaned.indexOf('[', start + 1)
  ) {
    candidates++;
    const attempt = parseArrayAtOffset(cleaned, start);
    // Captured on the FIRST iteration only — every reason string this function
    // can return still describes the first bracket, unchanged.
    if (firstAttempt === null) firstAttempt = attempt;
    const nested = enclosed(start);
    priorStarts.push(start);
    if (attempt.ok) {
      if (attempt.parsed.length === 0) { if (!nested) sawEmptyArray = true; continue; }
      const atoms = atomsFromParsedArray(attempt.parsed);
      if (atoms.length > 0) return { ok: true, atoms };
    }
  }

  // Nothing yielded a real atom. An honest `[]` anywhere is the zero-yield
  // success (#4148 keeps "found nothing" distinct from "malformed"); otherwise
  // fall back to the FIRST offset's outcome — never a later one — so the
  // failure reason the drain surfaces as `last_error` still describes the
  // first bracket.
  if (sawEmptyArray) return { ok: true, atoms: [] };
  if (firstAttempt === null) return { ok: false, reason: 'no JSON array in response' };
  if (firstAttempt.ok) return { ok: false, reason: 'array had no atom-shaped elements' };
  return firstAttempt;
}

/**
 * Back-compat wrapper: parse the response into ExtractedAtom[], returning []
 * for BOTH malformed output and a legitimate zero-yield (legacy callers/tests
 * that don't need the typed distinction — new code uses parseAtomsOutcome).
 */
export function parseAtomsResponse(raw: string): ExtractedAtom[] {
  const outcome = parseAtomsOutcome(raw);
  return outcome.ok ? outcome.atoms : [];
}

function atomsFromParsedArray(parsed: unknown[]): ExtractedAtom[] {

  const atoms: ExtractedAtom[] = [];
  for (const item of parsed) {
    if (typeof item !== 'object' || item === null) continue;
    const obj = item as Record<string, unknown>;
    const title = typeof obj.title === 'string' ? obj.title.slice(0, 200) : null;
    const atomType = typeof obj.atom_type === 'string' ? obj.atom_type.trim().toLowerCase() : null;
    const body = typeof obj.body === 'string' ? obj.body : null;
    if (!title || !atomType || !body) continue;
    if (!ATOM_TYPES.includes(atomType as typeof ATOM_TYPES[number])) continue;
    atoms.push({
      title,
      atom_type: atomType as typeof ATOM_TYPES[number],
      body,
      source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined,
      lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined,
      concepts: (() => {
        if (!Array.isArray(obj.concepts)) return undefined;
        const labels = obj.concepts
          .filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c))
          .slice(0, 3);
        return labels.length > 0 ? labels : undefined;
      })(),
      virality_score:
        typeof obj.virality_score === 'number' &&
        obj.virality_score >= 0 &&
        obj.virality_score <= 100
          ? obj.virality_score
          : undefined,
      emotional_register:
        typeof obj.emotional_register === 'string' ? obj.emotional_register : undefined,
    });
  }
  return atoms;
}