Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.6 KB
Raw Blame
import type { ChunkInput } from './types.ts';
import type { BrainEngine } from './engine.ts';
import { chunkText } from './chunkers/recursive.ts';
import { chunkCodeText, detectCodeLanguage } from './chunkers/code.ts';
import { sanitizeRemoteBody } from './remote-body.ts';
import { credentialSafeProjection } from './credential-projection.ts';
import { scanFencedBlocks, MAX_FENCES_PER_PAGE } from './fence-scan.ts';
import { isQuarantined } from './quarantine.ts';
import { resolveMaxChunkTokens } from './embedding-input-limit.ts';
import { estimateEmbedTokens } from './chunkers/token-estimate.ts';
import { fenceTrustMarker, loadFenceChunkOverlay, markFenceChunk, splitFenceOverlay, type FenceChunkOverlay } from './eligibility/fence-overlay.ts';

/** Recognized fence tags select the existing code grammar; unknown tags stay prose. */
const FENCE_TAG_TO_PSEUDO_PATH: Record<string, string> = {
  ts: 'fence.ts', typescript: 'fence.ts',
  tsx: 'fence.tsx',
  js: 'fence.js', javascript: 'fence.js',
  jsx: 'fence.jsx',
  py: 'fence.py', python: 'fence.py',
  rb: 'fence.rb', ruby: 'fence.rb',
  go: 'fence.go', golang: 'fence.go',
  rs: 'fence.rs', rust: 'fence.rs',
  java: 'fence.java',
  'c#': 'fence.cs', cs: 'fence.cs', csharp: 'fence.cs',
  cpp: 'fence.cpp', 'c++': 'fence.cpp',
  c: 'fence.c',
  php: 'fence.php',
  swift: 'fence.swift',
  kt: 'fence.kt', kotlin: 'fence.kt',
  scala: 'fence.scala',
  lua: 'fence.lua',
  ex: 'fence.ex', elixir: 'fence.ex',
  elm: 'fence.elm',
  ml: 'fence.ml', ocaml: 'fence.ml',
  dart: 'fence.dart',
  zig: 'fence.zig',
  sol: 'fence.sol', solidity: 'fence.sol',
  sh: 'fence.sh', bash: 'fence.sh', shell: 'fence.sh', zsh: 'fence.sh',
  css: 'fence.css',
  html: 'fence.html',
  vue: 'fence.vue',
  json: 'fence.json',
  yaml: 'fence.yaml', yml: 'fence.yaml',
  toml: 'fence.toml',
};

function fenceTagToPseudoPath(lang: string | undefined): string | null {
  if (!lang) return null;
  return FENCE_TAG_TO_PSEUDO_PATH[lang.toLowerCase().trim()] ?? null;
}

const positiveEnv = (name: string, fallback: number) => {
  const n = Number(process.env[name]);
  return Number.isFinite(n) && n > 0 ? n : fallback;
};
/** Per-fence tree-sitter cap during a page write (GBRAIN_FENCE_PARSE_TIMEOUT_MS, default 2 s). */
export const fenceParseTimeoutMs = () => positiveEnv('GBRAIN_FENCE_PARSE_TIMEOUT_MS', 2_000);
/** All fences of one page together (GBRAIN_FENCE_PARSE_BUDGET_MS, default 5 s), well inside the 30 s preparation budget. */
export const fenceParseBudgetMs = () => positiveEnv('GBRAIN_FENCE_PARSE_BUDGET_MS', 5_000);

/** Sanitize the complete body before scanning any fenced code. */
function extractFencedChunks(markdown: string, startChunkIndex: number): Promise<ChunkInput[]> {
  return fencedCodeChunks(credentialSafeProjection(sanitizeRemoteBody(markdown)), startChunkIndex);
}

/**
 * The fence cap (MAX_FENCES_PER_PAGE) and parse budget one page shares. A
 * segmented page passes one budget to every segment's fencedCodeChunks call,
 * so the caps stay per page.
 */
export interface FenceChunkBudget { fences: number; spentMs: number }

export function newFenceChunkBudget(): FenceChunkBudget {
  return { fences: MAX_FENCES_PER_PAGE, spentMs: 0 };
}

/** Recognized code-fence chunks of an already projected (sanitized, credential-safe) text. */
export async function fencedCodeChunks(markdown: string, startChunkIndex: number,
  budget: FenceChunkBudget = newFenceChunkBudget()): Promise<ChunkInput[]> {
  const out: ChunkInput[] = [];
  // Fast path: most pages (prose, tables, converted docs) contain no code
  // fence at all, so there is nothing for this function to extract — skip
  // even the line split when no fence marker (``` or ~~~) is present.
  // The `\r` in the line-start class mirrors the scanner's `\r\n|\r → \n`
  // line splitting, so CR/CRLF-only documents don't lose a real fence.
  if (!/(^|[\r\n])[ \t]{0,3}(```|~~~)/.test(markdown)) return out;

  const { fences, capped } = scanFencedBlocks(markdown, budget.fences);
  budget.fences -= fences.length;
  if (capped) {
    console.warn(
      `[gbrain] markdown fence cap hit (${MAX_FENCES_PER_PAGE} fences/page); skipping additional fences. ` +
      `Override via GBRAIN_MAX_FENCES_PER_PAGE env var.`,
    );
  }

  let indexOffset = 0;
  const started = performance.now() - budget.spentMs;
  for (const fence of fences) {
    const text = fence.text.trim();
    if (!text) continue;
    const pseudoPath = fenceTagToPseudoPath(fence.lang);
    if (!pseudoPath) continue; // unknown or missing lang tag → prose fallback
    const lang = detectCodeLanguage(pseudoPath);
    if (!lang) continue;
    // A fence is part of a page write, which has its own preparation budget:
    // each fence parses under a short cap (falling back to text chunks), and
    // once the page's fence budget is spent the remaining fences stay prose
    // (their text is still in the page's body chunks).
    const remaining = fenceParseBudgetMs() - (performance.now() - started);
    if (remaining <= 0) {
      console.warn(`[gbrain] fence parse budget (${fenceParseBudgetMs()}ms/page) spent; remaining fenced code stays in the prose chunks.`);
      break;
    }
    try {
      const chunks = await chunkCodeText(text, pseudoPath, { timeoutMs: Math.max(1, Math.min(fenceParseTimeoutMs(), remaining)) });
      for (const c of chunks) {
        out.push({
          chunk_index: startChunkIndex + indexOffset++,
          chunk_text: c.text,
          chunk_source: 'fenced_code',
          language: c.metadata.language,
          symbol_name: c.metadata.symbolName || undefined,
          symbol_type: c.metadata.symbolType,
          start_line: c.metadata.startLine,
          end_line: c.metadata.endLine,
        });
      }
    } catch (e: unknown) {
      // One fence failing shouldn't sink the page. Log + continue.
      console.warn(
        `[gbrain] fence extraction failed for lang=${fence.lang}: ${e instanceof Error ? e.message : String(e)}`,
      );
    }
  }
  budget.spentMs = performance.now() - started;
  return out;
}


/**
 * Truth then timeline prose chunks through a fence overlay, plus the
 * overlay's low-tier facts-fence rows as trailing chunks, each piece led by
 * the fence trust marker naming the row tier (#5575 ENG-1). Prose chunks
 * carry their projected UTF-8 span (v244), the coordinates windowed reads use. `fenced` slots
 * the recognized code-fence chunks of the overlaid truth before them.
 */
async function overlaidChunks(page: { compiled_truth: string; timeline?: string | null }, maxTokens: number,
  overlay: FenceChunkOverlay | undefined, fenced?: (truth: string, start: number) => Promise<ChunkInput[]>): Promise<ChunkInput[]> {
  const chunks: ChunkInput[] = [];
  const truth = splitFenceOverlay(page.compiled_truth, overlay);
  const timeline = splitFenceOverlay(page.timeline ?? '', overlay);
  for (const [text, source] of [[truth.main, 'compiled_truth'], [timeline.main, 'timeline']] as const) {
    if (!text.trim()) continue;
    for (const c of chunkText(text, { maxTokens, spans: true })) {
      chunks.push({ chunk_index: chunks.length, chunk_text: c.text, chunk_source: source,
        ...(c.byte_start !== undefined && c.byte_end !== undefined ? { projected_byte_start: c.byte_start, projected_byte_end: c.byte_end } : {}) });
    }
  }
  if (fenced && truth.main.trim()) chunks.push(...await fenced(truth.main, chunks.length));
  for (const { tier, unconfirmed, body } of [...truth.lowTier, ...timeline.lowTier]) {
    const room = Math.max(1, maxTokens - estimateEmbedTokens(fenceTrustMarker(tier, unconfirmed)) - 1);
    for (const c of chunkText(body, { maxTokens: room })) {
      chunks.push({ chunk_index: chunks.length, chunk_text: markFenceChunk(tier, c.text, unconfirmed), chunk_source: 'compiled_truth' });
    }
  }
  return chunks;
}

/**
 * Provider-free Markdown projection shared by canonical imports and DB-only
 * rebuilds. The caller owns content/disposition policy and the token limit;
 * chunkers enforce the full-body privacy boundary before splitting. Order and
 * metadata match the importer: truth, timeline, then recognized truth fences.
 * A fence overlay (loadFenceChunkOverlay) leaves held and purged fence rows
 * out and moves rows below the page tier into trailing marked chunks; without
 * one the output is unchanged.
 */
export async function prepareMarkdownChunks(page: {
  compiled_truth: string;
  timeline?: string;
  frontmatter?: Record<string, unknown> | null;
}, maxChunkTokens?: number, overlay?: FenceChunkOverlay): Promise<ChunkInput[]> {
  // A quarantined page has no live chunks. An embed_skip page keeps its chunks (keyword search) and gets no vectors:
  // every embedding path skips it (#6722), so a page under the ceiling never drops out of search for its size.
  if (isQuarantined(page.frontmatter)) return [];
  return overlaidChunks(page, maxChunkTokens ?? resolveMaxChunkTokens(), overlay, extractFencedChunks);
}

/** Truth and timeline prose only (no code-fence chunks), through the overlay: embed's chunkless-page rebuilds. */
export function prepareProseChunks(page: { compiled_truth: string; timeline?: string | null }, maxChunkTokens: number,
  overlay?: FenceChunkOverlay): Promise<ChunkInput[]> {
  return overlaidChunks(page, maxChunkTokens, overlay);
}

/** The importer's projection of one page: its fence overlay read in one query, then prepareMarkdownChunks. */
export async function prepareFenceAwareMarkdownChunks(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, slug: string,
  page: Parameters<typeof prepareMarkdownChunks>[0]): Promise<ChunkInput[]> {
  return prepareMarkdownChunks(page, undefined, await loadFenceChunkOverlay(engine, { sourceId, slug, compiled_truth: page.compiled_truth, timeline: page.timeline }));
}