Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.7 KB
Raw Blame
import { PageRevisionConflictError } from './page-state/types.ts';
import { readProjectionSnapshot, installPageProjection } from './page-state/projections.ts';
/**
 * SUP-3874 — heal already-stored chunks that exceed the active embedding
 * model's per-input token limit.
 *
 * New imports respect `resolveMaxChunkTokens()` (#4530 + mxbai recipe cap),
 * but `gbrain embed --stale` re-embeds EXISTING `chunk_text` rows. Pages
 * chunked under the historical 2000-token default (or before a model-specific
 * `max_input_tokens` was declared) can still carry a single oversized row that
 * fails forever with "input length exceeds the context length" and makes the
 * whole stale sweep exit non-zero — even when sibling chunks embed cleanly.
 *
 * This helper SPLITS those oversized rows in place (never truncates) and
 * leaves embeddings unset. `upsertChunks` preserves vectors for rows whose
 * `(chunk_index, chunk_text)` pair is unchanged, and NULLs vectors when the
 * text at an index changes — so split pieces and shifted siblings become
 * stale for the next embed pass. Shared by CLI embed + embed-stale.
 */

import { chunkText } from './chunkers/recursive.ts';
import { fenceTrustMarker, markFenceChunk, unmarkFenceChunk } from './eligibility/fence-overlay.ts';
import { estimateEmbedTokens } from './chunkers/token-estimate.ts';
import { resolveMaxChunkTokens } from './embedding-input-limit.ts';
import type { BrainEngine } from './engine.ts';
import type { Chunk, ChunkInput, StaleChunkRow } from './types.ts';

export interface HealOversizedChunksResult {
  /** True when at least one chunk was split. */
  changed: boolean;
  /** How many original chunks exceeded the cap. */
  splitCount: number;
  /** Contiguous, re-indexed chunk list ready for upsertChunks. */
  chunks: ChunkInput[];
}

type HealableChunk = Pick<
  Chunk,
  | 'chunk_index'
  | 'chunk_text'
  | 'chunk_source'
  | 'token_count'
  | 'modality'
  | 'language'
  | 'symbol_name'
  | 'symbol_type'
  | 'start_line'
  | 'end_line'
  | 'parent_symbol_path'
  | 'doc_comment'
  | 'symbol_name_qualified'
>;

function carryHealedMetadata(chunk: HealableChunk, base: ChunkInput): ChunkInput {
  return {
    ...base,
    modality: chunk.modality ?? undefined,
    language: chunk.language ?? undefined,
    symbol_name: chunk.symbol_name ?? undefined,
    symbol_type: chunk.symbol_type ?? undefined,
    start_line: chunk.start_line ?? undefined,
    end_line: chunk.end_line ?? undefined,
    parent_symbol_path: chunk.parent_symbol_path ?? undefined,
    doc_comment: chunk.doc_comment ?? undefined,
    symbol_name_qualified: chunk.symbol_name_qualified ?? undefined,
  };
}

/**
 * Split any chunk whose estimated embed tokens exceed `maxTokens`.
 *
 * Embeddings are intentionally omitted — upsertChunks COALESCEs the stored
 * vector when chunk_text at that index is unchanged, and clears it when the
 * text changes (split / shifted siblings). Every split piece retains its
 * source chunk's modality and code-symbol metadata so a later provider failure
 * cannot persist a lossy replacement.
 */
export function healOversizedChunks(
  chunks: ReadonlyArray<HealableChunk>,
  maxTokens: number = resolveMaxChunkTokens(),
): HealOversizedChunksResult {
  const out: ChunkInput[] = [];
  let splitCount = 0;

  for (const c of chunks) {
    const tokens = estimateEmbedTokens(c.chunk_text);
    if (tokens <= maxTokens) {
      // Metadata fields are optional: upsertChunks COALESCEs them when
      // chunk_text at this index is unchanged, so omitting them is safe.
      out.push(carryHealedMetadata(c, {
        chunk_index: out.length,
        chunk_text: c.chunk_text,
        chunk_source: c.chunk_source,
        token_count: c.token_count || Math.ceil(c.chunk_text.length / 4),
      }));
      continue;
    }

    splitCount++;
    // #5575 ENG-1: a low-tier facts-fence chunk keeps its trust marker on every piece.
    const { tier, unconfirmed, body } = unmarkFenceChunk(c.chunk_text);
    const room = tier ? Math.max(1, maxTokens - estimateEmbedTokens(fenceTrustMarker(tier, unconfirmed)) - 1) : maxTokens;
    const parts = chunkText(body, { maxTokens: room });
    // Pathological: estimator / splitter disagreement — hard-split by chars
    // so we never re-emit the original oversized row unchanged.
    const pieces =
      parts.length > 1 || (parts[0] && estimateEmbedTokens(parts[0].text) <= room)
        ? parts.map((p) => p.text)
        : hardSplitByChars(body, room);

    for (const piece of pieces) {
      if (!piece) continue;
      const text = tier ? markFenceChunk(tier, piece, unconfirmed) : piece;
      out.push(carryHealedMetadata(c, {
        chunk_index: out.length,
        chunk_text: text,
        chunk_source: c.chunk_source,
        token_count: Math.ceil(text.length / 4),
      }));
    }
  }

  return {
    changed: splitCount > 0,
    splitCount,
    chunks: out,
  };
}

/** Last-resort splitter when chunkText cannot shrink a pathological blob. */
function hardSplitByChars(text: string, maxTokens: number): string[] {
  // ~4 chars/token under the heuristic; stay under the estimate with margin.
  const maxChars = Math.max(64, Math.floor(maxTokens * 3));
  const out: string[] = [];
  for (let i = 0; i < text.length; i += maxChars) {
    out.push(text.slice(i, i + maxChars));
  }
  return out.length > 0 ? out : [text];
}

/**
 * Remap a post-heal chunk list back into the stale-row shape the `--stale`
 * drains iterate (only rows still needing embeddings). Shared by
 * `src/commands/embed.ts:embedAllStale` and
 * `src/core/embed-stale.ts:embedStaleForSource` so the two drains cannot
 * drift.
 *
 * `chunk_source` passes through UNCHANGED: coercing `fenced_code` to
 * `compiled_truth` here would make `wrapChunkTextsForStoredMode` prepend a
 * contextual prefix to code chunks, violating the D20-T4 never-wrap
 * convention (`src/core/embedding-context.ts`).
 */
export function healedChunksToStaleRows(
  chunks: ReadonlyArray<Chunk>,
  slug: string,
  sourceId: string,
): StaleChunkRow[] {
  return chunks
    .filter((c) => !c.embedded_at || c.embedding_is_null === true)
    .map((c) => ({
      page_id: c.page_id,
      chunk_index: c.chunk_index,
      chunk_text: c.chunk_text,
      chunk_source: c.chunk_source,
      model: c.model ?? null,
      token_count: c.token_count ?? null,
      slug,
      source_id: sourceId,
    }));
}

/**
 * Load a page's chunks, split any that exceed the active model cap, upsert,
 * and re-load. No-op when every chunk already fits.
 */
export async function healOversizedPageChunks(
  engine: BrainEngine,
  slug: string,
  opts: {
    sourceId?: string;
    maxTokens?: number;
    onSplit?: (splitCount: number) => void;
  } = {},
): Promise<{ changed: boolean; splitCount: number; chunks: Chunk[] }> {
  const sourceId = opts.sourceId ?? 'default';
  const getOpts = { sourceId };
  const prepared = await readProjectionSnapshot(engine, slug, sourceId, { maxChunkTokens: opts.maxTokens, requireLiveSource: true });
  if (!prepared) return { changed: false, splitCount: 0, chunks: [] };
  const existing = prepared.chunks;
  const healed = healOversizedChunks(existing, prepared.maxChunkTokens);
  if (!healed.changed) {
    return { changed: false, splitCount: 0, chunks: existing };
  }
  // The final compare and replacement share the same page guard.
  try {
    await installPageProjection(engine, prepared, healed.chunks, { seal: true, preserveEmbeddings: true });
  } catch (error) {
    if (!(error instanceof PageRevisionConflictError)) throw error;
    return { changed: false, splitCount: 0, chunks: await engine.getChunks(slug, getOpts) };
  }
  opts.onSplit?.(healed.splitCount);
  const refreshed = await engine.getChunks(slug, getOpts);
  return { changed: true, splitCount: healed.splitCount, chunks: refreshed };
}