Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.3 KB
Raw Blame
/**
 * Embedding-token estimation — shared by both chunkers (#3477 follow-up).
 *
 * Moved out of code.ts so recursive.ts can measure with the same estimator:
 * code.ts imports recursive.ts, so recursive.ts could never import these from
 * code.ts without a cycle. The natural home suggested in #3477's merge review
 * was cjk.ts, but cjk.ts is a check:fuzz-purity bundle target and tiktoken's
 * loader pulls node:fs into the bundle — so the estimators live here, one
 * layer above cjk.ts (whose char classes they reuse) and below both chunkers.
 */

import { CJK_SLUG_CHARS } from '../cjk.ts';

/**
 * Default hard budget for any emitted chunk's estimated embedding tokens.
 * Shared by capOversizedChunks (code.ts, #1675) and capByChars (recursive.ts).
 * 2000 keeps a margin under the smallest common strict embedder contexts
 * (nomic-embed-text 2048 — #3037; llama-server -ub 2048 — #2826).
 */
export const DEFAULT_MAX_CHUNK_TOKENS = 2000;

// v0.19.0 (Layer 5): accurate token count via @dqbd/tiktoken cl100k_base,
// the same encoder text-embedding-3-large uses. The old len/4 heuristic was
// 2-3x off for code. Lazy-init so dev and compiled-binary both only pay
// the init cost once. Falls back to the heuristic if the encoder fails
// to load (vanishingly unlikely but keeps the chunker available).
let tiktokenEncoder: { encode: (s: string) => Uint32Array; encode_ordinary: (s: string) => Uint32Array; free: () => void } | null = null;
let tiktokenInitialized = false;

// v0.20.0 Cathedral II Layer 8 (D1) — re-exported from code.ts so
// commands/sync.ts can estimate embed cost before a --all sync blows a
// surprise OpenAI bill. Same cl100k_base tokenizer the embedding path
// actually uses, so cost estimates match actual billing within tokenizer
// noise.
function loadEncoder(): void {
  if (tiktokenInitialized) return;
  try {
    // eslint-disable-next-line @typescript-eslint/no-var-requires
    const m = require('@dqbd/tiktoken');
    tiktokenEncoder = m.get_encoding('cl100k_base');
  } catch {
    tiktokenEncoder = null;
  }
  tiktokenInitialized = true;
}

/** Whether estimateTokens counts with cl100k (false = the len/4 fallback). */
export function cl100kAvailable(): boolean {
  loadEncoder();
  return tiktokenEncoder !== null;
}

export function estimateTokens(text: string): number {
  if (!text) return 0;
  loadEncoder();
  if (!tiktokenEncoder) return Math.max(1, Math.ceil(text.length / 4));
  return pieceCounting ? countByPieces(tiktokenEncoder, text) : encodeCount(tiktokenEncoder, text);
}

function encodeCount(encoder: NonNullable<typeof tiktokenEncoder>, text: string): number {
  try {
    return encoder.encode(text).length;
  } catch {
    // Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT
    // tokenizers embed the literal "<|endoftext|>"). The default encode() uses
    // disallowed_special='all' and THROWS on those, crashing reindex-code on
    // valid source files. For a token COUNT we don't need special-token
    // semantics: re-encode treating them as ordinary text (never throws),
    // heuristic only if even that fails.
    try {
      return (
        encoder as unknown as {
          encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array;
        }
      ).encode(text, [], []).length;
    } catch {
      return Math.max(1, Math.ceil(text.length / 4));
    }
  }
}

/**
 * cl100k's pre-tokenizer pattern restricted to ASCII (\p{L} = [A-Za-z],
 * \p{N} = [0-9], \s = [\t\n\v\f\r ]). cl100k encodes each pre-token
 * independently and the count is the sum over pre-tokens, so ASCII text can be
 * split here and each pre-token's count memoized: prose repeats a small
 * vocabulary, which turns ~0.5 ms of WASM encoding per 3k-char chunk into
 * map lookups.
 */
const ASCII_PRETOKEN = /'(?:[sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD])|[^\r\nA-Za-z0-9]?[A-Za-z]+|[0-9]{1,3}| ?[^\t\n\v\f\r A-Za-z0-9]+[\r\n]*|[\t\n\v\f\r ]*[\r\n]+|[\t\n\v\f\r ]+(?![^\t\n\v\f\r ])|[\t\n\v\f\r ]+/y;

/**
 * Two cuts always end a pre-token whatever script surrounds them: a newline
 * followed by an ASCII letter or digit (no cl100k alternative continues from
 * a newline into one, and none that can end there looks ahead), and a space
 * between two ASCII letters (the letter run before it ends there and the
 * space opens the next word's pre-token). Segments between cuts count
 * independently, so only segments holding non-ASCII text go to the encoder:
 * lines first, then words inside a line that has curly quotes or accents.
 */
const LINE_CUT = /\n(?=[A-Za-z0-9])/g;
const WORD_CUT = /[A-Za-z](?= [A-Za-z])/g;
const NON_ASCII = /[^\x00-\x7f]/g;
const PRETOKEN_MEMO_MAX_ENTRIES = 50_000;
const PRETOKEN_MEMO_MAX_CHARS = 64;
const pretokenCounts = new Map<string, number>();
let pieceCounting = true;
let byteBound = true;

/**
 * Same count as encodeCount: segments and pre-tokens go through
 * encode_ordinary, which is what encodeCount's special-token retry computes
 * and what encode() returns on text without special tokens, minus encode()'s
 * ~50 µs per-call special-token regex build.
 */
function countByPieces(encoder: NonNullable<typeof tiktokenEncoder>, text: string): number {
  let total = 0;
  let encodeFrom = -1;
  const visit = (from: number, to: number, cut: RegExp, inner?: RegExp): void => {
    let nonAscii = -1;
    let start = from;
    while (start < to) {
      cut.lastIndex = start;
      const end = Math.min(to, (cut.exec(text)?.index ?? to) + 1);
      if (nonAscii < start) {
        NON_ASCII.lastIndex = start;
        nonAscii = NON_ASCII.exec(text)?.index ?? text.length;
      }
      if (nonAscii >= end) {
        if (encodeFrom !== -1) total += encoder.encode_ordinary(text.slice(encodeFrom, start)).length;
        encodeFrom = -1;
        total += countAscii(encoder, text, start, end);
      } else if (inner) {
        visit(start, end, inner);
      } else if (encodeFrom === -1) {
        encodeFrom = start;
      }
      start = end;
    }
  };
  visit(0, text.length, LINE_CUT, WORD_CUT);
  if (encodeFrom !== -1) total += encoder.encode_ordinary(text.slice(encodeFrom)).length;
  return total;
}

function countAscii(encoder: NonNullable<typeof tiktokenEncoder>, text: string, start: number, end: number): number {
  let total = 0;
  ASCII_PRETOKEN.lastIndex = start;
  while (ASCII_PRETOKEN.lastIndex < end) {
    const piece = ASCII_PRETOKEN.exec(text)![0];
    let count = pretokenCounts.get(piece);
    if (count === undefined) {
      count = encoder.encode_ordinary(piece).length;
      if (piece.length <= PRETOKEN_MEMO_MAX_CHARS) {
        if (pretokenCounts.size >= PRETOKEN_MEMO_MAX_ENTRIES) pretokenCounts.clear();
        pretokenCounts.set(piece, count);
      }
    }
    total += count;
  }
  return total;
}

/** Test seam: false counts every text with one whole-text encode (the reference path). */
export const __testing = {
  setPieceCounting(on: boolean): void {
    pieceCounting = on;
  },
  /** false makes fitsEmbedTokens never vouch, so every cap decision counts (the reference path). */
  setByteBound(on: boolean): void {
    byteBound = on;
  },
};

const CJK_CHARS_G = new RegExp(`[${CJK_SLUG_CHARS}]`, 'g');

/**
 * Embedding-safe token estimate for the oversize caps. estimateTokens
 * (cl100k) matches embedding-family tokenizers closely on pure-ASCII source
 * (measured identical on English prose and JSON vs Qwen3-Embedding), but
 * UNDERCOUNTS mixed CJK+ASCII chunks — measured −31% on URL-dense Korean
 * text vs the Qwen3 embedding tokenizer, which is exactly the shape that
 * overflows strict embedding backends (#2826). For chunks containing CJK,
 * take the max of cl100k and a per-char-class overestimate (CJK 1.0/char,
 * other non-whitespace 0.75/char, whitespace 0.1/char). CJK-DOMINANT text
 * is unaffected too: cl100k already counts it above the weighted form, so
 * max() returns the same value as today. Only mixed-script chunks — the
 * measured divergence class — estimate higher.
 */
export function estimateEmbedTokens(text: string): number {
  const cjk = (text.match(CJK_CHARS_G) || []).length;
  if (cjk === 0) return estimateTokens(text);
  return Math.max(estimateTokens(text), weightedTokens(text, cjk));
}

/**
 * True when `text` provably fits `maxTokens` under estimateEmbedTokens without
 * counting: its UTF-8 byte length is an upper bound on both halves. cl100k is
 * a byte-level BPE, so every token covers at least one byte (the len/4
 * fallback is smaller still), and the weighted form charges at most 1 per
 * UTF-16 unit while every unit encodes to at least one byte. A typical CJK
 * chunk (~350 chars, ~1 KB) fits without touching the encoder.
 */
export function fitsEmbedTokens(text: string, maxTokens: number): boolean {
  return byteBound && (text.length * 3 <= maxTokens || Buffer.byteLength(text, 'utf8') <= maxTokens);
}

/** The per-char-class overestimate half of estimateEmbedTokens. Linear (two
 *  regex scans), unlike the cl100k encoder — see estimateEmbedTokensCeiling. */
function weightedTokens(text: string, cjk: number): number {
  const ws = (text.match(/\s/g) || []).length;
  return Math.ceil(cjk + (text.length - cjk - ws) * 0.75 + ws * 0.1);
}

/**
 * Upper bound on what `text` contributes to `estimateEmbedTokens(text + rest)`
 * for ANY `rest` — i.e. the figure to RESERVE when a fragment will be glued
 * onto a body whose script mix is not yet known.
 *
 * estimateEmbedTokens is super-additive across a mixed-script join. It only
 * switches to the weighted branch when the text it is handed contains CJK, so
 * a pure-ASCII fragment measured ALONE costs cl100k (a 59-char structured
 * chunk header = 17 tokens) while the SAME fragment inside a chunk whose body
 * contains CJK costs ~0.75/char on the weighted branch (~42 tokens). Reserving
 * the standalone figure under-counts ~2.5x, and capOversizedChunks then emits
 * body-capped pieces that re-emerge over the cap once the header is re-added
 * (measured: 2,006- and 2,023-token chunks on src/core/migrate.ts against a
 * 2,000 cap, where the pre-#3564 chunker emitted none).
 *
 * Taking max(cl100k, weighted) unconditionally is a true bound because the
 * weighted form is additive per char class, so weighted(a + b) <=
 * weighted(a) + weighted(b), and cl100k does not gain tokens across the
 * header's trailing blank line.
 */
export function estimateEmbedTokensCeiling(text: string): number {
  return Math.max(estimateTokens(text), weightedTokens(text, (text.match(CJK_CHARS_G) || []).length));
}