Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
2.2 KB
Raw Blame
/**
 * BPE token estimate for the OpenAI-family tokenizer.
 *
 * Healthy estimates use bounded cl100k_base encoding. Encoder failures use
 * the full UTF-8 byte length, including zero for empty text, without dividing
 * by a guessed bytes-per-token ratio.
 *
 * The cl100k_base encoder is loaded lazily on first use, from the same
 * `get_encoding` helper `OpenAITokenCounter` already imports, and reused for
 * the life of the process. A load or encode failure never throws and never
 * aborts startup: the exact byte length is returned, one bounded metric
 * counter and one structured warning record the reason. A failed initial load
 * keeps using the byte bound for this process rather than repeatedly loading
 * an unavailable encoder.
 */
import { type Tiktoken, get_encoding } from 'tiktoken'
import { logger } from '../../logger'
import { encodeLength } from './encodeLength'
import { tokenizerFallbackTotal } from './metrics'

let encoder: Tiktoken | null = null
let encoderLoadFailed = false
let warned = false

/** Fixed reason values: never an exception name and never text content. */
export type BpeFallbackReason = 'encoder_load_failed' | 'encode_failed'

function warnOnce(reason: BpeFallbackReason): void {
  if (warned) return
  warned = true
  logger.warn(
    { component: 'BpeEstimate', reason },
    'BPE token estimate unavailable; using UTF-8 byte length'
  )
}

function getEncoder(): Tiktoken | null {
  if (encoder) return encoder
  if (encoderLoadFailed) return null
  try {
    encoder = get_encoding('cl100k_base')
    return encoder
  } catch {
    encoderLoadFailed = true
    tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'bpe_estimate_failed' })
    warnOnce('encoder_load_failed')
    return null
  }
}

/**
 * Token estimate for `text` under cl100k_base, with the exact UTF-8 byte
 * length as the documented conservative fallback.
 */
export function bpeTokenEstimate(text: string): number {
  if (text.length === 0) return 0
  const encoderOrNull = getEncoder()
  if (!encoderOrNull) return Buffer.byteLength(text, 'utf8')
  try {
    return encodeLength(encoderOrNull, text)
  } catch {
    tokenizerFallbackTotal.inc({ provider: 'openai', reason: 'bpe_estimate_failed' })
    warnOnce('encode_failed')
    return Buffer.byteLength(text, 'utf8')
  }
}