Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

8.7 KB
Raw Blame
/**
 * Pure policy for sizing the llama-server context window.
 *
 * Replaces the flat SERVER_CONTEXT_SIZE = 16384 cap (#2142), which blocked
 * every transcript over ~80 minutes on every bundled model. The cap itself was
 * not gratuitous: it fixed #1203, where an unbounded context allocated a ~15 GB
 * Metal-wired KV cache and froze a 16 GB Mac into a watchdog kernel panic. So
 * the replacement is not a bigger constant but a per-machine, per-model budget.
 *
 * Everything here is pure arithmetic so the memory guarantee can be tested
 * without a GPU, a model file, or a spawned process.
 */

// Latin text tokenizes at roughly 3.5-4.2 chars/token on the Qwen/Llama BPEs
// used here (measured: 4.16 on a synthetic English transcript). 3 leaves
// headroom, because under-estimating is the bug and over-estimating only costs
// KV bytes that the ceiling already bounds.
const LATIN_CHARS_PER_TOKEN = 3;

// CJK runs about 1-1.5 chars/token. Charging 1 keeps the estimate on the safe
// side for the ja / zh-CN / zh-TW locales the app ships.
const CJK_RANGES = [
  [0x1100, 0x11ff], // Hangul Jamo
  [0x3000, 0x303f], // CJK symbols and punctuation
  [0x3040, 0x30ff], // Hiragana + Katakana
  [0x3400, 0x4dbf], // CJK Unified Ideographs Extension A
  [0x4e00, 0x9fff], // CJK Unified Ideographs
  [0xac00, 0xd7af], // Hangul Syllables
  [0xf900, 0xfaff], // CJK Compatibility Ideographs
  [0xff00, 0xffef], // Halfwidth and Fullwidth Forms
  [0x20000, 0x3ffff], // CJK Unified Ideographs Extensions B and beyond
];

function isCjkCodePoint(codePoint) {
  for (const [start, end] of CJK_RANGES) {
    if (codePoint >= start && codePoint <= end) return true;
  }
  return false;
}

/**
 * Cheap, deliberately high estimate of how many tokens a string will occupy.
 *
 * Only used to choose the context size a server STARTS at. Once a server is
 * running, POST /tokenize returns the exact count in ~13 ms and supersedes
 * this; the estimate just avoids paying a restart in the common case.
 */
function estimateTokens(text) {
  if (typeof text !== "string" || text.length === 0) return 0;

  let cjkCount = 0;
  let otherCount = 0;
  for (const character of text) {
    if (isCjkCodePoint(character.codePointAt(0))) cjkCount += 1;
    else otherCount += 1;
  }

  return cjkCount + Math.ceil(otherCount / LATIN_CHARS_PER_TOKEN);
}

// K and V are cached as f16 by default (llama.cpp --cache-type-k/-v).
const KV_ELEMENT_BYTES = 2;

function resolveHeadCountKv(metadata) {
  const raw = metadata.headCountKv;
  // GGUF permits a per-layer array. The widest layer is the safe reading.
  if (Array.isArray(raw)) {
    const widest = raw.reduce((max, value) => (value > max ? value : max), 0);
    return widest > 0 ? widest : null;
  }
  return typeof raw === "number" && raw > 0 ? raw : null;
}

function resolveHeadWidths(metadata) {
  const { keyLength, valueLength, embeddingLength, headCount } = metadata;
  if (keyLength > 0 && valueLength > 0) return { keyLength, valueLength };

  // llama.cpp's own fallback when the widths are not declared.
  if (embeddingLength > 0 && headCount > 0) {
    const width = embeddingLength / headCount;
    if (Number.isInteger(width) && width > 0) return { keyLength: width, valueLength: width };
  }
  return null;
}

/**
 * Bytes of KV cache each token of context costs, from the model's own
 * architecture. Assumes every layer is full attention, which over-estimates
 * sliding-window models (Gemma 3/4) and models that interleave full attention
 * (Qwen3.5). Over-estimating shrinks the ceiling, so it errs safe.
 *
 * Returns null when the architecture cannot be read. Callers must treat null
 * as "fall back to the historical 16384", never as "assume it is cheap".
 */
function kvBytesPerToken(metadata) {
  if (!metadata || typeof metadata !== "object") return null;

  const blockCount = metadata.blockCount;
  if (!(blockCount > 0)) return null;

  const headCountKv = resolveHeadCountKv(metadata);
  if (headCountKv === null) return null;

  const widths = resolveHeadWidths(metadata);
  if (widths === null) return null;

  return blockCount * headCountKv * (widths.keyLength + widths.valueLength) * KV_ELEMENT_BYTES;
}

// The historical cap (#1236). Now the floor and the universal fallback rather
// than the ceiling, so no configuration that works today can get smaller.
const BASELINE_CONTEXT_SIZE = 16384;

// Half the machine, minus a fixed slice for the OS itself. Deliberately ~1.7x
// tighter than llama.cpp's own --fit, which would permit ~9.6 GiB of wired
// memory on the 16 GB Mac that panicked in #1203.
const USABLE_MEMORY_FRACTION = 0.5;
const FIXED_OS_RESERVE_BYTES = 2 * 1024 * 1024 * 1024;

// Compute buffers also grow with context: measured 144 MiB total at 16384 and
// 240 MiB at 65536 for a 9B, i.e. ~2 KiB/token plus a fixed base. Doubled for
// margin, and the base sized for the 31B-class graphs.
const COMPUTE_FIXED_BYTES = 384 * 1024 * 1024;
const COMPUTE_BYTES_PER_TOKEN = 4096;

// Hybrid models (Qwen3.5, LFM2) carry recurrent state that is not part of the
// KV cache and does not scale with context.
const RECURRENT_STATE_BYTES = 512 * 1024 * 1024;

// A discrete GPU's VRAM is not system RAM and we do not probe it. Until a real
// measurement exists, stay modest and let the existing GPU fallback ladder
// handle anything that still will not fit.
const UNVERIFIED_GPU_CEILING = 65536;

// Contexts are aligned down to a round number so that near-identical requests
// land on the same value and cannot cause a second restart.
const CONTEXT_ALIGNMENT = 512;

const MIB = 1024 * 1024;
const MIN_CACHE_RAM_MIB = 512;
const MAX_CACHE_RAM_MIB = 2048;

function clamp(value, low, high) {
  return Math.min(high, Math.max(low, value));
}

function fallbackCeiling(reason) {
  return { ceiling: BASELINE_CONTEXT_SIZE, reason, cacheRamMiB: MIN_CACHE_RAM_MIB };
}

/**
 * The largest context this machine may allocate for this model.
 *
 * Every path that cannot answer confidently returns BASELINE_CONTEXT_SIZE, so
 * the worst case of this whole mechanism is the behaviour that ships today.
 */
function resolveContextCeiling({
  totalMemoryBytes,
  weightsBytes = 0,
  draftWeightsBytes = 0,
  kvBytesPerToken,
  hasRecurrentState = false,
  trainedContextTokens = 0,
  discreteGpuUnverified = false,
}) {
  if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) {
    return fallbackCeiling("no-memory-info");
  }
  if (!Number.isFinite(kvBytesPerToken) || kvBytesPerToken <= 0) {
    return fallbackCeiling("unknown-kv-cost");
  }

  const budgetBytes = totalMemoryBytes * USABLE_MEMORY_FRACTION - FIXED_OS_RESERVE_BYTES;
  const kvBudgetBytes =
    budgetBytes -
    weightsBytes -
    draftWeightsBytes -
    COMPUTE_FIXED_BYTES -
    (hasRecurrentState ? RECURRENT_STATE_BYTES : 0);

  if (kvBudgetBytes <= 0) return fallbackCeiling("floor");

  const cacheRamMiB = clamp(
    Math.floor(kvBudgetBytes / 4 / MIB),
    MIN_CACHE_RAM_MIB,
    MAX_CACHE_RAM_MIB
  );

  const perTokenBytes = kvBytesPerToken + COMPUTE_BYTES_PER_TOKEN;
  const affordable =
    Math.floor(Math.floor(kvBudgetBytes / perTokenBytes) / CONTEXT_ALIGNMENT) * CONTEXT_ALIGNMENT;

  let hardMax = trainedContextTokens > 0 ? trainedContextTokens : Number.POSITIVE_INFINITY;
  let hardMaxReason = "trained-context-bound";
  if (discreteGpuUnverified && UNVERIFIED_GPU_CEILING < hardMax) {
    hardMax = UNVERIFIED_GPU_CEILING;
    hardMaxReason = "gpu-cap";
  }

  const ceiling = Math.min(affordable, hardMax);
  if (ceiling <= BASELINE_CONTEXT_SIZE) return fallbackCeiling("floor");

  return {
    ceiling,
    reason: ceiling === hardMax ? hardMaxReason : "memory-bound",
    cacheRamMiB,
  };
}

// Quantizing the chosen context bounds how many distinct restarts a session can
// cause: at most one per rung per model, however many differently-sized
// requests arrive.
const CONTEXT_STEPS = [16384, 32768, 65536, 131072, 262144];

/**
 * The context size to start (or grow) the server at for a request needing
 * `needed` tokens, given a caller floor and the machine's ceiling.
 *
 * The ceiling always wins: it is the memory guarantee, and it is not
 * necessarily on a rung.
 */
function resolveContextSize({ needed = 0, floor = 0, ceiling = BASELINE_CONTEXT_SIZE }) {
  const target = Math.max(needed || 0, floor || 0, BASELINE_CONTEXT_SIZE);
  const rung = CONTEXT_STEPS.find((step) => step >= target) ?? Number.POSITIVE_INFINITY;
  const bounded = Math.min(rung, Math.max(ceiling, BASELINE_CONTEXT_SIZE));
  return Math.max(bounded, BASELINE_CONTEXT_SIZE);
}

module.exports = {
  LATIN_CHARS_PER_TOKEN,
  KV_ELEMENT_BYTES,
  estimateTokens,
  kvBytesPerToken,
  resolveContextCeiling,
  resolveContextSize,
  CONTEXT_STEPS,
  BASELINE_CONTEXT_SIZE,
  USABLE_MEMORY_FRACTION,
  FIXED_OS_RESERVE_BYTES,
  COMPUTE_FIXED_BYTES,
  COMPUTE_BYTES_PER_TOKEN,
  RECURRENT_STATE_BYTES,
  UNVERIFIED_GPU_CEILING,
  CONTEXT_ALIGNMENT,
};