Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
2.9 KB
Raw Blame
import { describe, expect, test } from "bun:test";

import { estimateTokens } from "../../../../src/core/brain/text/tokenizer.ts";

// The heuristic is `ceil(utf8_bytes / 4)`, so tests assert language-
// agnostic invariants (determinism, monotonicity, non-zero on
// non-empty input) plus a handful of computed byte-based ground
// truths. No script-specific magic numbers - the implementation
// must not special-case any human language.

const utf8Len = (s: string) => new TextEncoder().encode(s).length;

describe("estimateTokens", () => {
  test("empty input returns 0", () => {
    expect(estimateTokens("")).toBe(0);
  });

  test("matches ceil(utf8_bytes / 4) for ASCII", () => {
    const s = "Use imperative voice";
    expect(estimateTokens(s)).toBe(Math.ceil(utf8Len(s) / 4));
  });

  test("matches ceil(utf8_bytes / 4) for arbitrary multibyte scripts", () => {
    // No script is special-cased: the formula must hold for every
    // input, regardless of which scripts it mixes.
    const samples = [
      "Hello", // Latin ASCII
      "café au lait", // Latin extended (NFC)
      "Тест", // Cyrillic
      "テスト", // Hiragana / Katakana
      "안녕하세요", // Hangul
      "مرحبا", // Arabic
      "สวัสดี", // Thai (with combining marks)
      "नमस्ते", // Devanagari (Hindi)
      "アイウ", // Halfwidth katakana
      "🦊🚀", // Emoji
      "Hello テスト مرحبا 🦊", // Mixed
    ];
    for (const s of samples) {
      expect(estimateTokens(s)).toBe(Math.ceil(utf8Len(s) / 4));
    }
  });

  test("non-empty input always yields at least one token", () => {
    const samples = ["a", "テ", "🦊", " ", "."];
    for (const s of samples) {
      expect(estimateTokens(s)).toBeGreaterThanOrEqual(1);
    }
  });

  test("monotonic - concatenating strings can only grow the count", () => {
    const a = "alpha beta gamma";
    const b = " delta epsilon";
    expect(estimateTokens(a + b)).toBeGreaterThanOrEqual(estimateTokens(a));
  });

  test("deterministic across runs", () => {
    const sample = "The quick brown fox jumps over the lazy dog 123 #tag テスト";
    expect(estimateTokens(sample)).toBe(estimateTokens(sample));
  });

  test("scales linearly with repetition", () => {
    const single = "word ";
    const x100 = single.repeat(100);
    // 100 repetitions of the same byte sequence yields exactly 100
    // times the byte count, so the heuristic is linear.
    expect(estimateTokens(x100)).toBe(Math.ceil(utf8Len(x100) / 4));
  });

  test("no special-case for any script - equal byte length yields equal count", () => {
    // Construct two inputs in different scripts that share a byte
    // length, then assert the heuristic ignores which script they
    // come from.
    const cyrillic = "тестт";
    const len = utf8Len(cyrillic);
    const ascii = "x".repeat(len);
    expect(utf8Len(ascii)).toBe(len);
    expect(estimateTokens(cyrillic)).toBe(estimateTokens(ascii));
  });
});