Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): tokenizer crate with tiktoken-rs + HuggingFace + estimator

chopratejas9ce1c01b87a6
diff --git a/Cargo.lock b/Cargo.lock
index 069fdc13b..3f1e6793d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,20 @@
# It is not intended for manual editing.
version = 3
+[[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "getrandom 0.3.4",
+ "once_cell",
+ "serde",
+ "version_check",
+ "zerocopy",
+]
+
[[package]]
name = "aho-corasick"
version = "1.1.4"
@@ -11,6 +25,12 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
[[package]]
name = "anstream"
version = "1.0.0"
@@ -109,7 +129,7 @@ dependencies = [
"async-trait",
"axum-core",
"axum-macros",
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-util",
"http",
@@ -170,12 +190,33 @@ dependencies = [
"syn",
]
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "bit-set"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+
[[package]]
name = "bitflags"
version = "2.11.1"
@@ -191,6 +232,17 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "bstr"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.2"
@@ -215,6 +267,21 @@ version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
+[[package]]
+name = "castaway"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
+dependencies = [
+ "rustversion",
+]
+
[[package]]
name = "cc"
version = "1.2.60"
@@ -237,6 +304,33 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
[[package]]
name = "clap"
version = "4.6.1"
@@ -283,6 +377,34 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+[[package]]
+name = "compact_str"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "serde",
+ "static_assertions",
+]
+
+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "unicode-width",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -292,6 +414,73 @@ dependencies = [
"libc",
]
+[[package]]
+name = "criterion"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "is-terminal",
+ "itertools 0.10.5",
+ "num-traits",
+ "once_cell",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools 0.10.5",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
[[package]]
name = "crypto-common"
version = "0.1.7"
@@ -302,6 +491,50 @@ dependencies = [
"typenum",
]
+[[package]]
+name = "darling"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "dary_heap"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "data-encoding"
version = "2.11.0"
@@ -326,6 +559,37 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
+[[package]]
+name = "derive_builder"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
+dependencies = [
+ "derive_builder_core",
+ "syn",
+]
+
[[package]]
name = "digest"
version = "0.10.7"
@@ -347,6 +611,18 @@ dependencies = [
"syn",
]
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -363,6 +639,32 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "esaxx-rs"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "fancy-regex"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
+dependencies = [
+ "bit-set",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
@@ -547,6 +849,17 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "half"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+ "zerocopy",
+]
+
[[package]]
name = "hashbrown"
version = "0.15.5"
@@ -567,9 +880,13 @@ name = "headroom-core"
version = "0.1.0"
dependencies = [
"bytes",
+ "criterion",
+ "proptest",
"serde",
"serde_json",
"thiserror 1.0.69",
+ "tiktoken-rs",
+ "tokenizers",
"tracing",
]
@@ -734,7 +1051,7 @@ version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-channel",
"futures-util",
@@ -839,6 +1156,12 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
[[package]]
name = "idna"
version = "1.1.0"
@@ -872,6 +1195,19 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "indicatif"
+version = "0.17.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+dependencies = [
+ "console",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+ "web-time",
+]
+
[[package]]
name = "indoc"
version = "2.0.7"
@@ -897,12 +1233,41 @@ dependencies = [
"serde",
]
+[[package]]
+name = "is-terminal"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
@@ -939,6 +1304,12 @@ version = "0.2.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
+[[package]]
+name = "linux-raw-sys"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
+
[[package]]
name = "litemap"
version = "0.8.2"
@@ -957,6 +1328,22 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+[[package]]
+name = "macro_rules_attribute"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "paste",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
+
[[package]]
name = "matchers"
version = "0.2.0"
@@ -993,6 +1380,12 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
[[package]]
name = "mio"
version = "1.2.0"
@@ -1004,6 +1397,38 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "monostate"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"
+dependencies = [
+ "monostate-impl",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "monostate-impl"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -1013,6 +1438,15 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
[[package]]
name = "num_cpus"
version = "1.17.0"
@@ -1023,6 +1457,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -1035,6 +1475,40 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+[[package]]
+name = "onig"
+version = "6.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
+[[package]]
+name = "oorandom"
+version = "11.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -1047,6 +1521,40 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
[[package]]
name = "portable-atomic"
version = "1.13.1"
@@ -1090,6 +1598,25 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "proptest"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
+dependencies = [
+ "bit-set",
+ "bit-vec",
+ "bitflags",
+ "num-traits",
+ "rand 0.9.4",
+ "rand_chacha 0.9.0",
+ "rand_xorshift",
+ "regex-syntax",
+ "rusty-fork",
+ "tempfile",
+ "unarray",
+]
+
[[package]]
name = "pyo3"
version = "0.22.6"
@@ -1153,6 +1680,12 @@ dependencies = [
"syn",
]
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
[[package]]
name = "quinn"
version = "0.11.9"
@@ -1164,7 +1697,7 @@ dependencies = [
"pin-project-lite",
"quinn-proto",
"quinn-udp",
- "rustc-hash",
+ "rustc-hash 2.1.2",
"rustls",
"socket2",
"thiserror 2.0.18",
@@ -1184,7 +1717,7 @@ dependencies = [
"lru-slab",
"rand 0.9.4",
"ring",
- "rustc-hash",
+ "rustc-hash 2.1.2",
"rustls",
"rustls-pki-types",
"slab",
@@ -1288,6 +1821,46 @@ dependencies = [
"getrandom 0.3.4",
]
+[[package]]
+name = "rand_xorshift"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
+dependencies = [
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-cond"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
+dependencies = [
+ "either",
+ "itertools 0.14.0",
+ "rayon",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
[[package]]
name = "regex"
version = "1.12.3"
@@ -1323,7 +1896,7 @@ version = "0.12.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
@@ -1373,12 +1946,31 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
[[package]]
name = "rustc-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+[[package]]
+name = "rustix"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "rustls"
version = "0.23.39"
@@ -1420,12 +2012,33 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+[[package]]
+name = "rusty-fork"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2"
+dependencies = [
+ "fnv",
+ "quick-error",
+ "tempfile",
+ "wait-timeout",
+]
+
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
[[package]]
name = "semver"
version = "1.0.28"
@@ -1556,12 +2169,30 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "spm_precompiled"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
+dependencies = [
+ "base64 0.13.1",
+ "nom",
+ "serde",
+ "unicode-segmentation",
+]
+
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
[[package]]
name = "strsim"
version = "0.11.1"
@@ -1611,6 +2242,19 @@ version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+[[package]]
+name = "tempfile"
+version = "3.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
+dependencies = [
+ "fastrand",
+ "getrandom 0.4.2",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -1660,6 +2304,21 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "tiktoken-rs"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bstr",
+ "fancy-regex",
+ "lazy_static",
+ "regex",
+ "rustc-hash 1.1.0",
+]
+
[[package]]
name = "tinystr"
version = "0.8.3"
@@ -1670,6 +2329,16 @@ dependencies = [
"zerovec",
]
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "tinyvec"
version = "1.11.0"
@@ -1685,6 +2354,40 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+[[package]]
+name = "tokenizers"
+version = "0.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
+dependencies = [
+ "ahash",
+ "aho-corasick",
+ "compact_str",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom 0.3.4",
+ "indicatif",
+ "itertools 0.14.0",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand 0.9.4",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror 2.0.18",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
[[package]]
name = "tokio"
version = "1.52.1"
@@ -1917,18 +2620,51 @@ version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+[[package]]
+name = "unicode-normalization-alignments"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
+dependencies = [
+ "smallvec",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
+
+[[package]]
+name = "unicode-width"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
+
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
[[package]]
name = "unindent"
version = "0.2.4"
@@ -1994,6 +2730,25 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+[[package]]
+name = "wait-timeout"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
[[package]]
name = "want"
version = "0.3.1"
@@ -2167,6 +2922,15 @@ dependencies = [
"rustls-pki-types",
]
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "windows-link"
version = "0.2.1"
@@ -2182,6 +2946,15 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-sys"
version = "0.60.2"
@@ -2336,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"
dependencies = [
"assert-json-diff",
- "base64",
+ "base64 0.22.1",
"deadpool",
"futures",
"http",
diff --git a/Cargo.toml b/Cargo.toml
index 878fcdd90..80192ad8b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,7 +19,7 @@ default-members = [
[workspace.package]
edition = "2021"
-rust-version = "1.78"
+rust-version = "1.80"
license = "Apache-2.0"
repository = "https://github.com/chopratejas/headroom"
authors = ["Headroom Maintainers"]
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml
index 366a13872..fa3617696 100644
--- a/crates/headroom-core/Cargo.toml
+++ b/crates/headroom-core/Cargo.toml
@@ -13,3 +13,16 @@ serde_json = { workspace = true }
bytes = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
+tiktoken-rs = "0.11"
+# `tokenizers` is the HuggingFace pure-Rust tokenizer crate. Default features
+# pull in `onig` for the BPE pre-tokenizer regex; that vendors oniguruma so it
+# builds without a system dep on macOS/Linux.
+tokenizers = "0.21"
+
+[dev-dependencies]
+proptest = "1"
+criterion = { version = "0.5", features = ["html_reports"] }
+
+[[bench]]
+name = "tokenizer"
+harness = false
diff --git a/crates/headroom-core/benches/tokenizer.rs b/crates/headroom-core/benches/tokenizer.rs
new file mode 100644
index 000000000..c75831a70
--- /dev/null
+++ b/crates/headroom-core/benches/tokenizer.rs
@@ -0,0 +1,52 @@
+//! Throughput benchmark for `headroom_core::tokenizer`.
+//!
+//! Measures the tiktoken-rs–backed counter on a small / medium / large input.
+//! Used as a baseline; future stages can compare against this to catch
+//! regressions when we change tokenizer backends or add caching layers.
+
+use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion, Throughput};
+use headroom_core::tokenizer::{TiktokenCounter, Tokenizer};
+
+fn bench_count_text(c: &mut Criterion) {
+ let counter = TiktokenCounter::for_model("gpt-4o-mini").expect("init");
+
+ // Small: a typical short prompt.
+ let small = "Reply with exactly: PONG";
+ // Medium: a typical chat turn (~1KB).
+ let medium = "the quick brown fox jumps over the lazy dog\n".repeat(25);
+ // Large: a long context (~64KB) — stresses BPE inner loops.
+ let large = "the quick brown fox jumps over the lazy dog\n".repeat(1500);
+
+ let mut group = c.benchmark_group("tokenizer/count_text");
+ group.throughput(Throughput::Bytes(small.len() as u64));
+ group.bench_function("small", |b| {
+ b.iter_batched(
+ || small,
+ |s| black_box(counter.count_text(s)),
+ BatchSize::SmallInput,
+ )
+ });
+
+ group.throughput(Throughput::Bytes(medium.len() as u64));
+ group.bench_function("medium", |b| {
+ b.iter_batched(
+ || medium.as_str(),
+ |s| black_box(counter.count_text(s)),
+ BatchSize::SmallInput,
+ )
+ });
+
+ group.throughput(Throughput::Bytes(large.len() as u64));
+ group.bench_function("large", |b| {
+ b.iter_batched(
+ || large.as_str(),
+ |s| black_box(counter.count_text(s)),
+ BatchSize::SmallInput,
+ )
+ });
+
+ group.finish();
+}
+
+criterion_group!(benches, bench_count_text);
+criterion_main!(benches);
diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rs
index baac3bad5..53502062a 100644
--- a/crates/headroom-core/src/lib.rs
+++ b/crates/headroom-core/src/lib.rs
@@ -1,7 +1,6 @@
//! headroom-core: foundation crate for the Rust port of Headroom.
-//!
-//! Phase 0: only exposes stubs. No algorithm implementations yet.
+pub mod tokenizer;
pub mod transforms;
/// Identity stub used by downstream crates and the Python binding to verify
diff --git a/crates/headroom-core/src/tokenizer/estimator.rs b/crates/headroom-core/src/tokenizer/estimator.rs
new file mode 100644
index 000000000..5dde222d3
--- /dev/null
+++ b/crates/headroom-core/src/tokenizer/estimator.rs
@@ -0,0 +1,152 @@
+//! Character-density estimator. Used as a fallback for any tokenizer family
+//! we haven't wired in yet (Anthropic Claude, Google Gemini, Cohere, …).
+//!
+//! Mirrors `headroom.tokenizers.estimator.EstimatingTokenCounter`. The formula
+//! is `ceil(chars / chars_per_token)`. `chars` is *Unicode scalar count*, not
+//! byte length, to match Python's `len(text)` semantics on str.
+
+use super::{Backend, Tokenizer};
+
+#[derive(Debug, Clone, Copy)]
+pub struct EstimatingCounter {
+ chars_per_token: f64,
+}
+
+impl Default for EstimatingCounter {
+ fn default() -> Self {
+ Self {
+ chars_per_token: 4.0,
+ }
+ }
+}
+
+impl EstimatingCounter {
+ /// `chars_per_token` must be `> 0.0`. Common calibrations:
+ /// - 3.5 — Claude-family (Python uses this in `_create_anthropic`)
+ /// - 4.0 — Gemini, Cohere, generic fallback
+ pub fn new(chars_per_token: f64) -> Self {
+ assert!(
+ chars_per_token > 0.0,
+ "chars_per_token must be positive, got {chars_per_token}"
+ );
+ Self { chars_per_token }
+ }
+
+ pub fn chars_per_token(&self) -> f64 {
+ self.chars_per_token
+ }
+}
+
+impl Tokenizer for EstimatingCounter {
+ fn count_text(&self, text: &str) -> usize {
+ if text.is_empty() {
+ return 0;
+ }
+ // Match Python `EstimatingTokenCounter.count_text`:
+ // max(1, int(len(text) / chars_per_token + 0.5))
+ // Python `int()` truncates toward zero; for non-negative inputs that's
+ // identical to `as usize` saturating-cast semantics in Rust >= 1.45.
+ // Adding 0.5 then truncating yields round-half-up. We previously used
+ // ceil, which over-counted in the middle of the range (e.g. "aaaaa"
+ // at 4.0 cpt returned 2 here vs 1 in Python).
+ let chars = text.chars().count() as f64;
+ let raw = (chars / self.chars_per_token + 0.5) as usize;
+ raw.max(1)
+ }
+
+ fn backend(&self) -> Backend {
+ Backend::Estimation
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn empty_string_is_zero() {
+ let est = EstimatingCounter::default();
+ assert_eq!(est.count_text(""), 0);
+ }
+
+ /// Expected values cross-checked against Python:
+ /// max(1, int(len / chars_per_token + 0.5))
+ #[test]
+ fn default_is_four_chars_per_token() {
+ let est = EstimatingCounter::default();
+ // 4 / 4.0 = 1.0 + 0.5 -> int(1.5) -> 1
+ assert_eq!(est.count_text("aaaa"), 1);
+ // 5 / 4.0 = 1.25 + 0.5 -> int(1.75) -> 1 (Python rounds half-up)
+ assert_eq!(est.count_text("aaaaa"), 1);
+ // 6 / 4.0 = 1.5 + 0.5 -> int(2.0) -> 2
+ assert_eq!(est.count_text("aaaaaa"), 2);
+ // 40 / 4.0 = 10.0 + 0.5 -> int(10.5) -> 10
+ assert_eq!(est.count_text(&"a".repeat(40)), 10);
+ }
+
+ #[test]
+ fn claude_density_matches_python() {
+ let est = EstimatingCounter::new(3.5);
+ // 35 / 3.5 = 10.0 -> int(10.5) -> 10
+ assert_eq!(est.count_text(&"a".repeat(35)), 10);
+ // 36 / 3.5 ≈ 10.286 -> int(10.786) -> 10 (NOT 11 — that was ceil)
+ assert_eq!(est.count_text(&"a".repeat(36)), 10);
+ // 38 / 3.5 ≈ 10.857 -> int(11.357) -> 11
+ assert_eq!(est.count_text(&"a".repeat(38)), 11);
+ }
+
+ #[test]
+ fn unicode_uses_char_count_not_bytes() {
+ let est = EstimatingCounter::default();
+ // "héllo" is 5 chars, 6 bytes; 5/4.0 = 1.25 -> int(1.75) -> 1
+ assert_eq!(est.count_text("héllo"), 1);
+ // 4 emojis = 4 chars; 4/4.0 = 1.0 -> int(1.5) -> 1
+ assert_eq!(est.count_text("🦀🦀🦀🦀"), 1);
+ }
+
+ #[test]
+ fn min_is_one_for_non_empty_input() {
+ let est = EstimatingCounter::default();
+ // 1/4.0 = 0.25 + 0.5 -> int(0.75) -> 0; max(1, 0) -> 1
+ assert_eq!(est.count_text("a"), 1);
+ assert_eq!(est.count_text("ab"), 1);
+ // 3/4.0 = 0.75 + 0.5 -> int(1.25) -> 1
+ assert_eq!(est.count_text("abc"), 1);
+ // 6/4.0 = 1.5 + 0.5 -> int(2.0) -> 2
+ assert_eq!(est.count_text("aaaaaa"), 2);
+ }
+
+ #[test]
+ fn deterministic() {
+ let est = EstimatingCounter::default();
+ let s = "the quick brown fox jumps over the lazy dog";
+ let a = est.count_text(s);
+ let b = est.count_text(s);
+ assert_eq!(a, b);
+ }
+
+ #[test]
+ fn very_long_input_does_not_overflow() {
+ let est = EstimatingCounter::default();
+ let s = "a".repeat(1_000_000);
+ assert_eq!(est.count_text(&s), 250_000);
+ }
+
+ #[test]
+ #[should_panic(expected = "chars_per_token must be positive")]
+ fn rejects_zero_density() {
+ let _ = EstimatingCounter::new(0.0);
+ }
+
+ #[test]
+ #[should_panic(expected = "chars_per_token must be positive")]
+ fn rejects_negative_density() {
+ let _ = EstimatingCounter::new(-1.0);
+ }
+
+ #[test]
+ fn backend_is_estimation() {
+ let est = EstimatingCounter::default();
+ assert_eq!(est.backend(), Backend::Estimation);
+ }
+}
diff --git a/crates/headroom-core/src/tokenizer/hf_impl.rs b/crates/headroom-core/src/tokenizer/hf_impl.rs
new file mode 100644
index 000000000..3c0f46dda
--- /dev/null
+++ b/crates/headroom-core/src/tokenizer/hf_impl.rs
@@ -0,0 +1,240 @@
+//! HuggingFace `tokenizers`-crate adapter implementing [`Tokenizer`].
+//!
+//! Loads a `tokenizer.json` (HuggingFace's serialization format) and counts
+//! tokens via real BPE / Unigram / WordPiece — whatever the file describes.
+//! This closes the gap between OpenAI (tiktoken, byte-equal) and the
+//! Anthropic/Google chars-per-token fallback: every other major family
+//! (Cohere `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …) publishes a
+//! `tokenizer.json` on the HuggingFace Hub and the `tokenizers` crate is a
+//! pure-Rust loader, so we don't have to estimate.
+//!
+//! # What's NOT here
+//! - **No HuggingFace Hub auto-download.** Callers pass bytes or a path. A
+//! later stage can add `hf-hub` integration behind a Cargo feature; doing it
+//! here would drag in ureq/rustls, a `~/.cache/huggingface` dependency, and
+//! gated-model auth flows that don't belong in the core crate.
+//! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere
+//! tokenizers would add several MB of binary bloat for code paths most users
+//! don't hit.
+
+use std::path::Path;
+use std::sync::Arc;
+
+use thiserror::Error;
+use tokenizers::Tokenizer as HfInner;
+
+use super::{Backend, Tokenizer};
+
+#[derive(Debug, Error)]
+pub enum HfTokenizerError {
+ /// The bytes / file did not parse as a valid HuggingFace `tokenizer.json`,
+ /// or the model component referenced an unsupported algorithm.
+ #[error("failed to load tokenizer for `{name}`: {source}")]
+ Load {
+ name: String,
+ #[source]
+ source: Box<dyn std::error::Error + Send + Sync>,
+ },
+}
+
+/// Token counter backed by a HuggingFace `tokenizer.json`.
+///
+/// Cheap to clone — internally an `Arc<tokenizers::Tokenizer>`. Construct once
+/// at startup, share across handlers.
+#[derive(Clone)]
+pub struct HfTokenizer {
+ name: String,
+ inner: Arc<HfInner>,
+}
+
+impl std::fmt::Debug for HfTokenizer {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("HfTokenizer")
+ .field("name", &self.name)
+ .finish()
+ }
+}
+
+impl HfTokenizer {
+ /// Build from raw `tokenizer.json` bytes. Use this when the tokenizer is
+ /// embedded via `include_bytes!` or fetched from a non-HF source.
+ pub fn from_bytes(name: impl Into<String>, bytes: &[u8]) -> Result<Self, HfTokenizerError> {
+ let name = name.into();
+ let inner = HfInner::from_bytes(bytes).map_err(|e| HfTokenizerError::Load {
+ name: name.clone(),
+ source: e,
+ })?;
+ Ok(Self {
+ name,
+ inner: Arc::new(inner),
+ })
+ }
+
+ /// Build from a `tokenizer.json` on disk.
+ pub fn from_file(
+ name: impl Into<String>,
+ path: impl AsRef<Path>,
+ ) -> Result<Self, HfTokenizerError> {
+ let name = name.into();
+ let inner = HfInner::from_file(path.as_ref()).map_err(|e| HfTokenizerError::Load {
+ name: name.clone(),
+ source: e,
+ })?;
+ Ok(Self {
+ name,
+ inner: Arc::new(inner),
+ })
+ }
+
+ /// The logical name this tokenizer was registered under (e.g.
+ /// `"command-r-plus"`). Used in logs and metrics.
+ pub fn name(&self) -> &str {
+ &self.name
+ }
+}
+
+impl Tokenizer for HfTokenizer {
+ fn count_text(&self, text: &str) -> usize {
+ if text.is_empty() {
+ return 0;
+ }
+ // `add_special_tokens=false` matches the spirit of
+ // `tiktoken.encode_ordinary`: count *content* tokens, leaving
+ // BOS/EOS/CLS/SEP padding to be added (or not) by the upstream API.
+ // Different providers add different specials, so counting them here
+ // would systematically over-charge. Documented for future readers.
+ match self.inner.encode(text, false) {
+ Ok(enc) => enc.len(),
+ // `encode` only fails for malformed inputs that pass UTF-8 but
+ // violate the tokenizer's constraints (e.g. a normalizer that
+ // rejects certain code points). We degrade to "0 known tokens"
+ // rather than panic — the proxy must keep flowing requests.
+ Err(_) => 0,
+ }
+ }
+
+ fn backend(&self) -> Backend {
+ Backend::HuggingFace
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Minimal valid `tokenizer.json`: WordLevel model, Whitespace
+ /// pre-tokenizer, three-token vocabulary. Lets us test the API surface
+ /// without committing a multi-MB tokenizer fixture or hitting the
+ /// HuggingFace Hub.
+ const TINY_TOKENIZER_JSON: &str = r#"{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [],
+ "normalizer": null,
+ "pre_tokenizer": {"type": "Whitespace"},
+ "post_processor": null,
+ "decoder": null,
+ "model": {
+ "type": "WordLevel",
+ "vocab": {"hello": 0, "world": 1, "[UNK]": 2},
+ "unk_token": "[UNK]"
+ }
+ }"#;
+
+ fn tiny() -> HfTokenizer {
+ HfTokenizer::from_bytes("tiny-test", TINY_TOKENIZER_JSON.as_bytes())
+ .expect("tiny tokenizer.json parses")
+ }
+
+ #[test]
+ fn empty_is_zero() {
+ assert_eq!(tiny().count_text(""), 0);
+ }
+
+ #[test]
+ fn known_vocab_matches_count() {
+ let t = tiny();
+ // Each word in the vocab is one token; whitespace splits them.
+ assert_eq!(t.count_text("hello"), 1);
+ assert_eq!(t.count_text("hello world"), 2);
+ assert_eq!(t.count_text("hello world hello"), 3);
+ }
+
+ #[test]
+ fn unknown_words_become_unk() {
+ // OOV tokens collapse to [UNK] — still 1 per whitespace-split chunk.
+ let t = tiny();
+ assert_eq!(t.count_text("supercalifragilistic"), 1);
+ assert_eq!(t.count_text("foo bar baz"), 3);
+ }
+
+ #[test]
+ fn deterministic() {
+ let t = tiny();
+ let s = "hello world hello world";
+ let first = t.count_text(s);
+ for _ in 0..100 {
+ assert_eq!(t.count_text(s), first);
+ }
+ }
+
+ #[test]
+ fn unicode_does_not_panic() {
+ let t = tiny();
+ for s in ["héllo wörld", "你好世界", "🦀 ferris", "\n\t\r"] {
+ // We only assert non-panic and a reasonable upper bound; the exact
+ // count depends on Whitespace pre-tokenizer behavior, which is
+ // tokenizers-crate internal.
+ let n = t.count_text(s);
+ assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");
+ }
+ }
+
+ #[test]
+ fn invalid_bytes_returns_error() {
+ let r = HfTokenizer::from_bytes("bad", b"not a tokenizer.json");
+ assert!(matches!(r, Err(HfTokenizerError::Load { .. })));
+ }
+
+ #[test]
+ fn name_round_trips() {
+ let t = tiny();
+ assert_eq!(t.name(), "tiny-test");
+ }
+
+ #[test]
+ fn backend_is_huggingface() {
+ assert_eq!(tiny().backend(), Backend::HuggingFace);
+ }
+
+ #[test]
+ fn clone_shares_inner() {
+ let a = tiny();
+ let b = a.clone();
+ assert!(Arc::ptr_eq(&a.inner, &b.inner));
+ }
+
+ #[test]
+ fn from_file_loads_a_real_file() {
+ // Round-trip via a temp file to cover the on-disk constructor.
+ use std::io::Write;
+ let dir = std::env::temp_dir().join(format!(
+ "headroom-hf-test-{}",
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ ));
+ std::fs::create_dir_all(&dir).unwrap();
+ let path = dir.join("tokenizer.json");
+ let mut f = std::fs::File::create(&path).unwrap();
+ f.write_all(TINY_TOKENIZER_JSON.as_bytes()).unwrap();
+ drop(f);
+
+ let t = HfTokenizer::from_file("from-file", &path).expect("loads");
+ assert_eq!(t.count_text("hello world"), 2);
+
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+}
diff --git a/crates/headroom-core/src/tokenizer/mod.rs b/crates/headroom-core/src/tokenizer/mod.rs
new file mode 100644
index 000000000..1260ee4cc
--- /dev/null
+++ b/crates/headroom-core/src/tokenizer/mod.rs
@@ -0,0 +1,54 @@
+//! Token counting for Headroom transforms.
+//!
+//! Mirrors the public surface of the Python `headroom.tokenizers` package:
+//! a `Tokenizer` trait, a tiktoken-backed counter for OpenAI / o-series models
+//! (via the `tiktoken-rs` crate, which uses the same BPE data files as Python's
+//! `tiktoken` and therefore returns byte-identical token IDs), and an estimation
+//! fallback for everything else.
+//!
+//! # Why this exists
+//! Counting tokens currently round-trips into Python's `tiktoken` (itself a
+//! Rust extension under the hood). For Rust transforms running on the proxy
+//! hot path, counting natively avoids the Python-Rust FFI cost and keeps the
+//! Rust binary self-contained.
+//!
+//! # What this is NOT
+//! - Not used by `headroom-proxy` yet (Stage 2 is library-only; no production
+//! wiring).
+//! - Not a real Anthropic Claude tokenizer (Anthropic doesn't publish theirs;
+//! estimation matches what the Python implementation does).
+//! - SentencePiece — Gemini's tokenizer is SP-based but Google doesn't
+//! publish the model. Falls through to estimation.
+//!
+//! # What is here
+//! - [`TiktokenCounter`] — OpenAI / o-series, byte-equal to Python `tiktoken`.
+//! - [`HfTokenizer`] — any model with a public `tokenizer.json` (Cohere
+//! `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …). Construct it from
+//! bytes or a file path; register it via [`register_hf`] to make it the
+//! default for a given model-name prefix.
+//! - [`EstimatingCounter`] — last-resort `chars / cpt` fallback for Anthropic
+//! and Gemini, calibrated to match the Python implementation.
+
+mod estimator;
+mod hf_impl;
+mod registry;
+mod tiktoken_impl;
+
+pub use estimator::EstimatingCounter;
+pub use hf_impl::{HfTokenizer, HfTokenizerError};
+pub use registry::{clear_hf_registrations, detect_backend, get_tokenizer, register_hf, Backend};
+pub use tiktoken_impl::{TiktokenCounter, TiktokenError};
+
+/// Counts tokens. Implementations must be thread-safe (`Send + Sync`).
+///
+/// # Conventions (preserved across all built-in implementations)
+/// - `count_text("")` returns `0`.
+/// - Counts are deterministic for a given input and instance.
+/// - For non-empty input, counts are `>= 1`.
+pub trait Tokenizer: Send + Sync + std::fmt::Debug {
+ /// Number of tokens that this tokenizer assigns to `text`.
+ fn count_text(&self, text: &str) -> usize;
+
+ /// Which backend produced the count. Useful for logs and metrics.
+ fn backend(&self) -> Backend;
+}
diff --git a/crates/headroom-core/src/tokenizer/registry.rs b/crates/headroom-core/src/tokenizer/registry.rs
new file mode 100644
index 000000000..2ef14e671
--- /dev/null
+++ b/crates/headroom-core/src/tokenizer/registry.rs
@@ -0,0 +1,323 @@
+//! Model-name → tokenizer dispatch.
+//!
+//! Mirrors `MODEL_PATTERNS` in `headroom/tokenizers/registry.py`. Three
+//! backends in priority order:
+//!
+//! 1. **HuggingFace** — anything the caller has registered via
+//! [`register_hf`] for a given model-name prefix. Real BPE/Unigram/
+//! WordPiece counts. This is opt-in: tokenizer.json files aren't bundled,
+//! so nothing routes here until the embedding application calls
+//! [`register_hf`] at startup. Wins over the rules below when registered.
+//! 2. **Tiktoken** — OpenAI / o-series via `tiktoken-rs`. Byte-identical to
+//! Python `tiktoken`.
+//! 3. **Estimation** — `chars / cpt` fallback for Anthropic Claude (3.5),
+//! Gemini / Cohere / Command without an HF registration (4.0), and
+//! everything else (4.0).
+
+use std::collections::HashMap;
+use std::sync::{OnceLock, RwLock};
+
+use super::{EstimatingCounter, HfTokenizer, TiktokenCounter, Tokenizer};
+
+/// Which family of tokenizer was selected for a model.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Backend {
+ /// Real BPE via `tiktoken-rs`. Byte-identical to Python `tiktoken`.
+ Tiktoken,
+ /// HuggingFace `tokenizers` crate (`tokenizer.json` loaded by caller).
+ HuggingFace,
+ /// Character-density estimation (chars/token formula).
+ Estimation,
+}
+
+/// Pick a backend purely from the model name, ignoring runtime registrations.
+///
+/// Patterns and ordering match `headroom.tokenizers.registry.MODEL_PATTERNS`
+/// for the families this stage supports. Anything outside the OpenAI BPE
+/// family lands in `Estimation` here — even if [`register_hf`] would route
+/// it to `HuggingFace` at runtime. Use [`get_tokenizer`] for the real
+/// dispatch.
+pub fn detect_backend(model: &str) -> Backend {
+ let m = model.to_ascii_lowercase();
+
+ // OpenAI BPE-tokenized families (gpt-3.5/4/4o + o1/o3 reasoning + embeddings + legacy davinci/curie/babbage/ada + code-).
+ if m.starts_with("gpt-4o")
+ || m.starts_with("gpt-4")
+ || m.starts_with("gpt-3.5")
+ || m.starts_with("o1")
+ || m.starts_with("o3")
+ || m.starts_with("text-embedding")
+ || m.starts_with("text-davinci")
+ || m.starts_with("davinci")
+ || m.starts_with("curie")
+ || m.starts_with("babbage")
+ || m.starts_with("ada")
+ || m.starts_with("code-")
+ {
+ return Backend::Tiktoken;
+ }
+
+ Backend::Estimation
+}
+
+/// Return a tokenizer for `model`. Resolution order:
+/// 1. HuggingFace tokenizers registered via [`register_hf`] (longest matching
+/// prefix wins).
+/// 2. Tiktoken for OpenAI / o-series families.
+/// 3. Estimation, with density calibrated per family (Claude → 3.5, Gemini /
+/// Cohere / Command → 4.0, otherwise 4.0).
+pub fn get_tokenizer(model: &str) -> Box<dyn Tokenizer> {
+ if let Some(hf) = lookup_hf(model) {
+ return Box::new(hf);
+ }
+ match detect_backend(model) {
+ Backend::Tiktoken => match TiktokenCounter::for_model(model) {
+ Ok(t) => Box::new(t),
+ Err(_) => Box::new(default_estimator_for(model)),
+ },
+ // Backend::HuggingFace from detect_backend is unreachable — only
+ // runtime registrations produce HF, and we already checked above.
+ Backend::HuggingFace | Backend::Estimation => Box::new(default_estimator_for(model)),
+ }
+}
+
+fn default_estimator_for(model: &str) -> EstimatingCounter {
+ let m = model.to_ascii_lowercase();
+ if m.starts_with("claude-") {
+ EstimatingCounter::new(3.5)
+ } else if m.starts_with("gemini") || m.starts_with("palm") || m.starts_with("command") {
+ EstimatingCounter::new(4.0)
+ } else {
+ EstimatingCounter::default()
+ }
+}
+
+// ---- HuggingFace runtime registry --------------------------------------
+//
+// A process-global table mapping a lowercased model-name *prefix* to a loaded
+// `HfTokenizer`. The embedding application owns this — there's no autoloader
+// in core because we don't want to bundle tokenizer.json files (multi-MB) or
+// pull in HF Hub networking here.
+
+fn hf_table() -> &'static RwLock<HashMap<String, HfTokenizer>> {
+ static TABLE: OnceLock<RwLock<HashMap<String, HfTokenizer>>> = OnceLock::new();
+ TABLE.get_or_init(|| RwLock::new(HashMap::new()))
+}
+
+/// Register `tokenizer` to handle every model whose lowercased name starts
+/// with `prefix`. Multiple prefixes can coexist; on lookup the longest
+/// matching prefix wins, so registering `"command-r-plus"` overrides a more
+/// general `"command-"` registration for that one model.
+///
+/// Calling this with the same `prefix` twice replaces the previous tokenizer.
+pub fn register_hf(prefix: impl Into<String>, tokenizer: HfTokenizer) {
+ let key = prefix.into().to_ascii_lowercase();
+ hf_table()
+ .write()
+ .expect("hf registry poisoned")
+ .insert(key, tokenizer);
+}
+
+/// Drop every HuggingFace registration. Intended for tests; production code
+/// should register once at startup.
+pub fn clear_hf_registrations() {
+ hf_table().write().expect("hf registry poisoned").clear();
+}
+
+fn lookup_hf(model: &str) -> Option<HfTokenizer> {
+ let m = model.to_ascii_lowercase();
+ let table = hf_table().read().expect("hf registry poisoned");
+ // Longest prefix wins: a `command-r-plus` registration must beat
+ // `command-` for that model, regardless of insertion order.
+ table
+ .iter()
+ .filter(|(prefix, _)| m.starts_with(prefix.as_str()))
+ .max_by_key(|(prefix, _)| prefix.len())
+ .map(|(_, t)| t.clone())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Same minimal tokenizer.json used by hf_impl tests; inlined here so we
+ /// can register it without depending on a fixture file.
+ const TINY_TOKENIZER_JSON: &str = r#"{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [],
+ "normalizer": null,
+ "pre_tokenizer": {"type": "Whitespace"},
+ "post_processor": null,
+ "decoder": null,
+ "model": {
+ "type": "WordLevel",
+ "vocab": {"hello": 0, "world": 1, "[UNK]": 2},
+ "unk_token": "[UNK]"
+ }
+ }"#;
+
+ fn tiny(name: &str) -> HfTokenizer {
+ HfTokenizer::from_bytes(name, TINY_TOKENIZER_JSON.as_bytes()).unwrap()
+ }
+
+ /// Tests share a process-global table. `cargo test` runs them in parallel
+ /// by default, so without serialization one test's `clear` would wipe
+ /// another test's registrations mid-assertion. We serialize the tests
+ /// that touch the registry behind a single mutex.
+ static REGISTRY_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+ struct RegistryGuard<'a> {
+ _g: std::sync::MutexGuard<'a, ()>,
+ }
+ impl<'a> RegistryGuard<'a> {
+ fn acquire() -> Self {
+ // Recover from a poisoned lock — a panic in one test should not
+ // break every subsequent test in the file.
+ let g = REGISTRY_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+ clear_hf_registrations();
+ Self { _g: g }
+ }
+ }
+ impl<'a> Drop for RegistryGuard<'a> {
+ fn drop(&mut self) {
+ clear_hf_registrations();
+ }
+ }
+
+ #[test]
+ fn openai_models_pick_tiktoken() {
+ for m in [
+ "gpt-4o",
+ "gpt-4o-mini",
+ "gpt-4",
+ "gpt-4-turbo",
+ "gpt-3.5-turbo",
+ "o1-preview",
+ "o3-mini",
+ "text-embedding-3-small",
+ "text-davinci-003",
+ "davinci",
+ "babbage-002",
+ "code-davinci-002",
+ ] {
+ assert_eq!(detect_backend(m), Backend::Tiktoken, "{m}");
+ }
+ }
+
+ #[test]
+ fn non_openai_models_fall_through_to_estimation() {
+ for m in [
+ "claude-haiku-4-5-20251001",
+ "claude-3-opus",
+ "gemini-1.5-pro",
+ "command-r-plus",
+ "llama-3-70b",
+ "mistral-large",
+ "qwen-72b",
+ "made-up-model-name",
+ ] {
+ assert_eq!(detect_backend(m), Backend::Estimation, "{m}");
+ }
+ }
+
+ #[test]
+ fn case_insensitive() {
+ assert_eq!(detect_backend("GPT-4o"), Backend::Tiktoken);
+ assert_eq!(detect_backend("Claude-haiku"), Backend::Estimation);
+ }
+
+ #[test]
+ fn estimator_density_per_family() {
+ let _g = RegistryGuard::acquire();
+ // Round-trip through the public dispatch and check that the chosen
+ // estimator behaves with the right density. We can't introspect the
+ // trait object's chars_per_token directly, so we use a known-length
+ // string and back-compute.
+ let claude = get_tokenizer("claude-3-opus");
+ // 3.5 chars/token: 35 chars -> 10 tokens.
+ assert_eq!(claude.count_text(&"a".repeat(35)), 10);
+
+ let gemini = get_tokenizer("gemini-1.5-pro");
+ // 4.0 chars/token: 40 chars -> 10 tokens.
+ assert_eq!(gemini.count_text(&"a".repeat(40)), 10);
+ }
+
+ #[test]
+ fn registered_hf_wins_over_estimator() {
+ let _g = RegistryGuard::acquire();
+ register_hf("command-", tiny("cohere"));
+ let t = get_tokenizer("command-r-plus");
+ // Whitespace tokenization → 2 tokens, *not* the chars/4 estimator.
+ assert_eq!(t.count_text("hello world"), 2);
+ assert_eq!(t.backend(), Backend::HuggingFace);
+ }
+
+ #[test]
+ fn registered_hf_does_not_override_tiktoken() {
+ // A user accidentally registers an HF tokenizer with the prefix
+ // `gpt-` — should the OpenAI route still win? Per docstring: HF
+ // always wins when registered. We document this behavior so the
+ // test pins it down: registration is a deliberate override.
+ let _g = RegistryGuard::acquire();
+ register_hf("gpt-4o", tiny("oops"));
+ let t = get_tokenizer("gpt-4o-mini");
+ assert_eq!(t.backend(), Backend::HuggingFace);
+ }
+
+ #[test]
+ fn longest_prefix_wins() {
+ let _g = RegistryGuard::acquire();
+ register_hf("command-", tiny("general"));
+ register_hf("command-r-plus", tiny("specific"));
+ // `command-r-plus` should pick the more specific one.
+ let t = get_tokenizer("command-r-plus");
+ assert_eq!(t.backend(), Backend::HuggingFace);
+ // Counting the same input via both should give identical counts since
+ // we used the same tokenizer.json — but the *path* through the
+ // registry is different. This locks in that the longer prefix is
+ // selected, not the shorter one.
+ let count_specific = t.count_text("hello world hello");
+ assert_eq!(count_specific, 3);
+
+ // A model only matched by the shorter prefix still works.
+ let t2 = get_tokenizer("command-light");
+ assert_eq!(t2.backend(), Backend::HuggingFace);
+ }
+
+ #[test]
+ fn case_insensitive_registration() {
+ let _g = RegistryGuard::acquire();
+ register_hf("Command-", tiny("cohere"));
+ let t = get_tokenizer("COMMAND-R-PLUS");
+ assert_eq!(t.backend(), Backend::HuggingFace);
+ }
+
+ #[test]
+ fn clear_resets_state() {
+ let _g = RegistryGuard::acquire();
+ register_hf("command-", tiny("cohere"));
+ clear_hf_registrations();
+ let t = get_tokenizer("command-r-plus");
+ // Without an HF registration, falls back to estimator (4.0 cpt).
+ assert_eq!(t.backend(), Backend::Estimation);
+ }
+
+ #[test]
+ fn unrelated_models_still_estimate() {
+ let _g = RegistryGuard::acquire();
+ register_hf("command-", tiny("cohere"));
+ let t = get_tokenizer("claude-3-opus");
+ assert_eq!(t.backend(), Backend::Estimation);
+ }
+
+ #[test]
+ fn detect_backend_ignores_runtime_registrations() {
+ let _g = RegistryGuard::acquire();
+ register_hf("command-", tiny("cohere"));
+ // detect_backend is a pure function of the model name; runtime
+ // registrations don't show up here.
+ assert_eq!(detect_backend("command-r-plus"), Backend::Estimation);
+ }
+}
diff --git a/crates/headroom-core/src/tokenizer/tiktoken_impl.rs b/crates/headroom-core/src/tokenizer/tiktoken_impl.rs
new file mode 100644
index 000000000..4910f71cf
--- /dev/null
+++ b/crates/headroom-core/src/tokenizer/tiktoken_impl.rs
@@ -0,0 +1,265 @@
+//! `tiktoken-rs` adapter implementing [`Tokenizer`].
+//!
+//! `tiktoken-rs` and Python `tiktoken` use the same BPE merge tables; for the
+//! same model and same input, this returns byte-identical token IDs and
+//! therefore byte-identical token *counts*. This is what makes the parity
+//! tests "byte-equal" rather than "approximate".
+//!
+//! Initialization (loading the BPE table) is non-trivial. Each encoding is
+//! built lazily on first use and shared via `LazyLock<Arc<CoreBPE>>`, so the
+//! first `for_model` call pays the cost and every subsequent call is cheap.
+
+use std::sync::{Arc, LazyLock};
+
+use thiserror::Error;
+use tiktoken_rs::CoreBPE;
+
+use super::{Backend, Tokenizer};
+
+#[derive(Debug, Error)]
+pub enum TiktokenError {
+ /// We don't know which encoding `model` should use. The caller can fall
+ /// back to estimation; the registry handles that automatically.
+ #[error("unknown encoding for model `{0}`")]
+ UnknownEncoding(String),
+}
+
+/// Lazy-built shared BPE for the four named encodings. Init failure here would
+/// indicate `tiktoken-rs` itself is broken; we treat that as a programmer error
+/// and panic.
+static O200K: LazyLock<Arc<CoreBPE>> =
+ LazyLock::new(|| Arc::new(tiktoken_rs::o200k_base().expect("o200k_base init")));
+static CL100K: LazyLock<Arc<CoreBPE>> =
+ LazyLock::new(|| Arc::new(tiktoken_rs::cl100k_base().expect("cl100k_base init")));
+static P50K: LazyLock<Arc<CoreBPE>> =
+ LazyLock::new(|| Arc::new(tiktoken_rs::p50k_base().expect("p50k_base init")));
+static R50K: LazyLock<Arc<CoreBPE>> =
+ LazyLock::new(|| Arc::new(tiktoken_rs::r50k_base().expect("r50k_base init")));
+
+/// BPE token counter for OpenAI / o-series models.
+pub struct TiktokenCounter {
+ model: String,
+ encoding_name: &'static str,
+ bpe: Arc<CoreBPE>,
+}
+
+impl std::fmt::Debug for TiktokenCounter {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("TiktokenCounter")
+ .field("model", &self.model)
+ .field("encoding", &self.encoding_name)
+ .finish()
+ }
+}
+
+impl TiktokenCounter {
+ /// Build a counter for `model`. Returns `UnknownEncoding` if the model
+ /// doesn't fall into any of the supported BPE families.
+ pub fn for_model(model: &str) -> Result<Self, TiktokenError> {
+ let encoding_name = encoding_for(model)?;
+ let bpe = match encoding_name {
+ "o200k_base" => O200K.clone(),
+ "cl100k_base" => CL100K.clone(),
+ "p50k_base" => P50K.clone(),
+ "r50k_base" => R50K.clone(),
+ // unreachable: encoding_for only returns the four names above.
+ _ => return Err(TiktokenError::UnknownEncoding(model.to_string())),
+ };
+ Ok(Self {
+ model: model.to_string(),
+ encoding_name,
+ bpe,
+ })
+ }
+
+ pub fn model(&self) -> &str {
+ &self.model
+ }
+
+ pub fn encoding_name(&self) -> &'static str {
+ self.encoding_name
+ }
+}
+
+impl Tokenizer for TiktokenCounter {
+ fn count_text(&self, text: &str) -> usize {
+ if text.is_empty() {
+ // Match Python `TiktokenCounter.count_text`: short-circuit empty.
+ return 0;
+ }
+ // For ORDINARY input (no literal special-token strings) `encode_ordinary`
+ // here and `encoding.encode(text)` in Python yield identical token IDs
+ // and counts — that's the byte-equality the parity harness verifies.
+ //
+ // Divergence (rare in practice): if `text` contains a literal
+ // `<|endoftext|>` (or any other special-token string), Python's default
+ // `encode` raises (because `disallowed_special="all"`) while we treat
+ // it as ordinary text. We chose tolerance over panic since proxy users
+ // can legitimately send those substrings; document for future readers.
+ self.bpe.encode_ordinary(text).len()
+ }
+
+ fn backend(&self) -> Backend {
+ Backend::Tiktoken
+ }
+}
+
+/// Map model → encoding name. Mirrors `MODEL_ENCODINGS` and the prefix
+/// fallbacks in `headroom/tokenizers/tiktoken_counter.py`.
+fn encoding_for(model: &str) -> Result<&'static str, TiktokenError> {
+ let m = model.to_ascii_lowercase();
+
+ // o200k_base: GPT-4o + o1/o3 reasoning families.
+ if m.starts_with("gpt-4o") || m.starts_with("o1") || m.starts_with("o3") {
+ return Ok("o200k_base");
+ }
+
+ // cl100k_base: GPT-4, GPT-3.5-turbo, embeddings.
+ if m.starts_with("gpt-4") || m.starts_with("gpt-3.5") || m.starts_with("text-embedding") {
+ return Ok("cl100k_base");
+ }
+
+ // p50k_base: code-* and the davinci-002/003 text-completion line.
+ if m.starts_with("code-")
+ || m.starts_with("text-davinci-002")
+ || m.starts_with("text-davinci-003")
+ {
+ return Ok("p50k_base");
+ }
+
+ // r50k_base: legacy davinci-001 and earlier completion families.
+ if m.starts_with("text-davinci")
+ || m.starts_with("davinci")
+ || m.starts_with("curie")
+ || m.starts_with("babbage")
+ || m.starts_with("ada")
+ {
+ return Ok("r50k_base");
+ }
+
+ Err(TiktokenError::UnknownEncoding(model.to_string()))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn empty_string_is_zero() {
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ assert_eq!(t.count_text(""), 0);
+ }
+
+ #[test]
+ fn nonempty_text_is_at_least_one_token() {
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ assert!(t.count_text("a") >= 1);
+ }
+
+ #[test]
+ fn known_token_counts_for_o200k() {
+ // These constants are the o200k_base BPE token counts produced by
+ // both Python `tiktoken.encoding_for_model("gpt-4o-mini")` and
+ // `tiktoken-rs::o200k_base()` for the given strings. They lock in
+ // byte-equal parity so a future tiktoken-rs upgrade that subtly
+ // changes BPE behavior would fail this test.
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ assert_eq!(t.count_text("hello"), 1);
+ assert_eq!(t.count_text("Hello, world!"), 4);
+ assert_eq!(
+ t.count_text("the quick brown fox jumps over the lazy dog"),
+ 9
+ );
+ }
+
+ #[test]
+ fn determinism() {
+ let t = TiktokenCounter::for_model("gpt-4o").unwrap();
+ let s = "Determinism check across many calls.";
+ let first = t.count_text(s);
+ for _ in 0..1000 {
+ assert_eq!(t.count_text(s), first);
+ }
+ }
+
+ #[test]
+ fn unicode_input_does_not_panic() {
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ // Each call should produce a reasonable count (>=1 for non-empty),
+ // not panic, and not return absurd values.
+ for s in [
+ "héllo wörld", // accented
+ "你好世界", // CJK
+ "مرحبا بالعالم", // Arabic (RTL)
+ "🦀 ferris the crab", // emoji
+ "\n\t\r\x07", // control chars
+ ] {
+ let n = t.count_text(s);
+ assert!(n >= 1, "{s:?}");
+ assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");
+ }
+ }
+
+ #[test]
+ fn very_long_input() {
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ let s = "the quick brown fox ".repeat(50_000); // ~1MB
+ let n = t.count_text(&s);
+ // 50k repeats * ~5 tokens per repeat = ~250k tokens, sanity bound.
+ assert!(n > 100_000 && n < 1_000_000, "n={n}");
+ }
+
+ #[test]
+ fn encoding_dispatch() {
+ for (model, expected) in [
+ ("gpt-4o", "o200k_base"),
+ ("gpt-4o-mini", "o200k_base"),
+ ("gpt-4o-2024-08-06", "o200k_base"),
+ ("o1-preview", "o200k_base"),
+ ("o3-mini", "o200k_base"),
+ ("gpt-4", "cl100k_base"),
+ ("gpt-4-turbo", "cl100k_base"),
+ ("gpt-3.5-turbo", "cl100k_base"),
+ ("text-embedding-3-small", "cl100k_base"),
+ ("code-davinci-002", "p50k_base"),
+ ("text-davinci-002", "p50k_base"),
+ ("text-davinci-003", "p50k_base"),
+ ("text-davinci-001", "r50k_base"),
+ ("davinci", "r50k_base"),
+ ("curie", "r50k_base"),
+ ("babbage", "r50k_base"),
+ ("ada", "r50k_base"),
+ ] {
+ let t = TiktokenCounter::for_model(model)
+ .unwrap_or_else(|e| panic!("for_model({model}) failed: {e}"));
+ assert_eq!(t.encoding_name(), expected, "{model}");
+ }
+ }
+
+ #[test]
+ fn unknown_model_returns_error() {
+ let r = TiktokenCounter::for_model("claude-3-opus");
+ assert!(matches!(r, Err(TiktokenError::UnknownEncoding(_))));
+ }
+
+ #[test]
+ fn case_insensitive_dispatch() {
+ let t = TiktokenCounter::for_model("GPT-4o-Mini").unwrap();
+ assert_eq!(t.encoding_name(), "o200k_base");
+ }
+
+ #[test]
+ fn shared_bpe_instances() {
+ // Two counters for the same encoding should share the underlying BPE
+ // (same Arc), proving the LazyLock cache works.
+ let a = TiktokenCounter::for_model("gpt-4o").unwrap();
+ let b = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ assert!(Arc::ptr_eq(&a.bpe, &b.bpe));
+ }
+
+ #[test]
+ fn backend_is_tiktoken() {
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ assert_eq!(t.backend(), Backend::Tiktoken);
+ }
+}
diff --git a/crates/headroom-core/tests/tokenizer_proptest.rs b/crates/headroom-core/tests/tokenizer_proptest.rs
new file mode 100644
index 000000000..abd52c11f
--- /dev/null
+++ b/crates/headroom-core/tests/tokenizer_proptest.rs
@@ -0,0 +1,82 @@
+//! Property tests for the tokenizer module.
+//!
+//! Invariants we lean on for downstream callers (cost tracking, compression
+//! decisions, cache keys). These are small enough to also serve as quick
+//! regression catchers if a tokenizer-rs upgrade breaks the surface API.
+
+use headroom_core::tokenizer::{get_tokenizer, EstimatingCounter, TiktokenCounter, Tokenizer};
+use proptest::prelude::*;
+
+proptest! {
+ #![proptest_config(ProptestConfig {
+ cases: 256,
+ .. ProptestConfig::default()
+ })]
+
+ /// Determinism: counting the same text twice yields the same count
+ /// regardless of which tokenizer is used.
+ #[test]
+ fn deterministic_per_instance(s in any::<String>()) {
+ let tt = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ let est = EstimatingCounter::default();
+ prop_assert_eq!(tt.count_text(&s), tt.count_text(&s));
+ prop_assert_eq!(est.count_text(&s), est.count_text(&s));
+ }
+
+ /// Empty input produces zero tokens for every backend.
+ #[test]
+ fn empty_is_zero_for_all_backends(_dummy in 0u8..1) {
+ for model in ["gpt-4o-mini", "claude-3-opus", "gemini-1.5-pro", "unknown-model"] {
+ let t = get_tokenizer(model);
+ prop_assert_eq!(t.count_text(""), 0, "{}", model);
+ }
+ }
+
+ /// Non-empty input produces at least one token. The regex `+` quantifier
+ /// already guarantees `s` is non-empty; no `prop_assume!` needed.
+ #[test]
+ fn nonempty_input_is_at_least_one_token(s in "[a-zA-Z0-9 ]+") {
+ let tt = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ prop_assert!(tt.count_text(&s) >= 1);
+ let est = EstimatingCounter::default();
+ prop_assert!(est.count_text(&s) >= 1);
+ }
+
+ /// Concatenation behaves on the same scale as the parts. We *cannot*
+ /// claim true subadditivity (`count(a+b) <= count(a) + count(b)`): BPE
+ /// runs on top of a regex pre-tokenizer, and pre-tokenization of `a+b`
+ /// can split differently than the union of pre-tokenizations of `a` and
+ /// `b` for some Unicode inputs (e.g. `"𝀀" + "(A𐲀"` produced 9 tokens
+ /// vs 3+5=8 separately during proptest exploration).
+ ///
+ /// The weaker, *true* invariant: concat doesn't blow up the count beyond
+ /// a small constant overhead. We bound it loosely — even with a
+ /// pre-tokenizer disagreement, the boundary can introduce at most a
+ /// handful of extra tokens, never a multiplicative blowup.
+ #[test]
+ fn concat_does_not_explode(a in any::<String>(), b in any::<String>()) {
+ let tt = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
+ let na = tt.count_text(&a);
+ let nb = tt.count_text(&b);
+ let mut combined = a.clone();
+ combined.push_str(&b);
+ let nc = tt.count_text(&combined);
+ // Allow up to 8 extra tokens at the boundary (generous) to absorb
+ // pre-tokenizer regex disagreements on exotic Unicode.
+ prop_assert!(nc <= na + nb + 8,
+ "concat blew up: count({a:?})={na} + count({b:?})={nb} = {} but count({combined:?}) = {nc}",
+ na + nb);
+ }
+
+ /// Estimator monotone in input length (chars/token formula is monotone
+ /// non-decreasing as char count grows).
+ #[test]
+ fn estimator_monotone_in_length(extra in "[a-z]{1,32}", base in "[a-z]{0,32}") {
+ let est = EstimatingCounter::default();
+ let n_base = est.count_text(&base);
+ let mut longer = base.clone();
+ longer.push_str(&extra);
+ let n_long = est.count_text(&longer);
+ prop_assert!(n_long >= n_base);
+ }
+}
diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rs
index cb4e88550..51f7cbf6b 100644
--- a/crates/headroom-parity/src/lib.rs
+++ b/crates/headroom-parity/src/lib.rs
@@ -160,9 +160,35 @@ macro_rules! stub_comparator {
stub_comparator!(LogCompressorComparator, "log_compressor");
stub_comparator!(DiffCompressorComparator, "diff_compressor");
stub_comparator!(CacheAlignerComparator, "cache_aligner");
-stub_comparator!(TokenizerComparator, "tokenizer");
stub_comparator!(CcrComparator, "ccr");
+/// Real comparator for the `tokenizer` transform. The recorder used
+/// `headroom.providers.openai.OpenAITokenCounter("gpt-4o-mini")`, so the
+/// fixture outputs are o200k_base BPE token counts. We rebuild the same
+/// encoding via `tiktoken-rs` and assert byte-equal counts.
+pub struct TokenizerComparator;
+
+impl TransformComparator for TokenizerComparator {
+ fn name(&self) -> &str {
+ "tokenizer"
+ }
+
+ fn run(
+ &self,
+ input: &serde_json::Value,
+ _config: &serde_json::Value,
+ ) -> Result<serde_json::Value> {
+ use headroom_core::tokenizer::{TiktokenCounter, Tokenizer};
+ let text = input
+ .as_str()
+ .context("tokenizer fixture input must be a JSON string")?;
+ let counter = TiktokenCounter::for_model("gpt-4o-mini")
+ .context("init TiktokenCounter for gpt-4o-mini")?;
+ let count = counter.count_text(text);
+ Ok(serde_json::json!(count))
+ }
+}
+
/// Every built-in comparator, in a stable order.
pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {
vec![