Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): tokenizer crate with tiktoken-rs + HuggingFace + estimator
chopratejas9ce1c01b87a6
diff --git a/Cargo.lock b/Cargo.lockindex 069fdc13b..3f1e6793d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2,6 +2,20 @@# It is not intended for manual editing.version = 3+[[package]]+name = "ahash"+version = "0.8.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"+dependencies = [+ "cfg-if",+ "getrandom 0.3.4",+ "once_cell",+ "serde",+ "version_check",+ "zerocopy",+]+[[package]]name = "aho-corasick"version = "1.1.4"@@ -11,6 +25,12 @@ dependencies = ["memchr",]+[[package]]+name = "anes"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"+[[package]]name = "anstream"version = "1.0.0"@@ -109,7 +129,7 @@ dependencies = ["async-trait","axum-core","axum-macros",- "base64",+ "base64 0.22.1","bytes","futures-util","http",@@ -170,12 +190,33 @@ dependencies = ["syn",]+[[package]]+name = "base64"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"+[[package]]name = "base64"version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "bit-set"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"+dependencies = [+ "bit-vec",+]++[[package]]+name = "bit-vec"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"+[[package]]name = "bitflags"version = "2.11.1"@@ -191,6 +232,17 @@ dependencies = ["generic-array",]+[[package]]+name = "bstr"+version = "1.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"+dependencies = [+ "memchr",+ "regex-automata",+ "serde",+]+[[package]]name = "bumpalo"version = "3.20.2"@@ -215,6 +267,21 @@ version = "1.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"+[[package]]+name = "cast"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"++[[package]]+name = "castaway"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"+dependencies = [+ "rustversion",+]+[[package]]name = "cc"version = "1.2.60"@@ -237,6 +304,33 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"+[[package]]+name = "ciborium"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"+dependencies = [+ "ciborium-io",+ "ciborium-ll",+ "serde",+]++[[package]]+name = "ciborium-io"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"++[[package]]+name = "ciborium-ll"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"+dependencies = [+ "ciborium-io",+ "half",+]+[[package]]name = "clap"version = "4.6.1"@@ -283,6 +377,34 @@ version = "1.0.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"+[[package]]+name = "compact_str"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"+dependencies = [+ "castaway",+ "cfg-if",+ "itoa",+ "rustversion",+ "ryu",+ "serde",+ "static_assertions",+]++[[package]]+name = "console"+version = "0.15.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"+dependencies = [+ "encode_unicode",+ "libc",+ "once_cell",+ "unicode-width",+ "windows-sys 0.59.0",+]+[[package]]name = "cpufeatures"version = "0.2.17"@@ -292,6 +414,73 @@ dependencies = ["libc",]+[[package]]+name = "criterion"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"+dependencies = [+ "anes",+ "cast",+ "ciborium",+ "clap",+ "criterion-plot",+ "is-terminal",+ "itertools 0.10.5",+ "num-traits",+ "once_cell",+ "oorandom",+ "plotters",+ "rayon",+ "regex",+ "serde",+ "serde_derive",+ "serde_json",+ "tinytemplate",+ "walkdir",+]++[[package]]+name = "criterion-plot"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"+dependencies = [+ "cast",+ "itertools 0.10.5",+]++[[package]]+name = "crossbeam-deque"+version = "0.8.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"+dependencies = [+ "crossbeam-epoch",+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-epoch"+version = "0.9.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"+dependencies = [+ "crossbeam-utils",+]++[[package]]+name = "crossbeam-utils"+version = "0.8.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"++[[package]]+name = "crunchy"+version = "0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"+[[package]]name = "crypto-common"version = "0.1.7"@@ -302,6 +491,50 @@ dependencies = ["typenum",]+[[package]]+name = "darling"+version = "0.20.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"+dependencies = [+ "darling_core",+ "darling_macro",+]++[[package]]+name = "darling_core"+version = "0.20.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"+dependencies = [+ "fnv",+ "ident_case",+ "proc-macro2",+ "quote",+ "strsim",+ "syn",+]++[[package]]+name = "darling_macro"+version = "0.20.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"+dependencies = [+ "darling_core",+ "quote",+ "syn",+]++[[package]]+name = "dary_heap"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"+dependencies = [+ "serde",+]+[[package]]name = "data-encoding"version = "2.11.0"@@ -326,6 +559,37 @@ version = "0.1.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"+[[package]]+name = "derive_builder"+version = "0.20.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"+dependencies = [+ "derive_builder_macro",+]++[[package]]+name = "derive_builder_core"+version = "0.20.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"+dependencies = [+ "darling",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "derive_builder_macro"+version = "0.20.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"+dependencies = [+ "derive_builder_core",+ "syn",+]+[[package]]name = "digest"version = "0.10.7"@@ -347,6 +611,18 @@ dependencies = ["syn",]+[[package]]+name = "either"+version = "1.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"++[[package]]+name = "encode_unicode"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"+[[package]]name = "equivalent"version = "1.0.2"@@ -363,6 +639,32 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "esaxx-rs"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"+dependencies = [+ "cc",+]++[[package]]+name = "fancy-regex"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"+dependencies = [+ "bit-set",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "fastrand"+version = "2.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"+[[package]]name = "find-msvc-tools"version = "0.1.9"@@ -547,6 +849,17 @@ dependencies = ["tracing",]+[[package]]+name = "half"+version = "2.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"+dependencies = [+ "cfg-if",+ "crunchy",+ "zerocopy",+]+[[package]]name = "hashbrown"version = "0.15.5"@@ -567,9 +880,13 @@ name = "headroom-core"version = "0.1.0"dependencies = ["bytes",+ "criterion",+ "proptest","serde","serde_json","thiserror 1.0.69",+ "tiktoken-rs",+ "tokenizers","tracing",]@@ -734,7 +1051,7 @@ version = "0.1.20"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"dependencies = [- "base64",+ "base64 0.22.1","bytes","futures-channel","futures-util",@@ -839,6 +1156,12 @@ version = "2.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"+[[package]]+name = "ident_case"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"+[[package]]name = "idna"version = "1.1.0"@@ -872,6 +1195,19 @@ dependencies = ["serde_core",]+[[package]]+name = "indicatif"+version = "0.17.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"+dependencies = [+ "console",+ "number_prefix",+ "portable-atomic",+ "unicode-width",+ "web-time",+]+[[package]]name = "indoc"version = "2.0.7"@@ -897,12 +1233,41 @@ dependencies = ["serde",]+[[package]]+name = "is-terminal"+version = "0.4.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"+dependencies = [+ "hermit-abi",+ "libc",+ "windows-sys 0.61.2",+]+[[package]]name = "is_terminal_polyfill"version = "1.70.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"+[[package]]+name = "itertools"+version = "0.10.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"+dependencies = [+ "either",+]++[[package]]+name = "itertools"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.18"@@ -939,6 +1304,12 @@ version = "0.2.185"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"+[[package]]+name = "linux-raw-sys"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"+[[package]]name = "litemap"version = "0.8.2"@@ -957,6 +1328,22 @@ version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"+[[package]]+name = "macro_rules_attribute"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520"+dependencies = [+ "macro_rules_attribute-proc_macro",+ "paste",+]++[[package]]+name = "macro_rules_attribute-proc_macro"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"+[[package]]name = "matchers"version = "0.2.0"@@ -993,6 +1380,12 @@ version = "0.3.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"+[[package]]+name = "minimal-lexical"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"+[[package]]name = "mio"version = "1.2.0"@@ -1004,6 +1397,38 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "monostate"+version = "0.1.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"+dependencies = [+ "monostate-impl",+ "serde",+ "serde_core",+]++[[package]]+name = "monostate-impl"+version = "0.1.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "nom"+version = "7.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"+dependencies = [+ "memchr",+ "minimal-lexical",+]+[[package]]name = "nu-ansi-term"version = "0.50.3"@@ -1013,6 +1438,15 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+]+[[package]]name = "num_cpus"version = "1.17.0"@@ -1023,6 +1457,12 @@ dependencies = ["libc",]+[[package]]+name = "number_prefix"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"+[[package]]name = "once_cell"version = "1.21.4"@@ -1035,6 +1475,40 @@ version = "1.70.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"+[[package]]+name = "onig"+version = "6.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"+dependencies = [+ "bitflags",+ "libc",+ "once_cell",+ "onig_sys",+]++[[package]]+name = "onig_sys"+version = "69.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"+dependencies = [+ "cc",+ "pkg-config",+]++[[package]]+name = "oorandom"+version = "11.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"++[[package]]+name = "paste"+version = "1.0.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"+[[package]]name = "percent-encoding"version = "2.3.2"@@ -1047,6 +1521,40 @@ version = "0.2.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"+[[package]]+name = "pkg-config"+version = "0.3.33"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"++[[package]]+name = "plotters"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"+dependencies = [+ "num-traits",+ "plotters-backend",+ "plotters-svg",+ "wasm-bindgen",+ "web-sys",+]++[[package]]+name = "plotters-backend"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"++[[package]]+name = "plotters-svg"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"+dependencies = [+ "plotters-backend",+]+[[package]]name = "portable-atomic"version = "1.13.1"@@ -1090,6 +1598,25 @@ dependencies = ["unicode-ident",]+[[package]]+name = "proptest"+version = "1.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"+dependencies = [+ "bit-set",+ "bit-vec",+ "bitflags",+ "num-traits",+ "rand 0.9.4",+ "rand_chacha 0.9.0",+ "rand_xorshift",+ "regex-syntax",+ "rusty-fork",+ "tempfile",+ "unarray",+]+[[package]]name = "pyo3"version = "0.22.6"@@ -1153,6 +1680,12 @@ dependencies = ["syn",]+[[package]]+name = "quick-error"+version = "1.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"+[[package]]name = "quinn"version = "0.11.9"@@ -1164,7 +1697,7 @@ dependencies = ["pin-project-lite","quinn-proto","quinn-udp",- "rustc-hash",+ "rustc-hash 2.1.2","rustls","socket2","thiserror 2.0.18",@@ -1184,7 +1717,7 @@ dependencies = ["lru-slab","rand 0.9.4","ring",- "rustc-hash",+ "rustc-hash 2.1.2","rustls","rustls-pki-types","slab",@@ -1288,6 +1821,46 @@ dependencies = ["getrandom 0.3.4",]+[[package]]+name = "rand_xorshift"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"+dependencies = [+ "rand_core 0.9.5",+]++[[package]]+name = "rayon"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"+dependencies = [+ "either",+ "rayon-core",+]++[[package]]+name = "rayon-cond"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"+dependencies = [+ "either",+ "itertools 0.14.0",+ "rayon",+]++[[package]]+name = "rayon-core"+version = "1.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"+dependencies = [+ "crossbeam-deque",+ "crossbeam-utils",+]+[[package]]name = "regex"version = "1.12.3"@@ -1323,7 +1896,7 @@ version = "0.12.28"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"dependencies = [- "base64",+ "base64 0.22.1","bytes","futures-core","futures-util",@@ -1373,12 +1946,31 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "rustc-hash"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"+[[package]]name = "rustc-hash"version = "2.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"+[[package]]+name = "rustix"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"+dependencies = [+ "bitflags",+ "errno",+ "libc",+ "linux-raw-sys",+ "windows-sys 0.61.2",+]+[[package]]name = "rustls"version = "0.23.39"@@ -1420,12 +2012,33 @@ version = "1.0.22"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"+[[package]]+name = "rusty-fork"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2"+dependencies = [+ "fnv",+ "quick-error",+ "tempfile",+ "wait-timeout",+]+[[package]]name = "ryu"version = "1.0.23"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"+[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+]+[[package]]name = "semver"version = "1.0.28"@@ -1556,12 +2169,30 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "spm_precompiled"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"+dependencies = [+ "base64 0.13.1",+ "nom",+ "serde",+ "unicode-segmentation",+]+[[package]]name = "stable_deref_trait"version = "1.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"+[[package]]+name = "static_assertions"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"+[[package]]name = "strsim"version = "0.11.1"@@ -1611,6 +2242,19 @@ version = "0.12.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"+[[package]]+name = "tempfile"+version = "3.27.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"+dependencies = [+ "fastrand",+ "getrandom 0.4.2",+ "once_cell",+ "rustix",+ "windows-sys 0.61.2",+]+[[package]]name = "thiserror"version = "1.0.69"@@ -1660,6 +2304,21 @@ dependencies = ["cfg-if",]+[[package]]+name = "tiktoken-rs"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"+dependencies = [+ "anyhow",+ "base64 0.22.1",+ "bstr",+ "fancy-regex",+ "lazy_static",+ "regex",+ "rustc-hash 1.1.0",+]+[[package]]name = "tinystr"version = "0.8.3"@@ -1670,6 +2329,16 @@ dependencies = ["zerovec",]+[[package]]+name = "tinytemplate"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"+dependencies = [+ "serde",+ "serde_json",+]+[[package]]name = "tinyvec"version = "1.11.0"@@ -1685,6 +2354,40 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"+[[package]]+name = "tokenizers"+version = "0.21.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"+dependencies = [+ "ahash",+ "aho-corasick",+ "compact_str",+ "dary_heap",+ "derive_builder",+ "esaxx-rs",+ "getrandom 0.3.4",+ "indicatif",+ "itertools 0.14.0",+ "log",+ "macro_rules_attribute",+ "monostate",+ "onig",+ "paste",+ "rand 0.9.4",+ "rayon",+ "rayon-cond",+ "regex",+ "regex-syntax",+ "serde",+ "serde_json",+ "spm_precompiled",+ "thiserror 2.0.18",+ "unicode-normalization-alignments",+ "unicode-segmentation",+ "unicode_categories",+]+[[package]]name = "tokio"version = "1.52.1"@@ -1917,18 +2620,51 @@ version = "1.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"+[[package]]+name = "unarray"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"+[[package]]name = "unicode-ident"version = "1.0.24"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"+[[package]]+name = "unicode-normalization-alignments"+version = "0.1.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"+dependencies = [+ "smallvec",+]++[[package]]+name = "unicode-segmentation"+version = "1.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"++[[package]]+name = "unicode-width"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"+[[package]]name = "unicode-xid"version = "0.2.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"+[[package]]+name = "unicode_categories"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"+[[package]]name = "unindent"version = "0.2.4"@@ -1994,6 +2730,25 @@ version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"+[[package]]+name = "wait-timeout"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11"+dependencies = [+ "libc",+]++[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]+[[package]]name = "want"version = "0.3.1"@@ -2167,6 +2922,15 @@ dependencies = ["rustls-pki-types",]+[[package]]+name = "winapi-util"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"+dependencies = [+ "windows-sys 0.61.2",+]+[[package]]name = "windows-link"version = "0.2.1"@@ -2182,6 +2946,15 @@ dependencies = ["windows-targets 0.52.6",]+[[package]]+name = "windows-sys"+version = "0.59.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"+dependencies = [+ "windows-targets 0.52.6",+]+[[package]]name = "windows-sys"version = "0.60.2"@@ -2336,7 +3109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"dependencies = ["assert-json-diff",- "base64",+ "base64 0.22.1","deadpool","futures","http",diff --git a/Cargo.toml b/Cargo.tomlindex 878fcdd90..80192ad8b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -19,7 +19,7 @@ default-members = [[workspace.package]edition = "2021"-rust-version = "1.78"+rust-version = "1.80"license = "Apache-2.0"repository = "https://github.com/chopratejas/headroom"authors = ["Headroom Maintainers"]diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 366a13872..fa3617696 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -13,3 +13,16 @@ serde_json = { workspace = true }bytes = { workspace = true }thiserror = { workspace = true }tracing = { workspace = true }+tiktoken-rs = "0.11"+# `tokenizers` is the HuggingFace pure-Rust tokenizer crate. Default features+# pull in `onig` for the BPE pre-tokenizer regex; that vendors oniguruma so it+# builds without a system dep on macOS/Linux.+tokenizers = "0.21"++[dev-dependencies]+proptest = "1"+criterion = { version = "0.5", features = ["html_reports"] }++[[bench]]+name = "tokenizer"+harness = falsediff --git a/crates/headroom-core/benches/tokenizer.rs b/crates/headroom-core/benches/tokenizer.rsnew file mode 100644index 000000000..c75831a70--- /dev/null+++ b/crates/headroom-core/benches/tokenizer.rs@@ -0,0 +1,52 @@+//! Throughput benchmark for `headroom_core::tokenizer`.+//!+//! Measures the tiktoken-rs–backed counter on a small / medium / large input.+//! Used as a baseline; future stages can compare against this to catch+//! regressions when we change tokenizer backends or add caching layers.++use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion, Throughput};+use headroom_core::tokenizer::{TiktokenCounter, Tokenizer};++fn bench_count_text(c: &mut Criterion) {+ let counter = TiktokenCounter::for_model("gpt-4o-mini").expect("init");++ // Small: a typical short prompt.+ let small = "Reply with exactly: PONG";+ // Medium: a typical chat turn (~1KB).+ let medium = "the quick brown fox jumps over the lazy dog\n".repeat(25);+ // Large: a long context (~64KB) — stresses BPE inner loops.+ let large = "the quick brown fox jumps over the lazy dog\n".repeat(1500);++ let mut group = c.benchmark_group("tokenizer/count_text");+ group.throughput(Throughput::Bytes(small.len() as u64));+ group.bench_function("small", |b| {+ b.iter_batched(+ || small,+ |s| black_box(counter.count_text(s)),+ BatchSize::SmallInput,+ )+ });++ group.throughput(Throughput::Bytes(medium.len() as u64));+ group.bench_function("medium", |b| {+ b.iter_batched(+ || medium.as_str(),+ |s| black_box(counter.count_text(s)),+ BatchSize::SmallInput,+ )+ });++ group.throughput(Throughput::Bytes(large.len() as u64));+ group.bench_function("large", |b| {+ b.iter_batched(+ || large.as_str(),+ |s| black_box(counter.count_text(s)),+ BatchSize::SmallInput,+ )+ });++ group.finish();+}++criterion_group!(benches, bench_count_text);+criterion_main!(benches);diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rsindex baac3bad5..53502062a 100644--- a/crates/headroom-core/src/lib.rs+++ b/crates/headroom-core/src/lib.rs@@ -1,7 +1,6 @@//! headroom-core: foundation crate for the Rust port of Headroom.-//!-//! Phase 0: only exposes stubs. No algorithm implementations yet.+pub mod tokenizer;pub mod transforms;/// Identity stub used by downstream crates and the Python binding to verifydiff --git a/crates/headroom-core/src/tokenizer/estimator.rs b/crates/headroom-core/src/tokenizer/estimator.rsnew file mode 100644index 000000000..5dde222d3--- /dev/null+++ b/crates/headroom-core/src/tokenizer/estimator.rs@@ -0,0 +1,152 @@+//! Character-density estimator. Used as a fallback for any tokenizer family+//! we haven't wired in yet (Anthropic Claude, Google Gemini, Cohere, …).+//!+//! Mirrors `headroom.tokenizers.estimator.EstimatingTokenCounter`. The formula+//! is `ceil(chars / chars_per_token)`. `chars` is *Unicode scalar count*, not+//! byte length, to match Python's `len(text)` semantics on str.++use super::{Backend, Tokenizer};++#[derive(Debug, Clone, Copy)]+pub struct EstimatingCounter {+ chars_per_token: f64,+}++impl Default for EstimatingCounter {+ fn default() -> Self {+ Self {+ chars_per_token: 4.0,+ }+ }+}++impl EstimatingCounter {+ /// `chars_per_token` must be `> 0.0`. Common calibrations:+ /// - 3.5 — Claude-family (Python uses this in `_create_anthropic`)+ /// - 4.0 — Gemini, Cohere, generic fallback+ pub fn new(chars_per_token: f64) -> Self {+ assert!(+ chars_per_token > 0.0,+ "chars_per_token must be positive, got {chars_per_token}"+ );+ Self { chars_per_token }+ }++ pub fn chars_per_token(&self) -> f64 {+ self.chars_per_token+ }+}++impl Tokenizer for EstimatingCounter {+ fn count_text(&self, text: &str) -> usize {+ if text.is_empty() {+ return 0;+ }+ // Match Python `EstimatingTokenCounter.count_text`:+ // max(1, int(len(text) / chars_per_token + 0.5))+ // Python `int()` truncates toward zero; for non-negative inputs that's+ // identical to `as usize` saturating-cast semantics in Rust >= 1.45.+ // Adding 0.5 then truncating yields round-half-up. We previously used+ // ceil, which over-counted in the middle of the range (e.g. "aaaaa"+ // at 4.0 cpt returned 2 here vs 1 in Python).+ let chars = text.chars().count() as f64;+ let raw = (chars / self.chars_per_token + 0.5) as usize;+ raw.max(1)+ }++ fn backend(&self) -> Backend {+ Backend::Estimation+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn empty_string_is_zero() {+ let est = EstimatingCounter::default();+ assert_eq!(est.count_text(""), 0);+ }++ /// Expected values cross-checked against Python:+ /// max(1, int(len / chars_per_token + 0.5))+ #[test]+ fn default_is_four_chars_per_token() {+ let est = EstimatingCounter::default();+ // 4 / 4.0 = 1.0 + 0.5 -> int(1.5) -> 1+ assert_eq!(est.count_text("aaaa"), 1);+ // 5 / 4.0 = 1.25 + 0.5 -> int(1.75) -> 1 (Python rounds half-up)+ assert_eq!(est.count_text("aaaaa"), 1);+ // 6 / 4.0 = 1.5 + 0.5 -> int(2.0) -> 2+ assert_eq!(est.count_text("aaaaaa"), 2);+ // 40 / 4.0 = 10.0 + 0.5 -> int(10.5) -> 10+ assert_eq!(est.count_text(&"a".repeat(40)), 10);+ }++ #[test]+ fn claude_density_matches_python() {+ let est = EstimatingCounter::new(3.5);+ // 35 / 3.5 = 10.0 -> int(10.5) -> 10+ assert_eq!(est.count_text(&"a".repeat(35)), 10);+ // 36 / 3.5 ≈ 10.286 -> int(10.786) -> 10 (NOT 11 — that was ceil)+ assert_eq!(est.count_text(&"a".repeat(36)), 10);+ // 38 / 3.5 ≈ 10.857 -> int(11.357) -> 11+ assert_eq!(est.count_text(&"a".repeat(38)), 11);+ }++ #[test]+ fn unicode_uses_char_count_not_bytes() {+ let est = EstimatingCounter::default();+ // "héllo" is 5 chars, 6 bytes; 5/4.0 = 1.25 -> int(1.75) -> 1+ assert_eq!(est.count_text("héllo"), 1);+ // 4 emojis = 4 chars; 4/4.0 = 1.0 -> int(1.5) -> 1+ assert_eq!(est.count_text("🦀🦀🦀🦀"), 1);+ }++ #[test]+ fn min_is_one_for_non_empty_input() {+ let est = EstimatingCounter::default();+ // 1/4.0 = 0.25 + 0.5 -> int(0.75) -> 0; max(1, 0) -> 1+ assert_eq!(est.count_text("a"), 1);+ assert_eq!(est.count_text("ab"), 1);+ // 3/4.0 = 0.75 + 0.5 -> int(1.25) -> 1+ assert_eq!(est.count_text("abc"), 1);+ // 6/4.0 = 1.5 + 0.5 -> int(2.0) -> 2+ assert_eq!(est.count_text("aaaaaa"), 2);+ }++ #[test]+ fn deterministic() {+ let est = EstimatingCounter::default();+ let s = "the quick brown fox jumps over the lazy dog";+ let a = est.count_text(s);+ let b = est.count_text(s);+ assert_eq!(a, b);+ }++ #[test]+ fn very_long_input_does_not_overflow() {+ let est = EstimatingCounter::default();+ let s = "a".repeat(1_000_000);+ assert_eq!(est.count_text(&s), 250_000);+ }++ #[test]+ #[should_panic(expected = "chars_per_token must be positive")]+ fn rejects_zero_density() {+ let _ = EstimatingCounter::new(0.0);+ }++ #[test]+ #[should_panic(expected = "chars_per_token must be positive")]+ fn rejects_negative_density() {+ let _ = EstimatingCounter::new(-1.0);+ }++ #[test]+ fn backend_is_estimation() {+ let est = EstimatingCounter::default();+ assert_eq!(est.backend(), Backend::Estimation);+ }+}diff --git a/crates/headroom-core/src/tokenizer/hf_impl.rs b/crates/headroom-core/src/tokenizer/hf_impl.rsnew file mode 100644index 000000000..3c0f46dda--- /dev/null+++ b/crates/headroom-core/src/tokenizer/hf_impl.rs@@ -0,0 +1,240 @@+//! HuggingFace `tokenizers`-crate adapter implementing [`Tokenizer`].+//!+//! Loads a `tokenizer.json` (HuggingFace's serialization format) and counts+//! tokens via real BPE / Unigram / WordPiece — whatever the file describes.+//! This closes the gap between OpenAI (tiktoken, byte-equal) and the+//! Anthropic/Google chars-per-token fallback: every other major family+//! (Cohere `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …) publishes a+//! `tokenizer.json` on the HuggingFace Hub and the `tokenizers` crate is a+//! pure-Rust loader, so we don't have to estimate.+//!+//! # What's NOT here+//! - **No HuggingFace Hub auto-download.** Callers pass bytes or a path. A+//! later stage can add `hf-hub` integration behind a Cargo feature; doing it+//! here would drag in ureq/rustls, a `~/.cache/huggingface` dependency, and+//! gated-model auth flows that don't belong in the core crate.+//! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere+//! tokenizers would add several MB of binary bloat for code paths most users+//! don't hit.++use std::path::Path;+use std::sync::Arc;++use thiserror::Error;+use tokenizers::Tokenizer as HfInner;++use super::{Backend, Tokenizer};++#[derive(Debug, Error)]+pub enum HfTokenizerError {+ /// The bytes / file did not parse as a valid HuggingFace `tokenizer.json`,+ /// or the model component referenced an unsupported algorithm.+ #[error("failed to load tokenizer for `{name}`: {source}")]+ Load {+ name: String,+ #[source]+ source: Box<dyn std::error::Error + Send + Sync>,+ },+}++/// Token counter backed by a HuggingFace `tokenizer.json`.+///+/// Cheap to clone — internally an `Arc<tokenizers::Tokenizer>`. Construct once+/// at startup, share across handlers.+#[derive(Clone)]+pub struct HfTokenizer {+ name: String,+ inner: Arc<HfInner>,+}++impl std::fmt::Debug for HfTokenizer {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("HfTokenizer")+ .field("name", &self.name)+ .finish()+ }+}++impl HfTokenizer {+ /// Build from raw `tokenizer.json` bytes. Use this when the tokenizer is+ /// embedded via `include_bytes!` or fetched from a non-HF source.+ pub fn from_bytes(name: impl Into<String>, bytes: &[u8]) -> Result<Self, HfTokenizerError> {+ let name = name.into();+ let inner = HfInner::from_bytes(bytes).map_err(|e| HfTokenizerError::Load {+ name: name.clone(),+ source: e,+ })?;+ Ok(Self {+ name,+ inner: Arc::new(inner),+ })+ }++ /// Build from a `tokenizer.json` on disk.+ pub fn from_file(+ name: impl Into<String>,+ path: impl AsRef<Path>,+ ) -> Result<Self, HfTokenizerError> {+ let name = name.into();+ let inner = HfInner::from_file(path.as_ref()).map_err(|e| HfTokenizerError::Load {+ name: name.clone(),+ source: e,+ })?;+ Ok(Self {+ name,+ inner: Arc::new(inner),+ })+ }++ /// The logical name this tokenizer was registered under (e.g.+ /// `"command-r-plus"`). Used in logs and metrics.+ pub fn name(&self) -> &str {+ &self.name+ }+}++impl Tokenizer for HfTokenizer {+ fn count_text(&self, text: &str) -> usize {+ if text.is_empty() {+ return 0;+ }+ // `add_special_tokens=false` matches the spirit of+ // `tiktoken.encode_ordinary`: count *content* tokens, leaving+ // BOS/EOS/CLS/SEP padding to be added (or not) by the upstream API.+ // Different providers add different specials, so counting them here+ // would systematically over-charge. Documented for future readers.+ match self.inner.encode(text, false) {+ Ok(enc) => enc.len(),+ // `encode` only fails for malformed inputs that pass UTF-8 but+ // violate the tokenizer's constraints (e.g. a normalizer that+ // rejects certain code points). We degrade to "0 known tokens"+ // rather than panic — the proxy must keep flowing requests.+ Err(_) => 0,+ }+ }++ fn backend(&self) -> Backend {+ Backend::HuggingFace+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Minimal valid `tokenizer.json`: WordLevel model, Whitespace+ /// pre-tokenizer, three-token vocabulary. Lets us test the API surface+ /// without committing a multi-MB tokenizer fixture or hitting the+ /// HuggingFace Hub.+ const TINY_TOKENIZER_JSON: &str = r#"{+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [],+ "normalizer": null,+ "pre_tokenizer": {"type": "Whitespace"},+ "post_processor": null,+ "decoder": null,+ "model": {+ "type": "WordLevel",+ "vocab": {"hello": 0, "world": 1, "[UNK]": 2},+ "unk_token": "[UNK]"+ }+ }"#;++ fn tiny() -> HfTokenizer {+ HfTokenizer::from_bytes("tiny-test", TINY_TOKENIZER_JSON.as_bytes())+ .expect("tiny tokenizer.json parses")+ }++ #[test]+ fn empty_is_zero() {+ assert_eq!(tiny().count_text(""), 0);+ }++ #[test]+ fn known_vocab_matches_count() {+ let t = tiny();+ // Each word in the vocab is one token; whitespace splits them.+ assert_eq!(t.count_text("hello"), 1);+ assert_eq!(t.count_text("hello world"), 2);+ assert_eq!(t.count_text("hello world hello"), 3);+ }++ #[test]+ fn unknown_words_become_unk() {+ // OOV tokens collapse to [UNK] — still 1 per whitespace-split chunk.+ let t = tiny();+ assert_eq!(t.count_text("supercalifragilistic"), 1);+ assert_eq!(t.count_text("foo bar baz"), 3);+ }++ #[test]+ fn deterministic() {+ let t = tiny();+ let s = "hello world hello world";+ let first = t.count_text(s);+ for _ in 0..100 {+ assert_eq!(t.count_text(s), first);+ }+ }++ #[test]+ fn unicode_does_not_panic() {+ let t = tiny();+ for s in ["héllo wörld", "你好世界", "🦀 ferris", "\n\t\r"] {+ // We only assert non-panic and a reasonable upper bound; the exact+ // count depends on Whitespace pre-tokenizer behavior, which is+ // tokenizers-crate internal.+ let n = t.count_text(s);+ assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");+ }+ }++ #[test]+ fn invalid_bytes_returns_error() {+ let r = HfTokenizer::from_bytes("bad", b"not a tokenizer.json");+ assert!(matches!(r, Err(HfTokenizerError::Load { .. })));+ }++ #[test]+ fn name_round_trips() {+ let t = tiny();+ assert_eq!(t.name(), "tiny-test");+ }++ #[test]+ fn backend_is_huggingface() {+ assert_eq!(tiny().backend(), Backend::HuggingFace);+ }++ #[test]+ fn clone_shares_inner() {+ let a = tiny();+ let b = a.clone();+ assert!(Arc::ptr_eq(&a.inner, &b.inner));+ }++ #[test]+ fn from_file_loads_a_real_file() {+ // Round-trip via a temp file to cover the on-disk constructor.+ use std::io::Write;+ let dir = std::env::temp_dir().join(format!(+ "headroom-hf-test-{}",+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_nanos()+ ));+ std::fs::create_dir_all(&dir).unwrap();+ let path = dir.join("tokenizer.json");+ let mut f = std::fs::File::create(&path).unwrap();+ f.write_all(TINY_TOKENIZER_JSON.as_bytes()).unwrap();+ drop(f);++ let t = HfTokenizer::from_file("from-file", &path).expect("loads");+ assert_eq!(t.count_text("hello world"), 2);++ let _ = std::fs::remove_dir_all(&dir);+ }+}diff --git a/crates/headroom-core/src/tokenizer/mod.rs b/crates/headroom-core/src/tokenizer/mod.rsnew file mode 100644index 000000000..1260ee4cc--- /dev/null+++ b/crates/headroom-core/src/tokenizer/mod.rs@@ -0,0 +1,54 @@+//! Token counting for Headroom transforms.+//!+//! Mirrors the public surface of the Python `headroom.tokenizers` package:+//! a `Tokenizer` trait, a tiktoken-backed counter for OpenAI / o-series models+//! (via the `tiktoken-rs` crate, which uses the same BPE data files as Python's+//! `tiktoken` and therefore returns byte-identical token IDs), and an estimation+//! fallback for everything else.+//!+//! # Why this exists+//! Counting tokens currently round-trips into Python's `tiktoken` (itself a+//! Rust extension under the hood). For Rust transforms running on the proxy+//! hot path, counting natively avoids the Python-Rust FFI cost and keeps the+//! Rust binary self-contained.+//!+//! # What this is NOT+//! - Not used by `headroom-proxy` yet (Stage 2 is library-only; no production+//! wiring).+//! - Not a real Anthropic Claude tokenizer (Anthropic doesn't publish theirs;+//! estimation matches what the Python implementation does).+//! - SentencePiece — Gemini's tokenizer is SP-based but Google doesn't+//! publish the model. Falls through to estimation.+//!+//! # What is here+//! - [`TiktokenCounter`] — OpenAI / o-series, byte-equal to Python `tiktoken`.+//! - [`HfTokenizer`] — any model with a public `tokenizer.json` (Cohere+//! `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …). Construct it from+//! bytes or a file path; register it via [`register_hf`] to make it the+//! default for a given model-name prefix.+//! - [`EstimatingCounter`] — last-resort `chars / cpt` fallback for Anthropic+//! and Gemini, calibrated to match the Python implementation.++mod estimator;+mod hf_impl;+mod registry;+mod tiktoken_impl;++pub use estimator::EstimatingCounter;+pub use hf_impl::{HfTokenizer, HfTokenizerError};+pub use registry::{clear_hf_registrations, detect_backend, get_tokenizer, register_hf, Backend};+pub use tiktoken_impl::{TiktokenCounter, TiktokenError};++/// Counts tokens. Implementations must be thread-safe (`Send + Sync`).+///+/// # Conventions (preserved across all built-in implementations)+/// - `count_text("")` returns `0`.+/// - Counts are deterministic for a given input and instance.+/// - For non-empty input, counts are `>= 1`.+pub trait Tokenizer: Send + Sync + std::fmt::Debug {+ /// Number of tokens that this tokenizer assigns to `text`.+ fn count_text(&self, text: &str) -> usize;++ /// Which backend produced the count. Useful for logs and metrics.+ fn backend(&self) -> Backend;+}diff --git a/crates/headroom-core/src/tokenizer/registry.rs b/crates/headroom-core/src/tokenizer/registry.rsnew file mode 100644index 000000000..2ef14e671--- /dev/null+++ b/crates/headroom-core/src/tokenizer/registry.rs@@ -0,0 +1,323 @@+//! Model-name → tokenizer dispatch.+//!+//! Mirrors `MODEL_PATTERNS` in `headroom/tokenizers/registry.py`. Three+//! backends in priority order:+//!+//! 1. **HuggingFace** — anything the caller has registered via+//! [`register_hf`] for a given model-name prefix. Real BPE/Unigram/+//! WordPiece counts. This is opt-in: tokenizer.json files aren't bundled,+//! so nothing routes here until the embedding application calls+//! [`register_hf`] at startup. Wins over the rules below when registered.+//! 2. **Tiktoken** — OpenAI / o-series via `tiktoken-rs`. Byte-identical to+//! Python `tiktoken`.+//! 3. **Estimation** — `chars / cpt` fallback for Anthropic Claude (3.5),+//! Gemini / Cohere / Command without an HF registration (4.0), and+//! everything else (4.0).++use std::collections::HashMap;+use std::sync::{OnceLock, RwLock};++use super::{EstimatingCounter, HfTokenizer, TiktokenCounter, Tokenizer};++/// Which family of tokenizer was selected for a model.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Backend {+ /// Real BPE via `tiktoken-rs`. Byte-identical to Python `tiktoken`.+ Tiktoken,+ /// HuggingFace `tokenizers` crate (`tokenizer.json` loaded by caller).+ HuggingFace,+ /// Character-density estimation (chars/token formula).+ Estimation,+}++/// Pick a backend purely from the model name, ignoring runtime registrations.+///+/// Patterns and ordering match `headroom.tokenizers.registry.MODEL_PATTERNS`+/// for the families this stage supports. Anything outside the OpenAI BPE+/// family lands in `Estimation` here — even if [`register_hf`] would route+/// it to `HuggingFace` at runtime. Use [`get_tokenizer`] for the real+/// dispatch.+pub fn detect_backend(model: &str) -> Backend {+ let m = model.to_ascii_lowercase();++ // OpenAI BPE-tokenized families (gpt-3.5/4/4o + o1/o3 reasoning + embeddings + legacy davinci/curie/babbage/ada + code-).+ if m.starts_with("gpt-4o")+ || m.starts_with("gpt-4")+ || m.starts_with("gpt-3.5")+ || m.starts_with("o1")+ || m.starts_with("o3")+ || m.starts_with("text-embedding")+ || m.starts_with("text-davinci")+ || m.starts_with("davinci")+ || m.starts_with("curie")+ || m.starts_with("babbage")+ || m.starts_with("ada")+ || m.starts_with("code-")+ {+ return Backend::Tiktoken;+ }++ Backend::Estimation+}++/// Return a tokenizer for `model`. Resolution order:+/// 1. HuggingFace tokenizers registered via [`register_hf`] (longest matching+/// prefix wins).+/// 2. Tiktoken for OpenAI / o-series families.+/// 3. Estimation, with density calibrated per family (Claude → 3.5, Gemini /+/// Cohere / Command → 4.0, otherwise 4.0).+pub fn get_tokenizer(model: &str) -> Box<dyn Tokenizer> {+ if let Some(hf) = lookup_hf(model) {+ return Box::new(hf);+ }+ match detect_backend(model) {+ Backend::Tiktoken => match TiktokenCounter::for_model(model) {+ Ok(t) => Box::new(t),+ Err(_) => Box::new(default_estimator_for(model)),+ },+ // Backend::HuggingFace from detect_backend is unreachable — only+ // runtime registrations produce HF, and we already checked above.+ Backend::HuggingFace | Backend::Estimation => Box::new(default_estimator_for(model)),+ }+}++fn default_estimator_for(model: &str) -> EstimatingCounter {+ let m = model.to_ascii_lowercase();+ if m.starts_with("claude-") {+ EstimatingCounter::new(3.5)+ } else if m.starts_with("gemini") || m.starts_with("palm") || m.starts_with("command") {+ EstimatingCounter::new(4.0)+ } else {+ EstimatingCounter::default()+ }+}++// ---- HuggingFace runtime registry --------------------------------------+//+// A process-global table mapping a lowercased model-name *prefix* to a loaded+// `HfTokenizer`. The embedding application owns this — there's no autoloader+// in core because we don't want to bundle tokenizer.json files (multi-MB) or+// pull in HF Hub networking here.++fn hf_table() -> &'static RwLock<HashMap<String, HfTokenizer>> {+ static TABLE: OnceLock<RwLock<HashMap<String, HfTokenizer>>> = OnceLock::new();+ TABLE.get_or_init(|| RwLock::new(HashMap::new()))+}++/// Register `tokenizer` to handle every model whose lowercased name starts+/// with `prefix`. Multiple prefixes can coexist; on lookup the longest+/// matching prefix wins, so registering `"command-r-plus"` overrides a more+/// general `"command-"` registration for that one model.+///+/// Calling this with the same `prefix` twice replaces the previous tokenizer.+pub fn register_hf(prefix: impl Into<String>, tokenizer: HfTokenizer) {+ let key = prefix.into().to_ascii_lowercase();+ hf_table()+ .write()+ .expect("hf registry poisoned")+ .insert(key, tokenizer);+}++/// Drop every HuggingFace registration. Intended for tests; production code+/// should register once at startup.+pub fn clear_hf_registrations() {+ hf_table().write().expect("hf registry poisoned").clear();+}++fn lookup_hf(model: &str) -> Option<HfTokenizer> {+ let m = model.to_ascii_lowercase();+ let table = hf_table().read().expect("hf registry poisoned");+ // Longest prefix wins: a `command-r-plus` registration must beat+ // `command-` for that model, regardless of insertion order.+ table+ .iter()+ .filter(|(prefix, _)| m.starts_with(prefix.as_str()))+ .max_by_key(|(prefix, _)| prefix.len())+ .map(|(_, t)| t.clone())+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Same minimal tokenizer.json used by hf_impl tests; inlined here so we+ /// can register it without depending on a fixture file.+ const TINY_TOKENIZER_JSON: &str = r#"{+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [],+ "normalizer": null,+ "pre_tokenizer": {"type": "Whitespace"},+ "post_processor": null,+ "decoder": null,+ "model": {+ "type": "WordLevel",+ "vocab": {"hello": 0, "world": 1, "[UNK]": 2},+ "unk_token": "[UNK]"+ }+ }"#;++ fn tiny(name: &str) -> HfTokenizer {+ HfTokenizer::from_bytes(name, TINY_TOKENIZER_JSON.as_bytes()).unwrap()+ }++ /// Tests share a process-global table. `cargo test` runs them in parallel+ /// by default, so without serialization one test's `clear` would wipe+ /// another test's registrations mid-assertion. We serialize the tests+ /// that touch the registry behind a single mutex.+ static REGISTRY_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());++ struct RegistryGuard<'a> {+ _g: std::sync::MutexGuard<'a, ()>,+ }+ impl<'a> RegistryGuard<'a> {+ fn acquire() -> Self {+ // Recover from a poisoned lock — a panic in one test should not+ // break every subsequent test in the file.+ let g = REGISTRY_LOCK.lock().unwrap_or_else(|e| e.into_inner());+ clear_hf_registrations();+ Self { _g: g }+ }+ }+ impl<'a> Drop for RegistryGuard<'a> {+ fn drop(&mut self) {+ clear_hf_registrations();+ }+ }++ #[test]+ fn openai_models_pick_tiktoken() {+ for m in [+ "gpt-4o",+ "gpt-4o-mini",+ "gpt-4",+ "gpt-4-turbo",+ "gpt-3.5-turbo",+ "o1-preview",+ "o3-mini",+ "text-embedding-3-small",+ "text-davinci-003",+ "davinci",+ "babbage-002",+ "code-davinci-002",+ ] {+ assert_eq!(detect_backend(m), Backend::Tiktoken, "{m}");+ }+ }++ #[test]+ fn non_openai_models_fall_through_to_estimation() {+ for m in [+ "claude-haiku-4-5-20251001",+ "claude-3-opus",+ "gemini-1.5-pro",+ "command-r-plus",+ "llama-3-70b",+ "mistral-large",+ "qwen-72b",+ "made-up-model-name",+ ] {+ assert_eq!(detect_backend(m), Backend::Estimation, "{m}");+ }+ }++ #[test]+ fn case_insensitive() {+ assert_eq!(detect_backend("GPT-4o"), Backend::Tiktoken);+ assert_eq!(detect_backend("Claude-haiku"), Backend::Estimation);+ }++ #[test]+ fn estimator_density_per_family() {+ let _g = RegistryGuard::acquire();+ // Round-trip through the public dispatch and check that the chosen+ // estimator behaves with the right density. We can't introspect the+ // trait object's chars_per_token directly, so we use a known-length+ // string and back-compute.+ let claude = get_tokenizer("claude-3-opus");+ // 3.5 chars/token: 35 chars -> 10 tokens.+ assert_eq!(claude.count_text(&"a".repeat(35)), 10);++ let gemini = get_tokenizer("gemini-1.5-pro");+ // 4.0 chars/token: 40 chars -> 10 tokens.+ assert_eq!(gemini.count_text(&"a".repeat(40)), 10);+ }++ #[test]+ fn registered_hf_wins_over_estimator() {+ let _g = RegistryGuard::acquire();+ register_hf("command-", tiny("cohere"));+ let t = get_tokenizer("command-r-plus");+ // Whitespace tokenization → 2 tokens, *not* the chars/4 estimator.+ assert_eq!(t.count_text("hello world"), 2);+ assert_eq!(t.backend(), Backend::HuggingFace);+ }++ #[test]+ fn registered_hf_does_not_override_tiktoken() {+ // A user accidentally registers an HF tokenizer with the prefix+ // `gpt-` — should the OpenAI route still win? Per docstring: HF+ // always wins when registered. We document this behavior so the+ // test pins it down: registration is a deliberate override.+ let _g = RegistryGuard::acquire();+ register_hf("gpt-4o", tiny("oops"));+ let t = get_tokenizer("gpt-4o-mini");+ assert_eq!(t.backend(), Backend::HuggingFace);+ }++ #[test]+ fn longest_prefix_wins() {+ let _g = RegistryGuard::acquire();+ register_hf("command-", tiny("general"));+ register_hf("command-r-plus", tiny("specific"));+ // `command-r-plus` should pick the more specific one.+ let t = get_tokenizer("command-r-plus");+ assert_eq!(t.backend(), Backend::HuggingFace);+ // Counting the same input via both should give identical counts since+ // we used the same tokenizer.json — but the *path* through the+ // registry is different. This locks in that the longer prefix is+ // selected, not the shorter one.+ let count_specific = t.count_text("hello world hello");+ assert_eq!(count_specific, 3);++ // A model only matched by the shorter prefix still works.+ let t2 = get_tokenizer("command-light");+ assert_eq!(t2.backend(), Backend::HuggingFace);+ }++ #[test]+ fn case_insensitive_registration() {+ let _g = RegistryGuard::acquire();+ register_hf("Command-", tiny("cohere"));+ let t = get_tokenizer("COMMAND-R-PLUS");+ assert_eq!(t.backend(), Backend::HuggingFace);+ }++ #[test]+ fn clear_resets_state() {+ let _g = RegistryGuard::acquire();+ register_hf("command-", tiny("cohere"));+ clear_hf_registrations();+ let t = get_tokenizer("command-r-plus");+ // Without an HF registration, falls back to estimator (4.0 cpt).+ assert_eq!(t.backend(), Backend::Estimation);+ }++ #[test]+ fn unrelated_models_still_estimate() {+ let _g = RegistryGuard::acquire();+ register_hf("command-", tiny("cohere"));+ let t = get_tokenizer("claude-3-opus");+ assert_eq!(t.backend(), Backend::Estimation);+ }++ #[test]+ fn detect_backend_ignores_runtime_registrations() {+ let _g = RegistryGuard::acquire();+ register_hf("command-", tiny("cohere"));+ // detect_backend is a pure function of the model name; runtime+ // registrations don't show up here.+ assert_eq!(detect_backend("command-r-plus"), Backend::Estimation);+ }+}diff --git a/crates/headroom-core/src/tokenizer/tiktoken_impl.rs b/crates/headroom-core/src/tokenizer/tiktoken_impl.rsnew file mode 100644index 000000000..4910f71cf--- /dev/null+++ b/crates/headroom-core/src/tokenizer/tiktoken_impl.rs@@ -0,0 +1,265 @@+//! `tiktoken-rs` adapter implementing [`Tokenizer`].+//!+//! `tiktoken-rs` and Python `tiktoken` use the same BPE merge tables; for the+//! same model and same input, this returns byte-identical token IDs and+//! therefore byte-identical token *counts*. This is what makes the parity+//! tests "byte-equal" rather than "approximate".+//!+//! Initialization (loading the BPE table) is non-trivial. Each encoding is+//! built lazily on first use and shared via `LazyLock<Arc<CoreBPE>>`, so the+//! first `for_model` call pays the cost and every subsequent call is cheap.++use std::sync::{Arc, LazyLock};++use thiserror::Error;+use tiktoken_rs::CoreBPE;++use super::{Backend, Tokenizer};++#[derive(Debug, Error)]+pub enum TiktokenError {+ /// We don't know which encoding `model` should use. The caller can fall+ /// back to estimation; the registry handles that automatically.+ #[error("unknown encoding for model `{0}`")]+ UnknownEncoding(String),+}++/// Lazy-built shared BPE for the four named encodings. Init failure here would+/// indicate `tiktoken-rs` itself is broken; we treat that as a programmer error+/// and panic.+static O200K: LazyLock<Arc<CoreBPE>> =+ LazyLock::new(|| Arc::new(tiktoken_rs::o200k_base().expect("o200k_base init")));+static CL100K: LazyLock<Arc<CoreBPE>> =+ LazyLock::new(|| Arc::new(tiktoken_rs::cl100k_base().expect("cl100k_base init")));+static P50K: LazyLock<Arc<CoreBPE>> =+ LazyLock::new(|| Arc::new(tiktoken_rs::p50k_base().expect("p50k_base init")));+static R50K: LazyLock<Arc<CoreBPE>> =+ LazyLock::new(|| Arc::new(tiktoken_rs::r50k_base().expect("r50k_base init")));++/// BPE token counter for OpenAI / o-series models.+pub struct TiktokenCounter {+ model: String,+ encoding_name: &'static str,+ bpe: Arc<CoreBPE>,+}++impl std::fmt::Debug for TiktokenCounter {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("TiktokenCounter")+ .field("model", &self.model)+ .field("encoding", &self.encoding_name)+ .finish()+ }+}++impl TiktokenCounter {+ /// Build a counter for `model`. Returns `UnknownEncoding` if the model+ /// doesn't fall into any of the supported BPE families.+ pub fn for_model(model: &str) -> Result<Self, TiktokenError> {+ let encoding_name = encoding_for(model)?;+ let bpe = match encoding_name {+ "o200k_base" => O200K.clone(),+ "cl100k_base" => CL100K.clone(),+ "p50k_base" => P50K.clone(),+ "r50k_base" => R50K.clone(),+ // unreachable: encoding_for only returns the four names above.+ _ => return Err(TiktokenError::UnknownEncoding(model.to_string())),+ };+ Ok(Self {+ model: model.to_string(),+ encoding_name,+ bpe,+ })+ }++ pub fn model(&self) -> &str {+ &self.model+ }++ pub fn encoding_name(&self) -> &'static str {+ self.encoding_name+ }+}++impl Tokenizer for TiktokenCounter {+ fn count_text(&self, text: &str) -> usize {+ if text.is_empty() {+ // Match Python `TiktokenCounter.count_text`: short-circuit empty.+ return 0;+ }+ // For ORDINARY input (no literal special-token strings) `encode_ordinary`+ // here and `encoding.encode(text)` in Python yield identical token IDs+ // and counts — that's the byte-equality the parity harness verifies.+ //+ // Divergence (rare in practice): if `text` contains a literal+ // `<|endoftext|>` (or any other special-token string), Python's default+ // `encode` raises (because `disallowed_special="all"`) while we treat+ // it as ordinary text. We chose tolerance over panic since proxy users+ // can legitimately send those substrings; document for future readers.+ self.bpe.encode_ordinary(text).len()+ }++ fn backend(&self) -> Backend {+ Backend::Tiktoken+ }+}++/// Map model → encoding name. Mirrors `MODEL_ENCODINGS` and the prefix+/// fallbacks in `headroom/tokenizers/tiktoken_counter.py`.+fn encoding_for(model: &str) -> Result<&'static str, TiktokenError> {+ let m = model.to_ascii_lowercase();++ // o200k_base: GPT-4o + o1/o3 reasoning families.+ if m.starts_with("gpt-4o") || m.starts_with("o1") || m.starts_with("o3") {+ return Ok("o200k_base");+ }++ // cl100k_base: GPT-4, GPT-3.5-turbo, embeddings.+ if m.starts_with("gpt-4") || m.starts_with("gpt-3.5") || m.starts_with("text-embedding") {+ return Ok("cl100k_base");+ }++ // p50k_base: code-* and the davinci-002/003 text-completion line.+ if m.starts_with("code-")+ || m.starts_with("text-davinci-002")+ || m.starts_with("text-davinci-003")+ {+ return Ok("p50k_base");+ }++ // r50k_base: legacy davinci-001 and earlier completion families.+ if m.starts_with("text-davinci")+ || m.starts_with("davinci")+ || m.starts_with("curie")+ || m.starts_with("babbage")+ || m.starts_with("ada")+ {+ return Ok("r50k_base");+ }++ Err(TiktokenError::UnknownEncoding(model.to_string()))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn empty_string_is_zero() {+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ assert_eq!(t.count_text(""), 0);+ }++ #[test]+ fn nonempty_text_is_at_least_one_token() {+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ assert!(t.count_text("a") >= 1);+ }++ #[test]+ fn known_token_counts_for_o200k() {+ // These constants are the o200k_base BPE token counts produced by+ // both Python `tiktoken.encoding_for_model("gpt-4o-mini")` and+ // `tiktoken-rs::o200k_base()` for the given strings. They lock in+ // byte-equal parity so a future tiktoken-rs upgrade that subtly+ // changes BPE behavior would fail this test.+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ assert_eq!(t.count_text("hello"), 1);+ assert_eq!(t.count_text("Hello, world!"), 4);+ assert_eq!(+ t.count_text("the quick brown fox jumps over the lazy dog"),+ 9+ );+ }++ #[test]+ fn determinism() {+ let t = TiktokenCounter::for_model("gpt-4o").unwrap();+ let s = "Determinism check across many calls.";+ let first = t.count_text(s);+ for _ in 0..1000 {+ assert_eq!(t.count_text(s), first);+ }+ }++ #[test]+ fn unicode_input_does_not_panic() {+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ // Each call should produce a reasonable count (>=1 for non-empty),+ // not panic, and not return absurd values.+ for s in [+ "héllo wörld", // accented+ "你好世界", // CJK+ "مرحبا بالعالم", // Arabic (RTL)+ "🦀 ferris the crab", // emoji+ "\n\t\r\x07", // control chars+ ] {+ let n = t.count_text(s);+ assert!(n >= 1, "{s:?}");+ assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");+ }+ }++ #[test]+ fn very_long_input() {+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ let s = "the quick brown fox ".repeat(50_000); // ~1MB+ let n = t.count_text(&s);+ // 50k repeats * ~5 tokens per repeat = ~250k tokens, sanity bound.+ assert!(n > 100_000 && n < 1_000_000, "n={n}");+ }++ #[test]+ fn encoding_dispatch() {+ for (model, expected) in [+ ("gpt-4o", "o200k_base"),+ ("gpt-4o-mini", "o200k_base"),+ ("gpt-4o-2024-08-06", "o200k_base"),+ ("o1-preview", "o200k_base"),+ ("o3-mini", "o200k_base"),+ ("gpt-4", "cl100k_base"),+ ("gpt-4-turbo", "cl100k_base"),+ ("gpt-3.5-turbo", "cl100k_base"),+ ("text-embedding-3-small", "cl100k_base"),+ ("code-davinci-002", "p50k_base"),+ ("text-davinci-002", "p50k_base"),+ ("text-davinci-003", "p50k_base"),+ ("text-davinci-001", "r50k_base"),+ ("davinci", "r50k_base"),+ ("curie", "r50k_base"),+ ("babbage", "r50k_base"),+ ("ada", "r50k_base"),+ ] {+ let t = TiktokenCounter::for_model(model)+ .unwrap_or_else(|e| panic!("for_model({model}) failed: {e}"));+ assert_eq!(t.encoding_name(), expected, "{model}");+ }+ }++ #[test]+ fn unknown_model_returns_error() {+ let r = TiktokenCounter::for_model("claude-3-opus");+ assert!(matches!(r, Err(TiktokenError::UnknownEncoding(_))));+ }++ #[test]+ fn case_insensitive_dispatch() {+ let t = TiktokenCounter::for_model("GPT-4o-Mini").unwrap();+ assert_eq!(t.encoding_name(), "o200k_base");+ }++ #[test]+ fn shared_bpe_instances() {+ // Two counters for the same encoding should share the underlying BPE+ // (same Arc), proving the LazyLock cache works.+ let a = TiktokenCounter::for_model("gpt-4o").unwrap();+ let b = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ assert!(Arc::ptr_eq(&a.bpe, &b.bpe));+ }++ #[test]+ fn backend_is_tiktoken() {+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ assert_eq!(t.backend(), Backend::Tiktoken);+ }+}diff --git a/crates/headroom-core/tests/tokenizer_proptest.rs b/crates/headroom-core/tests/tokenizer_proptest.rsnew file mode 100644index 000000000..abd52c11f--- /dev/null+++ b/crates/headroom-core/tests/tokenizer_proptest.rs@@ -0,0 +1,82 @@+//! Property tests for the tokenizer module.+//!+//! Invariants we lean on for downstream callers (cost tracking, compression+//! decisions, cache keys). These are small enough to also serve as quick+//! regression catchers if a tokenizer-rs upgrade breaks the surface API.++use headroom_core::tokenizer::{get_tokenizer, EstimatingCounter, TiktokenCounter, Tokenizer};+use proptest::prelude::*;++proptest! {+ #![proptest_config(ProptestConfig {+ cases: 256,+ .. ProptestConfig::default()+ })]++ /// Determinism: counting the same text twice yields the same count+ /// regardless of which tokenizer is used.+ #[test]+ fn deterministic_per_instance(s in any::<String>()) {+ let tt = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ let est = EstimatingCounter::default();+ prop_assert_eq!(tt.count_text(&s), tt.count_text(&s));+ prop_assert_eq!(est.count_text(&s), est.count_text(&s));+ }++ /// Empty input produces zero tokens for every backend.+ #[test]+ fn empty_is_zero_for_all_backends(_dummy in 0u8..1) {+ for model in ["gpt-4o-mini", "claude-3-opus", "gemini-1.5-pro", "unknown-model"] {+ let t = get_tokenizer(model);+ prop_assert_eq!(t.count_text(""), 0, "{}", model);+ }+ }++ /// Non-empty input produces at least one token. The regex `+` quantifier+ /// already guarantees `s` is non-empty; no `prop_assume!` needed.+ #[test]+ fn nonempty_input_is_at_least_one_token(s in "[a-zA-Z0-9 ]+") {+ let tt = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ prop_assert!(tt.count_text(&s) >= 1);+ let est = EstimatingCounter::default();+ prop_assert!(est.count_text(&s) >= 1);+ }++ /// Concatenation behaves on the same scale as the parts. We *cannot*+ /// claim true subadditivity (`count(a+b) <= count(a) + count(b)`): BPE+ /// runs on top of a regex pre-tokenizer, and pre-tokenization of `a+b`+ /// can split differently than the union of pre-tokenizations of `a` and+ /// `b` for some Unicode inputs (e.g. `"𝀀" + "(A𐲀"` produced 9 tokens+ /// vs 3+5=8 separately during proptest exploration).+ ///+ /// The weaker, *true* invariant: concat doesn't blow up the count beyond+ /// a small constant overhead. We bound it loosely — even with a+ /// pre-tokenizer disagreement, the boundary can introduce at most a+ /// handful of extra tokens, never a multiplicative blowup.+ #[test]+ fn concat_does_not_explode(a in any::<String>(), b in any::<String>()) {+ let tt = TiktokenCounter::for_model("gpt-4o-mini").unwrap();+ let na = tt.count_text(&a);+ let nb = tt.count_text(&b);+ let mut combined = a.clone();+ combined.push_str(&b);+ let nc = tt.count_text(&combined);+ // Allow up to 8 extra tokens at the boundary (generous) to absorb+ // pre-tokenizer regex disagreements on exotic Unicode.+ prop_assert!(nc <= na + nb + 8,+ "concat blew up: count({a:?})={na} + count({b:?})={nb} = {} but count({combined:?}) = {nc}",+ na + nb);+ }++ /// Estimator monotone in input length (chars/token formula is monotone+ /// non-decreasing as char count grows).+ #[test]+ fn estimator_monotone_in_length(extra in "[a-z]{1,32}", base in "[a-z]{0,32}") {+ let est = EstimatingCounter::default();+ let n_base = est.count_text(&base);+ let mut longer = base.clone();+ longer.push_str(&extra);+ let n_long = est.count_text(&longer);+ prop_assert!(n_long >= n_base);+ }+}diff --git a/crates/headroom-parity/src/lib.rs b/crates/headroom-parity/src/lib.rsindex cb4e88550..51f7cbf6b 100644--- a/crates/headroom-parity/src/lib.rs+++ b/crates/headroom-parity/src/lib.rs@@ -160,9 +160,35 @@ macro_rules! stub_comparator {stub_comparator!(LogCompressorComparator, "log_compressor");stub_comparator!(DiffCompressorComparator, "diff_compressor");stub_comparator!(CacheAlignerComparator, "cache_aligner");-stub_comparator!(TokenizerComparator, "tokenizer");stub_comparator!(CcrComparator, "ccr");+/// Real comparator for the `tokenizer` transform. The recorder used+/// `headroom.providers.openai.OpenAITokenCounter("gpt-4o-mini")`, so the+/// fixture outputs are o200k_base BPE token counts. We rebuild the same+/// encoding via `tiktoken-rs` and assert byte-equal counts.+pub struct TokenizerComparator;++impl TransformComparator for TokenizerComparator {+ fn name(&self) -> &str {+ "tokenizer"+ }++ fn run(+ &self,+ input: &serde_json::Value,+ _config: &serde_json::Value,+ ) -> Result<serde_json::Value> {+ use headroom_core::tokenizer::{TiktokenCounter, Tokenizer};+ let text = input+ .as_str()+ .context("tokenizer fixture input must be a JSON string")?;+ let counter = TiktokenCounter::for_model("gpt-4o-mini")+ .context("init TiktokenCounter for gpt-4o-mini")?;+ let count = counter.count_text(text);+ Ok(serde_json::json!(count))+ }+}+/// Every built-in comparator, in a stable order.pub fn builtin_comparators() -> Vec<Box<dyn TransformComparator>> {vec![