Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
//! HuggingFace `tokenizers`-crate adapter implementing [`Tokenizer`].
//!
//! Loads a `tokenizer.json` (HuggingFace's serialization format) and counts
//! tokens via real BPE / Unigram / WordPiece — whatever the file describes.
//! This closes the gap between OpenAI (tiktoken, byte-equal) and the
//! Anthropic/Google chars-per-token fallback: every other major family
//! (Cohere `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …) publishes a
//! `tokenizer.json` on the HuggingFace Hub and the `tokenizers` crate is a
//! pure-Rust loader, so we don't have to estimate.
//!
//! # Sources
//! - [`HfTokenizer::from_bytes`] — for tokenizers embedded via `include_bytes!`.
//! - [`HfTokenizer::from_file`] — for a local `tokenizer.json`.
//! - [`HfTokenizer::from_pretrained`] — pulls `tokenizer.json` from the
//! HuggingFace Hub via `hf-hub`. First call downloads into the hub cache
//! Python's `huggingface_hub` uses (`HF_HUB_CACHE`, `HF_HOME/hub`,
//! `~/.cache/huggingface/hub`, …), subsequent calls hit it. Blocking;
//! call from `main()` or `tokio::task::spawn_blocking`. Gated repos
//! (Llama, Mistral) need a token: `HF_TOKEN`, or the file
//! `hf auth login` writes (`HF_HOME/token`, or `HF_TOKEN_PATH`).
//!
//! # What's NOT here
//! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere
//! tokenizers would add several MB of binary bloat for code paths most users
//! don't hit. `from_pretrained` lazily downloads instead.
use std::path::Path;
use std::sync::Arc;
use thiserror::Error;
use tokenizers::Tokenizer as HfInner;
use super::{Backend, Tokenizer};
#[derive(Debug, Error)]
pub enum HfTokenizerError {
/// The bytes / file did not parse as a valid HuggingFace `tokenizer.json`,
/// or the model component referenced an unsupported algorithm.
#[error("failed to load tokenizer for `{name}`: {source}")]
Load {
name: String,
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
/// The HuggingFace Hub fetch failed: network error, 404 on the repo, or
/// 401 on a gated model without an `HF_TOKEN`.
#[error("failed to download `{repo}` from HuggingFace Hub: {source}")]
Hub {
repo: String,
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
/// `HEADROOM_OFFLINE` is set, so the Hub fetch was refused before any
/// socket was opened. Deliberately a separate variant from [`Self::Hub`]:
/// a caller that retries or falls back on a transient download failure
/// must not do either here — the operator asked for no egress, and
/// retrying an air-gap refusal is just a slower refusal.
#[error(transparent)]
Offline(#[from] crate::offline::OfflineEgressBlocked),
}
/// Token counter backed by a HuggingFace `tokenizer.json`.
///
/// Cheap to clone — internally an `Arc<tokenizers::Tokenizer>`. Construct once
/// at startup, share across handlers.
#[derive(Clone)]
pub struct HfTokenizer {
name: String,
inner: Arc<HfInner>,
}
impl std::fmt::Debug for HfTokenizer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HfTokenizer")
.field("name", &self.name)
.finish()
}
}
impl HfTokenizer {
/// Build from raw `tokenizer.json` bytes. Use this when the tokenizer is
/// embedded via `include_bytes!` or fetched from a non-HF source.
pub fn from_bytes(name: impl Into<String>, bytes: &[u8]) -> Result<Self, HfTokenizerError> {
let name = name.into();
let inner = HfInner::from_bytes(bytes).map_err(|e| HfTokenizerError::Load {
name: name.clone(),
source: e,
})?;
Ok(Self {
name,
inner: Arc::new(inner),
})
}
/// Build from a `tokenizer.json` on disk.
pub fn from_file(
name: impl Into<String>,
path: impl AsRef<Path>,
) -> Result<Self, HfTokenizerError> {
let name = name.into();
let inner = HfInner::from_file(path.as_ref()).map_err(|e| HfTokenizerError::Load {
name: name.clone(),
source: e,
})?;
Ok(Self {
name,
inner: Arc::new(inner),
})
}
/// Download (or fetch from cache) `tokenizer.json` for `repo` from the
/// HuggingFace Hub and load it. `repo` is the canonical Hub identifier,
/// e.g. `"CohereForAI/c4ai-command-r-v01"` or `"meta-llama/Meta-Llama-3-8B"`.
///
/// Uses the `main` revision. Blocking — calls into `ureq` synchronously.
/// The file lands in the hub cache directory Python's `huggingface_hub`
/// would use ([`crate::hf_cache::hub_cache_dir`]: `HF_HUB_CACHE`, then
/// `HUGGINGFACE_HUB_CACHE`, `HF_HOME/hub`, `XDG_CACHE_HOME/huggingface/hub`,
/// `~/.cache/huggingface/hub`), so later calls, and the Python side, hit
/// the same copy. The token and `HF_ENDPOINT` are read as Python reads
/// them too ([`crate::hf_cache::hf_token`]); `hf-hub`'s own defaults
/// ignore every variable but `HF_HOME`, and with `Api::new` even that.
///
/// Errors:
/// - [`HfTokenizerError::Offline`] when `HEADROOM_OFFLINE` is set. Refused
/// before any socket is opened; see below.
/// - [`HfTokenizerError::Hub`] for download failures (no network, 404,
/// 401 on a gated model without `HF_TOKEN`).
/// - [`HfTokenizerError::Load`] if the downloaded bytes don't parse as
/// a valid `tokenizer.json`. Should not happen for healthy HF repos.
pub fn from_pretrained(repo: &str) -> Result<Self, HfTokenizerError> {
// Air-gap chokepoint, matching `headroom/offline.py`'s `guard_egress`.
// This is the Rust half of the same switch, so an operator who sets
// HEADROOM_OFFLINE=1 gets the same refusal whichever runtime reaches
// for the Hub.
//
// `crate::hf_cache::hub_api` runs the guard before it builds the
// client, rather than around the `get` below: building it creates the
// `ureq` agent and resolves the endpoint, and hf-hub's own
// HF_HUB_OFFLINE handling only covers the cache lookup — a cache miss
// still dials out. So the air-gapped case never constructs a client.
//
// The refusal is hard on purpose. Silently returning a cached or
// estimating tokenizer would leave token counts subtly wrong with no
// signal; the caller (`tokenizer::registry::try_register_hf`) already
// propagates the error so the operator sees which repo was wanted.
//
// Note it refuses even on a warm cache: `hf-hub` 0.5 exposes no
// cache-hit/cache-miss split at this layer, so "guard before the
// client exists" and "serve from the cache" cannot both hold here. An
// air-gapped deployment with pre-seeded artifacts should point at them
// with `from_file`, which is network-free by construction.
let api = hub_api(repo)?;
let path = api
.model(repo.to_string())
.get("tokenizer.json")
.map_err(|e| HfTokenizerError::Hub {
repo: repo.to_string(),
source: Box::new(e),
})?;
// `get` returns the on-disk path; reuse `from_file` to keep the load
// path identical to user-supplied tokenizer.json files.
Self::from_file(repo, path)
}
/// The logical name this tokenizer was registered under (e.g.
/// `"command-r-plus"`). Used in logs and metrics.
pub fn name(&self) -> &str {
&self.name
}
}
/// The Hub client for [`HfTokenizer::from_pretrained`] fetching `repo`:
/// caching in Python's hub cache directory, with Python's token and endpoint.
/// Refuses with [`HfTokenizerError::Offline`] under `HEADROOM_OFFLINE`.
fn hub_api(repo: &str) -> Result<hf_hub::api::sync::Api, HfTokenizerError> {
use crate::hf_cache::HubApiError;
const PURPOSE: &str = "HuggingFace tokenizer download";
let hub_error = |source: Box<dyn std::error::Error + Send + Sync>| HfTokenizerError::Hub {
repo: repo.to_string(),
source,
};
let Some(cache) = crate::hf_cache::hub_cache_dir() else {
// Offline still wins: the caller treats `Offline` as final, and a
// missing cache directory is no reason to report a Hub failure.
crate::offline::guard_egress(PURPOSE, repo)?;
return Err(hub_error(
"no HuggingFace cache directory (set HF_HUB_CACHE or HF_HOME)".into(),
));
};
crate::hf_cache::hub_api(&cache, PURPOSE, repo).map_err(|e| match e {
HubApiError::Offline(blocked) => HfTokenizerError::Offline(blocked),
HubApiError::Build(source) => hub_error(Box::new(source)),
})
}
impl Tokenizer for HfTokenizer {
fn count_text(&self, text: &str) -> usize {
if text.is_empty() {
return 0;
}
// `add_special_tokens=false` matches the spirit of
// `tiktoken.encode_ordinary`: count *content* tokens, leaving
// BOS/EOS/CLS/SEP padding to be added (or not) by the upstream API.
// Different providers add different specials, so counting them here
// would systematically over-charge. Documented for future readers.
match self.inner.encode(text, false) {
Ok(enc) => enc.len(),
// `encode` only fails for malformed inputs that pass UTF-8 but
// violate the tokenizer's constraints (e.g. a normalizer that
// rejects certain code points). We degrade to "0 known tokens"
// rather than panic — the proxy must keep flowing requests.
Err(_) => 0,
}
}
fn backend(&self) -> Backend {
Backend::HuggingFace
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Minimal valid `tokenizer.json`: WordLevel model, Whitespace
/// pre-tokenizer, three-token vocabulary. Lets us test the API surface
/// without committing a multi-MB tokenizer fixture or hitting the
/// HuggingFace Hub.
const TINY_TOKENIZER_JSON: &str = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": null,
"pre_tokenizer": {"type": "Whitespace"},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"hello": 0, "world": 1, "[UNK]": 2},
"unk_token": "[UNK]"
}
}"#;
fn tiny() -> HfTokenizer {
HfTokenizer::from_bytes("tiny-test", TINY_TOKENIZER_JSON.as_bytes())
.expect("tiny tokenizer.json parses")
}
#[test]
fn empty_is_zero() {
assert_eq!(tiny().count_text(""), 0);
}
#[test]
fn known_vocab_matches_count() {
let t = tiny();
// Each word in the vocab is one token; whitespace splits them.
assert_eq!(t.count_text("hello"), 1);
assert_eq!(t.count_text("hello world"), 2);
assert_eq!(t.count_text("hello world hello"), 3);
}
#[test]
fn unknown_words_become_unk() {
// OOV tokens collapse to [UNK] — still 1 per whitespace-split chunk.
let t = tiny();
assert_eq!(t.count_text("supercalifragilistic"), 1);
assert_eq!(t.count_text("foo bar baz"), 3);
}
#[test]
fn deterministic() {
let t = tiny();
let s = "hello world hello world";
let first = t.count_text(s);
for _ in 0..100 {
assert_eq!(t.count_text(s), first);
}
}
#[test]
fn unicode_does_not_panic() {
let t = tiny();
for s in ["héllo wörld", "你好世界", "🦀 ferris", "\n\t\r"] {
// We only assert non-panic and a reasonable upper bound; the exact
// count depends on Whitespace pre-tokenizer behavior, which is
// tokenizers-crate internal.
let n = t.count_text(s);
assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");
}
}
#[test]
fn invalid_bytes_returns_error() {
let r = HfTokenizer::from_bytes("bad", b"not a tokenizer.json");
assert!(matches!(r, Err(HfTokenizerError::Load { .. })));
}
#[test]
fn name_round_trips() {
let t = tiny();
assert_eq!(t.name(), "tiny-test");
}
#[test]
fn backend_is_huggingface() {
assert_eq!(tiny().backend(), Backend::HuggingFace);
}
#[test]
fn clone_shares_inner() {
let a = tiny();
let b = a.clone();
assert!(Arc::ptr_eq(&a.inner, &b.inner));
}
/// A-2: `HEADROOM_OFFLINE=1` must stop the Hub fetch before a socket exists.
///
/// The repo name is deliberately one that cannot exist, which is what makes
/// this a real assertion rather than a tautology: without the guard the call
/// reaches `hf-hub`, dials `huggingface.co`, gets a 401/404 and returns
/// `HfTokenizerError::Hub`. Only the guard can produce `Offline`, and only
/// by returning before `hf_cache::hub_api` builds the client — so `Offline` *is* the "no socket was
/// opened" assertion for this path. (A packet-level assertion is not
/// available to a `cargo test` unit test the way `socket.socket` patching is
/// in Python; this typed-error distinction is the equivalent signal.)
#[test]
fn from_pretrained_refuses_while_offline() {
let _guard = crate::test_support::env_lock();
std::env::set_var(crate::offline::OFFLINE_ENV, "1");
let result = HfTokenizer::from_pretrained("headroomlabs-ai/this-repo-does-not-exist-a2");
std::env::remove_var(crate::offline::OFFLINE_ENV);
match result {
Err(HfTokenizerError::Offline(blocked)) => {
assert_eq!(blocked.purpose, "HuggingFace tokenizer download");
assert_eq!(
blocked.destination,
"headroomlabs-ai/this-repo-does-not-exist-a2"
);
}
Err(other) => panic!("expected an offline refusal, got a network error: {other}"),
Ok(_) => panic!("expected an offline refusal, got a tokenizer"),
}
}
// The "switch off ⇒ guard is a no-op" half is covered in `crate::offline`'s
// own tests rather than here: asserting it through `from_pretrained` would
// mean actually dialling huggingface.co, which is exactly the thing a unit
// test must not do.
#[test]
fn from_file_loads_a_real_file() {
// Round-trip via a temp file to cover the on-disk constructor.
use std::io::Write;
let dir = std::env::temp_dir().join(format!(
"headroom-hf-test-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("tokenizer.json");
let mut f = std::fs::File::create(&path).unwrap();
f.write_all(TINY_TOKENIZER_JSON.as_bytes()).unwrap();
drop(f);
let t = HfTokenizer::from_file("from-file", &path).expect("loads");
assert_eq!(t.count_text("hello world"), 2);
let _ = std::fs::remove_dir_all(&dir);
}
/// Lay out `tokenizer.json` for `repo` in hub cache `cache`, the way
/// Python's `huggingface_hub` leaves a `main` download: a snapshot plus
/// `refs/main` naming it.
fn cache_main_tokenizer(cache: &std::path::Path, repo: &str) {
let repo_dir = cache.join(format!("models--{}", repo.replace('/', "--")));
let snapshot = repo_dir.join("snapshots").join("abc123");
std::fs::create_dir_all(&snapshot).unwrap();
std::fs::write(snapshot.join("tokenizer.json"), TINY_TOKENIZER_JSON).unwrap();
std::fs::create_dir_all(repo_dir.join("refs")).unwrap();
std::fs::write(repo_dir.join("refs").join("main"), "abc123").unwrap();
}
/// Variables `from_pretrained` reads, saved and restored around a test.
fn hub_env() -> crate::hf_cache::tests::CacheEnv {
crate::hf_cache::tests::CacheEnv::cleared_with(
[
"HF_TOKEN",
"HUGGING_FACE_HUB_TOKEN",
"HF_TOKEN_PATH",
"HF_ENDPOINT",
"HF_TEST_ROOT",
]
.map(String::from),
)
}
/// The tokenizer is read from (and would be downloaded into) the hub cache
/// Python uses, here an `HF_HUB_CACHE` given through `${VAR}`. Before, it
/// was always `~/.cache/huggingface/hub`, so this was a cache miss and a
/// download. `HF_ENDPOINT` points at a closed loopback port, so a miss
/// fails fast instead of reaching the internet.
#[test]
fn from_pretrained_reads_the_hub_cache_python_uses() {
let _guard = crate::test_support::env_lock();
let _env = hub_env();
let t = tempfile::tempdir().unwrap();
std::env::set_var("HOME", t.path().join("home"));
std::env::set_var("USERPROFILE", t.path().join("home"));
std::env::set_var("HF_TEST_ROOT", t.path());
std::env::set_var("HF_HUB_CACHE", "${HF_TEST_ROOT}/hub");
std::env::set_var("HF_ENDPOINT", "http://127.0.0.1:9");
cache_main_tokenizer(&t.path().join("hub"), "acme/tiny");
let tokenizer = HfTokenizer::from_pretrained("acme/tiny").expect("served from the cache");
assert_eq!(tokenizer.count_text("hello world"), 2);
}
/// A cache miss asks the Hub at `HF_ENDPOINT` with the token from
/// `HF_TOKEN`. `hf-hub`'s `Api::new` read neither, so a gated repo
/// (Llama, Mistral) failed with 401 even with `HF_TOKEN` set. A loopback
/// stand-in for the Hub records the request and answers 404.
#[test]
fn from_pretrained_sends_hf_token_to_hf_endpoint() {
use std::io::{Read, Write};
let _guard = crate::test_support::env_lock();
let _env = hub_env();
let t = tempfile::tempdir().unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
// Waits at most 10 s for the request, so a regression that never
// reaches the endpoint fails instead of hanging the suite.
listener.set_nonblocking(true).unwrap();
let hub = std::thread::spawn(move || {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
let mut conn = loop {
match listener.accept() {
Ok((conn, _)) => break conn,
Err(_) if std::time::Instant::now() < deadline => {
std::thread::sleep(std::time::Duration::from_millis(20))
}
Err(_) => return String::new(),
}
};
conn.set_nonblocking(false).unwrap();
let mut request = Vec::new();
let mut buf = [0u8; 1024];
while !request.windows(4).any(|w| w == b"\r\n\r\n") {
let n = conn.read(&mut buf).unwrap();
if n == 0 {
break;
}
request.extend_from_slice(&buf[..n]);
}
conn.write_all(
b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
)
.unwrap();
String::from_utf8_lossy(&request).to_ascii_lowercase()
});
std::env::set_var("HOME", t.path().join("home"));
std::env::set_var("USERPROFILE", t.path().join("home"));
std::env::set_var("HF_HUB_CACHE", t.path().join("hub"));
// Trailing slash on purpose: Python strips it, and a kept one
// would request `//acme/...`.
std::env::set_var("HF_ENDPOINT", format!("http://127.0.0.1:{port}/"));
std::env::set_var("HF_TOKEN", "hf_test_token");
let result = HfTokenizer::from_pretrained("acme/gated");
let request = hub.join().unwrap();
assert!(
matches!(result, Err(HfTokenizerError::Hub { .. })),
"{result:?}"
);
assert!(
request.contains(" /acme/gated/resolve/main/tokenizer.json "),
"{request}"
);
assert!(
request.contains("authorization: bearer hf_test_token"),
"{request}"
);
}
/// Network-dependent: hits HuggingFace Hub. Run with
/// `cargo test -p headroom-core -- --ignored from_pretrained_downloads_real_tokenizer`.
/// `gpt2` is a small public unauthenticated repo (~1.4 MB tokenizer.json).
#[test]
#[ignore = "network-dependent: hits HuggingFace Hub"]
fn from_pretrained_downloads_real_tokenizer() {
let t = HfTokenizer::from_pretrained("gpt2").expect("download succeeds");
// GPT-2 BPE: "hello world" is 2 tokens. Locks in that we got a real
// BPE tokenizer (not a WhitespaceSplit fixture) from HF.
assert_eq!(t.count_text("hello world"), 2);
assert_eq!(t.name(), "gpt2");
assert_eq!(t.backend(), Backend::HuggingFace);
}
/// Negative path: a malformed repo name fails before any network call
/// (the Hub URL builder rejects it). No network required to run.
#[test]
fn from_pretrained_invalid_repo_returns_hub_error() {
// Empty repo name — hf-hub rejects this without making a network
// request. Locks in that we propagate Hub errors as `Hub`, not
// `Load`, so callers can distinguish "couldn't fetch" from
// "fetched but malformed".
//
// Runs online: hold the env lock so a parallel test's
// `HEADROOM_OFFLINE=1` can't turn this into `Offline`, and clear an
// inherited value for the same reason.
let _guard = crate::test_support::env_lock();
let inherited = std::env::var_os(crate::offline::OFFLINE_ENV);
std::env::remove_var(crate::offline::OFFLINE_ENV);
let r = HfTokenizer::from_pretrained("");
if let Some(value) = inherited {
std::env::set_var(crate::offline::OFFLINE_ENV, value);
}
assert!(
matches!(r, Err(HfTokenizerError::Hub { .. })),
"expected HfTokenizerError::Hub, got {r:?}"
);
}
}