Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
deps: bump tokenizers from 0.22.2 to 0.23.1 (#3149)
dependabot[bot]6e2e10f67a73
diff --git a/Cargo.lock b/Cargo.lockindex fe0ddb991..c3bbaba14 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1256,6 +1256,12 @@ dependencies = ["cmov",]+[[package]]+name = "daachorse"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2"+[[package]]name = "darling"version = "0.20.11"@@ -1562,7 +1568,7 @@ dependencies = ["safetensors","serde","serde_json",- "tokenizers",+ "tokenizers 0.22.2",][[package]]@@ -1900,7 +1906,7 @@ dependencies = ["tempfile","thiserror 2.0.20","tiktoken-rs",- "tokenizers",+ "tokenizers 0.23.1","toml","tracing","tree-sitter",@@ -4318,6 +4324,39 @@ dependencies = ["derive_builder","esaxx-rs","getrandom 0.3.4",+ "itertools 0.14.0",+ "log",+ "macro_rules_attribute",+ "monostate",+ "onig",+ "paste",+ "rand 0.9.4",+ "rayon",+ "rayon-cond",+ "regex",+ "regex-syntax",+ "serde",+ "serde_json",+ "spm_precompiled",+ "thiserror 2.0.20",+ "unicode-normalization-alignments",+ "unicode-segmentation",+ "unicode_categories",+]++[[package]]+name = "tokenizers"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112"+dependencies = [+ "ahash",+ "compact_str",+ "daachorse",+ "dary_heap",+ "derive_builder",+ "esaxx-rs",+ "getrandom 0.3.4","indicatif","itertools 0.14.0","log",diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.tomlindex 903f4511f..d1488ef49 100644--- a/crates/headroom-core/Cargo.toml+++ b/crates/headroom-core/Cargo.toml@@ -17,7 +17,7 @@ tiktoken-rs = "0.11"# `tokenizers` is the HuggingFace pure-Rust tokenizer crate. Default features# pull in `onig` for the BPE pre-tokenizer regex; that vendors oniguruma so it# builds without a system dep on macOS/Linux.-tokenizers = "0.22"+tokenizers = "0.23"# `hf-hub` is the HuggingFace Hub client. We use the blocking `ureq` transport# with `rustls` (no system OpenSSL dep — keeps the binary static-linkable for# AWS deploys). `from_pretrained` is called once at startup, so blocking is