Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

deps: bump tokenizers from 0.22.2 to 0.23.1 (#3149)

dependabot[bot]6e2e10f67a73
diff --git a/Cargo.lock b/Cargo.lock
index fe0ddb991..c3bbaba14 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1256,6 +1256,12 @@ dependencies = [
"cmov",
]
+[[package]]
+name = "daachorse"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2"
+
[[package]]
name = "darling"
version = "0.20.11"
@@ -1562,7 +1568,7 @@ dependencies = [
"safetensors",
"serde",
"serde_json",
- "tokenizers",
+ "tokenizers 0.22.2",
]
[[package]]
@@ -1900,7 +1906,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.20",
"tiktoken-rs",
- "tokenizers",
+ "tokenizers 0.23.1",
"toml",
"tracing",
"tree-sitter",
@@ -4318,6 +4324,39 @@ dependencies = [
"derive_builder",
"esaxx-rs",
"getrandom 0.3.4",
+ "itertools 0.14.0",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand 0.9.4",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror 2.0.20",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112"
+dependencies = [
+ "ahash",
+ "compact_str",
+ "daachorse",
+ "dary_heap",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom 0.3.4",
"indicatif",
"itertools 0.14.0",
"log",
diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml
index 903f4511f..d1488ef49 100644
--- a/crates/headroom-core/Cargo.toml
+++ b/crates/headroom-core/Cargo.toml
@@ -17,7 +17,7 @@ tiktoken-rs = "0.11"
# `tokenizers` is the HuggingFace pure-Rust tokenizer crate. Default features
# pull in `onig` for the BPE pre-tokenizer regex; that vendors oniguruma so it
# builds without a system dep on macOS/Linux.
-tokenizers = "0.22"
+tokenizers = "0.23"
# `hf-hub` is the HuggingFace Hub client. We use the blocking `ureq` transport
# with `rustls` (no system OpenSSL dep — keeps the binary static-linkable for
# AWS deploys). `from_pretrained` is called once at startup, so blocking is