Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy
Rust
fix(codex): restore o200k token estimates (#90)
mulfyxef95d579ca23
diff --git a/Cargo.lock b/Cargo.lockindex 6cd1554..d5bf447 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -188,6 +188,21 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "bit-set"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"+dependencies = [+ "bit-vec",+]++[[package]]+name = "bit-vec"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"+[[package]]name = "bitflags"version = "1.3.2"@@ -350,6 +365,7 @@ dependencies = ["sha2","tempfile","thiserror 2.0.18",+ "tiktoken-rs","time","tokio","tokio-rustls",@@ -586,6 +602,17 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "fancy-regex"+version = "0.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"+dependencies = [+ "bit-set",+ "regex-automata",+ "regex-syntax",+]+[[package]]name = "fastrand"version = "2.4.1"@@ -2141,6 +2168,21 @@ dependencies = ["cfg-if",]+[[package]]+name = "tiktoken-rs"+version = "0.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"+dependencies = [+ "anyhow",+ "base64",+ "bstr",+ "fancy-regex",+ "lazy_static",+ "regex",+ "rustc-hash",+]+[[package]]name = "time"version = "0.3.51"diff --git a/Cargo.toml b/Cargo.tomlindex 1a475c7..866c368 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -45,6 +45,7 @@ flate2 = "1.0"ratatui = "0.29"crossterm = "0.28"jiff = { version = "0.2", default-features = false, features = ["std", "tz-system", "tzdb-zoneinfo"] }+tiktoken-rs = { version = "0.12", default-features = false }[dev-dependencies]tempfile = "3.14"diff --git a/docs/src/content/docs/how-it-works.md b/docs/src/content/docs/how-it-works.mdindex 1bb537e..124d0c5 100644--- a/docs/src/content/docs/how-it-works.md+++ b/docs/src/content/docs/how-it-works.md@@ -43,6 +43,6 @@ Claude Code sends `x-claude-code-session-id`. The proxy uses it for monitor grou## Count tokens-`POST /v1/messages/count_tokens` performs a local estimate with `gpt-tokenizer` and the `o200k_base` encoding. It supports Claude Code's compaction decisions without an upstream request.+`POST /v1/messages/count_tokens` stays local and does not make an upstream request. Codex uses `o200k_base` for text plus estimates for images, encrypted reasoning, and protocol framing. Other providers use their own local estimators. The result supports Claude Code's compaction decisions but is not a provider billing count.See [HTTP API](/reference/http-api/) for route contracts and [Compatibility and limitations](/reference/compatibility-and-limitations/) for translation boundaries.diff --git a/docs/src/content/docs/reference/http-api.md b/docs/src/content/docs/reference/http-api.mdindex 489cbce..3b25b60 100644--- a/docs/src/content/docs/reference/http-api.md+++ b/docs/src/content/docs/reference/http-api.md@@ -37,7 +37,7 @@ Accepts the same basic Anthropic request shape and returns:{"input_tokens":1234}```-Codex, Kimi, and Grok use a local `gpt-tokenizer` estimate with `o200k_base`. Cursor estimates the rendered prompt from its character length. Counts support Claude Code compaction behavior and are estimates rather than provider billing values.+Codex tokenizes text locally with `o200k_base` and adds estimates for images, encrypted reasoning, and protocol framing. Kimi and Grok use local text heuristics, while Cursor estimates the rendered prompt from its character length. Counts support Claude Code compaction behavior and are estimates rather than provider billing values.## `GET /v1/models`diff --git a/src/providers/codex/count_tokens.rs b/src/providers/codex/count_tokens.rsindex 57051f8..85c0b99 100644--- a/src/providers/codex/count_tokens.rs+++ b/src/providers/codex/count_tokens.rs@@ -2,10 +2,12 @@ use super::translate::request::{ResponsesContentPart, ResponsesFunctionCallOutput, ResponsesFunctionCallOutputContentPart,ResponsesInputItem, ResponsesRequest, ResponsesTool,};+use tiktoken_rs::o200k_base_singleton;/// Approximate token counter for Codex translated requests.-/// Uses a simple monotonic estimator that satisfies Claude Code's-/// compaction logic (needs approximate, not exact counts).+/// Text uses the OpenAI `o200k_base` tokenizer. Images, encrypted reasoning,+/// and protocol framing still use local estimates, so the result is not a+/// provider billing count.pub fn count_translated_tokens(translated: &ResponsesRequest) -> u64 {let mut total = 0u64;@@ -117,27 +119,30 @@ fn count_tool_tokens(tools: &[ResponsesTool]) -> u64 {}pub(crate) fn approx_token_count(text: &str) -> u64 {- if text.is_empty() {- return 0;+ u64::try_from(o200k_base_singleton().count_ordinary(text)).unwrap_or(u64::MAX)+}++pub(crate) fn truncate_to_token_budget(text: &str, max_tokens: u64) -> String {+ let max_tokens = usize::try_from(max_tokens).unwrap_or(usize::MAX);+ if max_tokens == 0 {+ return String::new();}- let mut count = 0u64;- let mut in_word = false;- for ch in text.chars() {- if ch.is_alphanumeric() || ch == '-' || ch == '_' {- if !in_word {- count += 1;- in_word = true;- }- } else {- in_word = false;- if !ch.is_whitespace() {- count += 1;- }+ let tokenizer = o200k_base_singleton();+ let tokens = tokenizer.encode_ordinary(text);+ if tokens.len() <= max_tokens {+ return text.to_string();+ }++ // A token boundary may split a multi-byte UTF-8 scalar. Back up until the+ // decoded prefix is valid rather than replacing bytes or splitting text.+ for end in (1..=max_tokens).rev() {+ if let Ok(prefix) = tokenizer.decode(&tokens[..end]) {+ return prefix;}}- count.max(1)+ String::new()}#[cfg(test)]@@ -199,6 +204,34 @@ mod tests {assert!(count_translated_tokens(&long) >= count_translated_tokens(&short));}+ #[test]+ fn o200k_counts_unbroken_text_instead_of_collapsing_it() {+ assert_eq!(approx_token_count(&"a".repeat(4096)), 512);+ assert_eq!(approx_token_count(&"汉字测试上下文压缩".repeat(384)), 2688);+ assert_eq!(+ approx_token_count(&"QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo".repeat(112)),+ 2240+ );+ assert_eq!(+ approx_token_count(&"function f(a){return a===0?1:a*f(a-1)};".repeat(96)),+ 1632+ );+ }++ #[test]+ fn truncation_respects_o200k_budget_and_utf8_boundaries() {+ for text in [+ "a".repeat(4096),+ "汉字测试上下文压缩".repeat(384),+ "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo".repeat(112),+ ] {+ let truncated = truncate_to_token_budget(&text, 100);+ assert!(text.starts_with(&truncated));+ assert!(approx_token_count(&truncated) <= 100);+ assert!(!truncated.is_empty());+ }+ }+#[test]fn structured_tool_output_counts_text_and_images() {let text = ResponsesFunctionCallOutput::Text("caption".to_string());diff --git a/src/providers/codex/search.rs b/src/providers/codex/search.rsindex 8a787af..60a846a 100644--- a/src/providers/codex/search.rs+++ b/src/providers/codex/search.rs@@ -7,7 +7,7 @@ use crate::anthropic::schema::MessagesRequest;use crate::anthropic::sse::encode_sse_event;use crate::traffic::TrafficCapture;-use super::count_tokens::approx_token_count;+use super::count_tokens::{approx_token_count, truncate_to_token_budget};const SEARCH_OUTPUT_TOKEN_BUDGET: u64 = 2_500;const SEARCH_ASSISTANT_CONTEXT_TOKEN_BUDGET: u64 = 1_000;@@ -364,7 +364,7 @@ fn search_input(req: &MessagesRequest) -> Option<Value> {if assistant_budget == 0 {return None;}- let text = truncate_to_approx_tokens(&text, assistant_budget);+ let text = truncate_to_token_budget(&text, assistant_budget);assistant_budget = assistant_budget.saturating_sub(approx_token_count(&text));("output_text", text)} else {@@ -382,33 +382,6 @@ fn search_input(req: &MessagesRequest) -> Option<Value> {(!items.is_empty()).then_some(Value::Array(items))}-fn truncate_to_approx_tokens(text: &str, max_tokens: u64) -> String {- if approx_token_count(text) <= max_tokens {- return text.to_string();- }-- let mut tokens = 0_u64;- let mut in_word = false;- let mut end = 0;- for (index, ch) in text.char_indices() {- let word_char = ch.is_alphanumeric() || ch == '-' || ch == '_';- let starts_token = if word_char {- !in_word- } else {- !ch.is_whitespace()- };- if starts_token {- if tokens == max_tokens {- break;- }- tokens += 1;- }- in_word = word_char;- end = index + ch.len_utf8();- }- text[..end].to_string()-}-fn content_text(content: &Value) -> String {match content {Value::String(text) => text.clone(),