Skip to content

armgit / codex

Public Imported from GitHub
0

Lightweight coding agent that runs in your terminal

Rust

Avoid full-string scans in token-budget truncation (#49712)

jif5e96aabd68dc
diff --git a/codex-rs/utils/string/src/truncate.rs b/codex-rs/utils/string/src/truncate.rs
index 2d09d22a1e..c0d1dd67d5 100644
--- a/codex-rs/utils/string/src/truncate.rs
+++ b/codex-rs/utils/string/src/truncate.rs
@@ -40,9 +40,8 @@ fn truncate_with_byte_estimate(s: &str, max_bytes: usize, use_tokens: bool) -> S
return String::new();
}
- let total_chars = s.chars().count();
-
if max_bytes == 0 {
+ let total_chars = if use_tokens { 0 } else { s.chars().count() };
return format_truncation_marker(
use_tokens,
removed_units(use_tokens, s.len(), total_chars),
@@ -55,7 +54,14 @@ fn truncate_with_byte_estimate(s: &str, max_bytes: usize, use_tokens: bool) -> S
let total_bytes = s.len();
let (left_budget, right_budget) = split_budget(max_bytes);
- let (removed_chars, left, right) = split_string(s, left_budget, right_budget);
+ let (removed_chars, left, right) = if use_tokens {
+ // Token markers depend only on byte counts; avoid scanning the discarded middle.
+ let prefix_end = s.floor_char_boundary(left_budget);
+ let suffix_start = s.ceil_char_boundary(total_bytes - right_budget);
+ (0, &s[..prefix_end], &s[suffix_start..])
+ } else {
+ split_string(s, left_budget, right_budget)
+ };
let marker = format_truncation_marker(
use_tokens,
removed_units(
diff --git a/codex-rs/utils/string/src/truncate/tests.rs b/codex-rs/utils/string/src/truncate/tests.rs
index 3549cb228c..a8b5fd3d13 100644
--- a/codex-rs/utils/string/src/truncate/tests.rs
+++ b/codex-rs/utils/string/src/truncate/tests.rs
@@ -109,6 +109,20 @@ fn truncate_middle_tokens_handles_utf8_content() {
assert_eq!(tokens, Some(16));
}
+#[test]
+fn truncate_middle_tokens_keeps_partial_characters_out_of_both_ends() {
+ for (input, expected) in [
+ ("a😀b😀c", "a…2 tokens truncated…c"),
+ ("é中😀é", "é…2 tokens truncated…é"),
+ ("😀😀😀", "…2 tokens truncated…"),
+ ] {
+ assert_eq!(
+ truncate_middle_with_token_budget(input, /*max_tokens*/ 1),
+ (expected.to_string(), Some(3)),
+ );
+ }
+}
+
#[test]
fn truncate_middle_bytes_handles_utf8_content() {
let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with text\n";