Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(consolidate): add input_token_safety_margin to bound approximate token budget (#884)

AkitaOnRails7ff429935fd4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83cb5716..451b3971 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
+- `[consolidation] input_token_safety_margin` (float, default `0.8`, validated
+ to `(0.0, 1.0]`) scales the approximate char-count input budget. The
+ `max_input_tokens` budget uses a flat chars-per-token heuristic that
+ under-budgets denser corpora — pt-BR text and source code tokenize at fewer
+ chars per token than English and could overshoot a provider's real input
+ limit by ~40%. The default tightens the common case modestly while leaving
+ such corpora headroom; lower it further for a mostly non-English or code
+ corpus. `max_input_tokens` is now documented as an approximate heuristic in
+ the config reference. (#884)
- `docs/jev-reranker-adapter.md` documents a stdlib-only adapter
(`docs/examples/jev-reranker-adapter/jev_rerank_shim.py`) that serves the
`AI_MEMORY_RERANKER=llm` request leg from a Jev `/v1/systemone` judge
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index d89c8d0e..688936cc 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -2499,6 +2499,7 @@ fn configure_consolidator(
.with_prompt_limits(
config.consolidation.max_input_tokens,
config.consolidation.max_output_tokens,
+ config.consolidation.input_token_safety_margin,
),
);
server = server.with_consolidator_arc(wiki.clone(), llm.clone(), consolidator.clone());
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 1e3eae2e..4b567121 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -910,6 +910,15 @@ pub struct ConsolidationSettings {
/// Maximum tokens the provider may generate for a consolidation response.
/// Small-context models must lower this together with `max_input_tokens`.
pub max_output_tokens: u32,
+ /// Safety margin applied to the `max_input_tokens` budget (`0 < m <= 1`).
+ ///
+ /// `max_input_tokens` is an approximate char-count heuristic (a flat
+ /// chars-per-token ratio), so it under-budgets denser corpora — pt-BR text
+ /// and source code tokenize at fewer chars per token than English prose and
+ /// can overshoot a provider's real input limit by ~40%. This margin shrinks
+ /// the effective char budget (default 0.8); lower it further for a corpus
+ /// that is mostly non-English or code. (#884)
+ pub input_token_safety_margin: f64,
}
impl Default for ConsolidationSettings {
@@ -917,6 +926,8 @@ impl Default for ConsolidationSettings {
Self {
max_input_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
max_output_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
+ input_token_safety_margin:
+ ai_memory_consolidate::DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN,
}
}
}
@@ -1437,6 +1448,21 @@ impl Config {
config.consolidation.max_output_tokens
);
}
+ // The safety margin scales the input budget, so a non-positive value
+ // would starve every prompt and one above 1.0 would loosen the budget
+ // past the nominal token limit it is meant to tighten (#884). NaN also
+ // fails every comparison below, so it is rejected here too.
+ let safety_margin = config.consolidation.input_token_safety_margin;
+ if !(safety_margin > 0.0 && safety_margin <= 1.0) {
+ anyhow::bail!(
+ "consolidation.input_token_safety_margin must be in (0.0, 1.0] \
+ (got {safety_margin}); it scales the approximate input-token budget"
+ );
+ }
+ // The safety margin scales the input budget, so a non-positive value
+ // would starve every prompt and one above 1.0 would loosen the budget
+ // past the nominal token limit it is meant to tighten (#884). NaN also
+ // fails every comparison below, so it is rejected here too.
// Zero (or a sub-second remainder rounded down) would cut every
// provider request off before it is sent.
if config.llm_timeout_secs == 0 {
@@ -2683,6 +2709,49 @@ mod tests {
}
}
+ /// #884: the input-token safety margin must stay in `(0.0, 1.0]` — a
+ /// non-positive value starves every prompt and a value above 1.0 loosens
+ /// the budget past the limit it exists to tighten.
+ #[test]
+ fn load_rejects_an_out_of_range_input_token_safety_margin() {
+ for value in ["0.0", "-0.1", "1.5", "nan"] {
+ let tmp = TempDir::new().unwrap();
+ let config_path = tmp.path().join("config.toml");
+ std::fs::write(
+ &config_path,
+ format!("[consolidation]\ninput_token_safety_margin = {value}\n"),
+ )
+ .unwrap();
+ let error = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))
+ .expect_err("an out-of-range safety margin must fail closed");
+ assert!(
+ error
+ .to_string()
+ .contains("consolidation.input_token_safety_margin"),
+ "unexpected error for {value}: {error:#}"
+ );
+ }
+ }
+
+ /// A valid margin survives the config round-trip and the default is 0.8.
+ #[test]
+ fn load_accepts_a_valid_input_token_safety_margin() {
+ let tmp = TempDir::new().unwrap();
+ let config_path = tmp.path().join("config.toml");
+ std::fs::write(
+ &config_path,
+ "[consolidation]\ninput_token_safety_margin = 0.6\n",
+ )
+ .unwrap();
+ let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))
+ .expect("a margin inside (0.0, 1.0] must load");
+ assert_eq!(config.consolidation.input_token_safety_margin, 0.6);
+ assert_eq!(
+ ConsolidationSettings::default().input_token_safety_margin,
+ 0.8
+ );
+ }
+
/// A small-context provider needs both sides of the context allocation to
/// survive the config round-trip.
#[test]
diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rs
index 578fd8cd..cd83efa4 100644
--- a/crates/ai-memory-consolidate/src/consolidator.rs
+++ b/crates/ai-memory-consolidate/src/consolidator.rs
@@ -154,9 +154,23 @@ impl Consolidator {
///
/// `max_input_tokens + max_output_tokens` must fit the provider's context
/// window. Callers validate the supported minimums when resolving config.
+ ///
+ /// `safety_margin` shrinks the char-count input budget so the flat
+ /// chars-per-token heuristic does not over-admit on denser-than-English
+ /// corpora (pt-BR, code); it is validated to `0 < margin <= 1` at config
+ /// load. See [`DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN`] (#884).
#[must_use]
- pub fn with_prompt_limits(mut self, max_input_tokens: usize, max_output_tokens: u32) -> Self {
- self.budgets = PromptBudgets::from_limits(max_input_tokens, max_output_tokens);
+ pub fn with_prompt_limits(
+ mut self,
+ max_input_tokens: usize,
+ max_output_tokens: u32,
+ safety_margin: f64,
+ ) -> Self {
+ self.budgets = PromptBudgets::from_limits_with_margin(
+ max_input_tokens,
+ max_output_tokens,
+ safety_margin,
+ );
self
}
@@ -1137,6 +1151,17 @@ pub const DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS: usize = 100_000;
/// Default maximum generated tokens for a consolidation response.
pub const DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS: u32 = 32_000;
+/// Default multiplier applied to the char-count input budget (#884).
+///
+/// `max_input_tokens` is turned into a char budget with a flat
+/// [`CHARS_PER_TOKEN`] heuristic. That ratio holds for English prose but
+/// over-admits on denser corpora — pt-BR prose and source code tokenize at
+/// closer to ~2.1 chars/token, so a 3:1 estimate overshot the real token
+/// count by ~40% and tripped provider `max_input_tokens` limits. Shrinking
+/// the effective char budget to 80% of the nominal value buys that headroom
+/// back for the common case while leaving English budgets close to before.
+pub const DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN: f64 = 0.8;
+
/// Conservative character-to-token estimate for provider-neutral budgeting.
/// The exact tokenizer is provider/model-specific, so this is a target rather
/// than a hard token count. Three characters per token plus the default
@@ -1190,8 +1215,26 @@ struct PromptBudgets {
impl PromptBudgets {
fn from_limits(max_input_tokens: usize, max_output_tokens: u32) -> Self {
+ // A bare limit applies no safety margin (margin 1.0), preserving the
+ // historical char budget; production tightens it via config (#884).
+ Self::from_limits_with_margin(max_input_tokens, max_output_tokens, 1.0)
+ }
+
+ /// Derive budgets from the token limits, shrinking the char-count input
+ /// budget by `safety_margin` (#884). The margin compensates for the flat
+ /// [`CHARS_PER_TOKEN`] heuristic over-admitting on denser-than-English
+ /// corpora (pt-BR, code). Callers pass a validated `0 < margin <= 1`.
+ fn from_limits_with_margin(
+ max_input_tokens: usize,
+ max_output_tokens: u32,
+ safety_margin: f64,
+ ) -> Self {
+ let nominal = max_input_tokens.saturating_mul(CHARS_PER_TOKEN);
+ // `safety_margin` is validated to `0 < margin <= 1` at config load, so
+ // the product never exceeds `nominal` and the cast cannot overflow.
+ let max_input_chars = (nominal as f64 * safety_margin) as usize;
Self {
- max_input_chars: max_input_tokens.saturating_mul(CHARS_PER_TOKEN),
+ max_input_chars,
max_output_tokens,
}
}
@@ -1752,6 +1795,54 @@ mod tests {
);
}
+ /// #884: the safety margin scales the char budget by exactly the factor,
+ /// and margin 1.0 reproduces the un-margined budget.
+ #[test]
+ fn safety_margin_scales_input_char_budget() {
+ let full = PromptBudgets::from_limits(10_000, 1_000);
+ assert_eq!(full.max_input_chars, 30_000);
+ assert_eq!(
+ PromptBudgets::from_limits_with_margin(10_000, 1_000, 1.0),
+ full,
+ "margin 1.0 must leave the budget unchanged"
+ );
+ let tightened = PromptBudgets::from_limits_with_margin(10_000, 1_000, 0.8);
+ assert_eq!(
+ tightened.max_input_chars, 24_000,
+ "margin 0.8 must shrink the char budget to 0.8x"
+ );
+ // Output allowance is independent of the input margin.
+ assert_eq!(tightened.max_output_tokens, 1_000);
+ }
+
+ /// #884: a large observation set is packed to the tightened budget, so a
+ /// margin actually reduces how much prompt content is admitted.
+ #[test]
+ fn safety_margin_trims_packed_observations() {
+ let tightened = PromptBudgets::from_limits_with_margin(
+ DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
+ DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
+ 0.8,
+ );
+ let full = PromptBudgets::from_limits(
+ DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
+ DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
+ );
+ assert!(tightened.max_input_chars < full.max_input_chars);
+ let observations = (0..256).map(|_| obs_of_size(4_000)).collect::<Vec<_>>();
+ let request = build_request(
+ SessionId::new(),
+ &observations,
+ &"x".repeat(50_000),
+ Some(&"preference ".repeat(500)),
+ tightened,
+ );
+ assert!(
+ estimated_input_chars::<ConsolidatedPage>(&request) <= tightened.max_input_chars,
+ "packed request must respect the tightened budget"
+ );
+ }
+
/// The batch path must include schema and dynamic slot snapshots in its
/// envelope instead of assuming a fixed number of slots.
#[test]
diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rs
index 32c3b6dc..a48b692c 100644
--- a/crates/ai-memory-consolidate/src/lib.rs
+++ b/crates/ai-memory-consolidate/src/lib.rs
@@ -62,8 +62,9 @@ pub use cold_cluster::{adaptive_eps, cosine_distance, dbscan};
pub use compaction::build_compacted_markdown;
pub use consolidator::{
BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult,
- DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS, DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
- MIN_CONSOLIDATION_MAX_INPUT_TOKENS, MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,
+ DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN, DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
+ DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS, MIN_CONSOLIDATION_MAX_INPUT_TOKENS,
+ MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,
};
pub use curator::{
CuratorFinding, CuratorParams, CuratorReport, render_curator_report_markdown,
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 7396f265..7936fef1 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -664,9 +664,17 @@ per_user = false # shared + own slots in agent context
[consolidation] # LLM consolidation prompt sizing
max_input_tokens = 100000 # approximate whole-input target; min 6000
+ # a flat chars-per-token heuristic, so it
+ # UNDER-budgets denser corpora: pt-BR prose
+ # and source code tokenize at fewer chars per
+ # token than English and can overshoot the
+ # provider's real limit by ~40% — lower this
+ # (or input_token_safety_margin) for such a corpus
max_output_tokens = 32000 # provider generation limit; min 1000
# their sum must fit the model context window;
# leave headroom for tokenizer variance
+input_token_safety_margin = 0.8 # scales the char budget, (0.0, 1.0]; the
+ # 0.8 default buys pt-BR/code headroom
[auto_improve] # default-available learning reviewer
require_approval = false # true leaves proposals pending for review