Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

Revert "feat(consolidate): add input_token_safety_margin to bound approximate token budget (#884)"

AkitaOnRails0fc60ffc2421
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 249e11e4..b7599878 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,15 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
-- `[consolidation] input_token_safety_margin` (float, default `0.8`, validated
- to `(0.0, 1.0]`) scales the approximate char-count input budget. The
- `max_input_tokens` budget uses a flat chars-per-token heuristic that
- under-budgets denser corpora — pt-BR text and source code tokenize at fewer
- chars per token than English and could overshoot a provider's real input
- limit by ~40%. The default tightens the common case modestly while leaving
- such corpora headroom; lower it further for a mostly non-English or code
- corpus. `max_input_tokens` is now documented as an approximate heuristic in
- the config reference. (#884)
- `docs/jev-reranker-adapter.md` documents a stdlib-only adapter
(`docs/examples/jev-reranker-adapter/jev_rerank_shim.py`) that serves the
`AI_MEMORY_RERANKER=llm` request leg from a Jev `/v1/systemone` judge
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index 195ba40c..ad8206c6 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -2566,7 +2566,6 @@ fn configure_consolidator(
.with_prompt_limits(
config.consolidation.max_input_tokens,
config.consolidation.max_output_tokens,
- config.consolidation.input_token_safety_margin,
),
);
server = server.with_consolidator_arc(wiki.clone(), llm.clone(), consolidator.clone());
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 4b567121..1e3eae2e 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -910,15 +910,6 @@ pub struct ConsolidationSettings {
/// Maximum tokens the provider may generate for a consolidation response.
/// Small-context models must lower this together with `max_input_tokens`.
pub max_output_tokens: u32,
- /// Safety margin applied to the `max_input_tokens` budget (`0 < m <= 1`).
- ///
- /// `max_input_tokens` is an approximate char-count heuristic (a flat
- /// chars-per-token ratio), so it under-budgets denser corpora — pt-BR text
- /// and source code tokenize at fewer chars per token than English prose and
- /// can overshoot a provider's real input limit by ~40%. This margin shrinks
- /// the effective char budget (default 0.8); lower it further for a corpus
- /// that is mostly non-English or code. (#884)
- pub input_token_safety_margin: f64,
}
impl Default for ConsolidationSettings {
@@ -926,8 +917,6 @@ impl Default for ConsolidationSettings {
Self {
max_input_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
max_output_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
- input_token_safety_margin:
- ai_memory_consolidate::DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN,
}
}
}
@@ -1448,21 +1437,6 @@ impl Config {
config.consolidation.max_output_tokens
);
}
- // The safety margin scales the input budget, so a non-positive value
- // would starve every prompt and one above 1.0 would loosen the budget
- // past the nominal token limit it is meant to tighten (#884). NaN also
- // fails every comparison below, so it is rejected here too.
- let safety_margin = config.consolidation.input_token_safety_margin;
- if !(safety_margin > 0.0 && safety_margin <= 1.0) {
- anyhow::bail!(
- "consolidation.input_token_safety_margin must be in (0.0, 1.0] \
- (got {safety_margin}); it scales the approximate input-token budget"
- );
- }
- // The safety margin scales the input budget, so a non-positive value
- // would starve every prompt and one above 1.0 would loosen the budget
- // past the nominal token limit it is meant to tighten (#884). NaN also
- // fails every comparison below, so it is rejected here too.
// Zero (or a sub-second remainder rounded down) would cut every
// provider request off before it is sent.
if config.llm_timeout_secs == 0 {
@@ -2709,49 +2683,6 @@ mod tests {
}
}
- /// #884: the input-token safety margin must stay in `(0.0, 1.0]` — a
- /// non-positive value starves every prompt and a value above 1.0 loosens
- /// the budget past the limit it exists to tighten.
- #[test]
- fn load_rejects_an_out_of_range_input_token_safety_margin() {
- for value in ["0.0", "-0.1", "1.5", "nan"] {
- let tmp = TempDir::new().unwrap();
- let config_path = tmp.path().join("config.toml");
- std::fs::write(
- &config_path,
- format!("[consolidation]\ninput_token_safety_margin = {value}\n"),
- )
- .unwrap();
- let error = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))
- .expect_err("an out-of-range safety margin must fail closed");
- assert!(
- error
- .to_string()
- .contains("consolidation.input_token_safety_margin"),
- "unexpected error for {value}: {error:#}"
- );
- }
- }
-
- /// A valid margin survives the config round-trip and the default is 0.8.
- #[test]
- fn load_accepts_a_valid_input_token_safety_margin() {
- let tmp = TempDir::new().unwrap();
- let config_path = tmp.path().join("config.toml");
- std::fs::write(
- &config_path,
- "[consolidation]\ninput_token_safety_margin = 0.6\n",
- )
- .unwrap();
- let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))
- .expect("a margin inside (0.0, 1.0] must load");
- assert_eq!(config.consolidation.input_token_safety_margin, 0.6);
- assert_eq!(
- ConsolidationSettings::default().input_token_safety_margin,
- 0.8
- );
- }
-
/// A small-context provider needs both sides of the context allocation to
/// survive the config round-trip.
#[test]
diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rs
index d9b59029..8afdf9ed 100644
--- a/crates/ai-memory-consolidate/src/consolidator.rs
+++ b/crates/ai-memory-consolidate/src/consolidator.rs
@@ -154,23 +154,9 @@ impl Consolidator {
///
/// `max_input_tokens + max_output_tokens` must fit the provider's context
/// window. Callers validate the supported minimums when resolving config.
- ///
- /// `safety_margin` shrinks the char-count input budget so the flat
- /// chars-per-token heuristic does not over-admit on denser-than-English
- /// corpora (pt-BR, code); it is validated to `0 < margin <= 1` at config
- /// load. See [`DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN`] (#884).
#[must_use]
- pub fn with_prompt_limits(
- mut self,
- max_input_tokens: usize,
- max_output_tokens: u32,
- safety_margin: f64,
- ) -> Self {
- self.budgets = PromptBudgets::from_limits_with_margin(
- max_input_tokens,
- max_output_tokens,
- safety_margin,
- );
+ pub fn with_prompt_limits(mut self, max_input_tokens: usize, max_output_tokens: u32) -> Self {
+ self.budgets = PromptBudgets::from_limits(max_input_tokens, max_output_tokens);
self
}
@@ -1151,17 +1137,6 @@ pub const DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS: usize = 100_000;
/// Default maximum generated tokens for a consolidation response.
pub const DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS: u32 = 32_000;
-/// Default multiplier applied to the char-count input budget (#884).
-///
-/// `max_input_tokens` is turned into a char budget with a flat
-/// [`CHARS_PER_TOKEN`] heuristic. That ratio holds for English prose but
-/// over-admits on denser corpora — pt-BR prose and source code tokenize at
-/// closer to ~2.1 chars/token, so a 3:1 estimate overshot the real token
-/// count by ~40% and tripped provider `max_input_tokens` limits. Shrinking
-/// the effective char budget to 80% of the nominal value buys that headroom
-/// back for the common case while leaving English budgets close to before.
-pub const DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN: f64 = 0.8;
-
/// Conservative character-to-token estimate for provider-neutral budgeting.
/// The exact tokenizer is provider/model-specific, so this is a target rather
/// than a hard token count. Three characters per token plus the default
@@ -1215,26 +1190,8 @@ struct PromptBudgets {
impl PromptBudgets {
fn from_limits(max_input_tokens: usize, max_output_tokens: u32) -> Self {
- // A bare limit applies no safety margin (margin 1.0), preserving the
- // historical char budget; production tightens it via config (#884).
- Self::from_limits_with_margin(max_input_tokens, max_output_tokens, 1.0)
- }
-
- /// Derive budgets from the token limits, shrinking the char-count input
- /// budget by `safety_margin` (#884). The margin compensates for the flat
- /// [`CHARS_PER_TOKEN`] heuristic over-admitting on denser-than-English
- /// corpora (pt-BR, code). Callers pass a validated `0 < margin <= 1`.
- fn from_limits_with_margin(
- max_input_tokens: usize,
- max_output_tokens: u32,
- safety_margin: f64,
- ) -> Self {
- let nominal = max_input_tokens.saturating_mul(CHARS_PER_TOKEN);
- // `safety_margin` is validated to `0 < margin <= 1` at config load, so
- // the product never exceeds `nominal` and the cast cannot overflow.
- let max_input_chars = (nominal as f64 * safety_margin) as usize;
Self {
- max_input_chars,
+ max_input_chars: max_input_tokens.saturating_mul(CHARS_PER_TOKEN),
max_output_tokens,
}
}
@@ -1799,54 +1756,6 @@ mod tests {
);
}
- /// #884: the safety margin scales the char budget by exactly the factor,
- /// and margin 1.0 reproduces the un-margined budget.
- #[test]
- fn safety_margin_scales_input_char_budget() {
- let full = PromptBudgets::from_limits(10_000, 1_000);
- assert_eq!(full.max_input_chars, 30_000);
- assert_eq!(
- PromptBudgets::from_limits_with_margin(10_000, 1_000, 1.0),
- full,
- "margin 1.0 must leave the budget unchanged"
- );
- let tightened = PromptBudgets::from_limits_with_margin(10_000, 1_000, 0.8);
- assert_eq!(
- tightened.max_input_chars, 24_000,
- "margin 0.8 must shrink the char budget to 0.8x"
- );
- // Output allowance is independent of the input margin.
- assert_eq!(tightened.max_output_tokens, 1_000);
- }
-
- /// #884: a large observation set is packed to the tightened budget, so a
- /// margin actually reduces how much prompt content is admitted.
- #[test]
- fn safety_margin_trims_packed_observations() {
- let tightened = PromptBudgets::from_limits_with_margin(
- DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
- DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
- 0.8,
- );
- let full = PromptBudgets::from_limits(
- DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
- DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
- );
- assert!(tightened.max_input_chars < full.max_input_chars);
- let observations = (0..256).map(|_| obs_of_size(4_000)).collect::<Vec<_>>();
- let request = build_request(
- SessionId::new(),
- &observations,
- &"x".repeat(50_000),
- Some(&"preference ".repeat(500)),
- tightened,
- );
- assert!(
- estimated_input_chars::<ConsolidatedPage>(&request) <= tightened.max_input_chars,
- "packed request must respect the tightened budget"
- );
- }
-
/// The batch path must include schema and dynamic slot snapshots in its
/// envelope instead of assuming a fixed number of slots.
#[test]
diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rs
index a48b692c..32c3b6dc 100644
--- a/crates/ai-memory-consolidate/src/lib.rs
+++ b/crates/ai-memory-consolidate/src/lib.rs
@@ -62,9 +62,8 @@ pub use cold_cluster::{adaptive_eps, cosine_distance, dbscan};
pub use compaction::build_compacted_markdown;
pub use consolidator::{
BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult,
- DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN, DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,
- DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS, MIN_CONSOLIDATION_MAX_INPUT_TOKENS,
- MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,
+ DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS, DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,
+ MIN_CONSOLIDATION_MAX_INPUT_TOKENS, MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,
};
pub use curator::{
CuratorFinding, CuratorParams, CuratorReport, render_curator_report_markdown,
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 9a701f4a..267b117c 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -688,17 +688,9 @@ per_user = false # shared + own slots in agent context
[consolidation] # LLM consolidation prompt sizing
max_input_tokens = 100000 # approximate whole-input target; min 6000
- # a flat chars-per-token heuristic, so it
- # UNDER-budgets denser corpora: pt-BR prose
- # and source code tokenize at fewer chars per
- # token than English and can overshoot the
- # provider's real limit by ~40% — lower this
- # (or input_token_safety_margin) for such a corpus
max_output_tokens = 32000 # provider generation limit; min 1000
# their sum must fit the model context window;
# leave headroom for tokenizer variance
-input_token_safety_margin = 0.8 # scales the char budget, (0.0, 1.0]; the
- # 0.8 default buys pt-BR/code headroom
[auto_improve] # default-available learning reviewer
require_approval = false # true leaves proposals pending for review