Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
Revert "feat(consolidate): add input_token_safety_margin to bound approximate token budget (#884)"
AkitaOnRails0fc60ffc2421
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 249e11e4..b7599878 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,15 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]### Added-- `[consolidation] input_token_safety_margin` (float, default `0.8`, validated- to `(0.0, 1.0]`) scales the approximate char-count input budget. The- `max_input_tokens` budget uses a flat chars-per-token heuristic that- under-budgets denser corpora — pt-BR text and source code tokenize at fewer- chars per token than English and could overshoot a provider's real input- limit by ~40%. The default tightens the common case modestly while leaving- such corpora headroom; lower it further for a mostly non-English or code- corpus. `max_input_tokens` is now documented as an approximate heuristic in- the config reference. (#884)- `docs/jev-reranker-adapter.md` documents a stdlib-only adapter(`docs/examples/jev-reranker-adapter/jev_rerank_shim.py`) that serves the`AI_MEMORY_RERANKER=llm` request leg from a Jev `/v1/systemone` judgediff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex 195ba40c..ad8206c6 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -2566,7 +2566,6 @@ fn configure_consolidator(.with_prompt_limits(config.consolidation.max_input_tokens,config.consolidation.max_output_tokens,- config.consolidation.input_token_safety_margin,),);server = server.with_consolidator_arc(wiki.clone(), llm.clone(), consolidator.clone());diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rsindex 4b567121..1e3eae2e 100644--- a/crates/ai-memory-cli/src/config.rs+++ b/crates/ai-memory-cli/src/config.rs@@ -910,15 +910,6 @@ pub struct ConsolidationSettings {/// Maximum tokens the provider may generate for a consolidation response./// Small-context models must lower this together with `max_input_tokens`.pub max_output_tokens: u32,- /// Safety margin applied to the `max_input_tokens` budget (`0 < m <= 1`).- ///- /// `max_input_tokens` is an approximate char-count heuristic (a flat- /// chars-per-token ratio), so it under-budgets denser corpora — pt-BR text- /// and source code tokenize at fewer chars per token than English prose and- /// can overshoot a provider's real input limit by ~40%. This margin shrinks- /// the effective char budget (default 0.8); lower it further for a corpus- /// that is mostly non-English or code. (#884)- pub input_token_safety_margin: f64,}impl Default for ConsolidationSettings {@@ -926,8 +917,6 @@ impl Default for ConsolidationSettings {Self {max_input_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,max_output_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,- input_token_safety_margin:- ai_memory_consolidate::DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN,}}}@@ -1448,21 +1437,6 @@ impl Config {config.consolidation.max_output_tokens);}- // The safety margin scales the input budget, so a non-positive value- // would starve every prompt and one above 1.0 would loosen the budget- // past the nominal token limit it is meant to tighten (#884). NaN also- // fails every comparison below, so it is rejected here too.- let safety_margin = config.consolidation.input_token_safety_margin;- if !(safety_margin > 0.0 && safety_margin <= 1.0) {- anyhow::bail!(- "consolidation.input_token_safety_margin must be in (0.0, 1.0] \- (got {safety_margin}); it scales the approximate input-token budget"- );- }- // The safety margin scales the input budget, so a non-positive value- // would starve every prompt and one above 1.0 would loosen the budget- // past the nominal token limit it is meant to tighten (#884). NaN also- // fails every comparison below, so it is rejected here too.// Zero (or a sub-second remainder rounded down) would cut every// provider request off before it is sent.if config.llm_timeout_secs == 0 {@@ -2709,49 +2683,6 @@ mod tests {}}- /// #884: the input-token safety margin must stay in `(0.0, 1.0]` — a- /// non-positive value starves every prompt and a value above 1.0 loosens- /// the budget past the limit it exists to tighten.- #[test]- fn load_rejects_an_out_of_range_input_token_safety_margin() {- for value in ["0.0", "-0.1", "1.5", "nan"] {- let tmp = TempDir::new().unwrap();- let config_path = tmp.path().join("config.toml");- std::fs::write(- &config_path,- format!("[consolidation]\ninput_token_safety_margin = {value}\n"),- )- .unwrap();- let error = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))- .expect_err("an out-of-range safety margin must fail closed");- assert!(- error- .to_string()- .contains("consolidation.input_token_safety_margin"),- "unexpected error for {value}: {error:#}"- );- }- }-- /// A valid margin survives the config round-trip and the default is 0.8.- #[test]- fn load_accepts_a_valid_input_token_safety_margin() {- let tmp = TempDir::new().unwrap();- let config_path = tmp.path().join("config.toml");- std::fs::write(- &config_path,- "[consolidation]\ninput_token_safety_margin = 0.6\n",- )- .unwrap();- let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))- .expect("a margin inside (0.0, 1.0] must load");- assert_eq!(config.consolidation.input_token_safety_margin, 0.6);- assert_eq!(- ConsolidationSettings::default().input_token_safety_margin,- 0.8- );- }-/// A small-context provider needs both sides of the context allocation to/// survive the config round-trip.#[test]diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rsindex d9b59029..8afdf9ed 100644--- a/crates/ai-memory-consolidate/src/consolidator.rs+++ b/crates/ai-memory-consolidate/src/consolidator.rs@@ -154,23 +154,9 @@ impl Consolidator {////// `max_input_tokens + max_output_tokens` must fit the provider's context/// window. Callers validate the supported minimums when resolving config.- ///- /// `safety_margin` shrinks the char-count input budget so the flat- /// chars-per-token heuristic does not over-admit on denser-than-English- /// corpora (pt-BR, code); it is validated to `0 < margin <= 1` at config- /// load. See [`DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN`] (#884).#[must_use]- pub fn with_prompt_limits(- mut self,- max_input_tokens: usize,- max_output_tokens: u32,- safety_margin: f64,- ) -> Self {- self.budgets = PromptBudgets::from_limits_with_margin(- max_input_tokens,- max_output_tokens,- safety_margin,- );+ pub fn with_prompt_limits(mut self, max_input_tokens: usize, max_output_tokens: u32) -> Self {+ self.budgets = PromptBudgets::from_limits(max_input_tokens, max_output_tokens);self}@@ -1151,17 +1137,6 @@ pub const DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS: usize = 100_000;/// Default maximum generated tokens for a consolidation response.pub const DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS: u32 = 32_000;-/// Default multiplier applied to the char-count input budget (#884).-///-/// `max_input_tokens` is turned into a char budget with a flat-/// [`CHARS_PER_TOKEN`] heuristic. That ratio holds for English prose but-/// over-admits on denser corpora — pt-BR prose and source code tokenize at-/// closer to ~2.1 chars/token, so a 3:1 estimate overshot the real token-/// count by ~40% and tripped provider `max_input_tokens` limits. Shrinking-/// the effective char budget to 80% of the nominal value buys that headroom-/// back for the common case while leaving English budgets close to before.-pub const DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN: f64 = 0.8;-/// Conservative character-to-token estimate for provider-neutral budgeting./// The exact tokenizer is provider/model-specific, so this is a target rather/// than a hard token count. Three characters per token plus the default@@ -1215,26 +1190,8 @@ struct PromptBudgets {impl PromptBudgets {fn from_limits(max_input_tokens: usize, max_output_tokens: u32) -> Self {- // A bare limit applies no safety margin (margin 1.0), preserving the- // historical char budget; production tightens it via config (#884).- Self::from_limits_with_margin(max_input_tokens, max_output_tokens, 1.0)- }-- /// Derive budgets from the token limits, shrinking the char-count input- /// budget by `safety_margin` (#884). The margin compensates for the flat- /// [`CHARS_PER_TOKEN`] heuristic over-admitting on denser-than-English- /// corpora (pt-BR, code). Callers pass a validated `0 < margin <= 1`.- fn from_limits_with_margin(- max_input_tokens: usize,- max_output_tokens: u32,- safety_margin: f64,- ) -> Self {- let nominal = max_input_tokens.saturating_mul(CHARS_PER_TOKEN);- // `safety_margin` is validated to `0 < margin <= 1` at config load, so- // the product never exceeds `nominal` and the cast cannot overflow.- let max_input_chars = (nominal as f64 * safety_margin) as usize;Self {- max_input_chars,+ max_input_chars: max_input_tokens.saturating_mul(CHARS_PER_TOKEN),max_output_tokens,}}@@ -1799,54 +1756,6 @@ mod tests {);}- /// #884: the safety margin scales the char budget by exactly the factor,- /// and margin 1.0 reproduces the un-margined budget.- #[test]- fn safety_margin_scales_input_char_budget() {- let full = PromptBudgets::from_limits(10_000, 1_000);- assert_eq!(full.max_input_chars, 30_000);- assert_eq!(- PromptBudgets::from_limits_with_margin(10_000, 1_000, 1.0),- full,- "margin 1.0 must leave the budget unchanged"- );- let tightened = PromptBudgets::from_limits_with_margin(10_000, 1_000, 0.8);- assert_eq!(- tightened.max_input_chars, 24_000,- "margin 0.8 must shrink the char budget to 0.8x"- );- // Output allowance is independent of the input margin.- assert_eq!(tightened.max_output_tokens, 1_000);- }-- /// #884: a large observation set is packed to the tightened budget, so a- /// margin actually reduces how much prompt content is admitted.- #[test]- fn safety_margin_trims_packed_observations() {- let tightened = PromptBudgets::from_limits_with_margin(- DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,- DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,- 0.8,- );- let full = PromptBudgets::from_limits(- DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,- DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,- );- assert!(tightened.max_input_chars < full.max_input_chars);- let observations = (0..256).map(|_| obs_of_size(4_000)).collect::<Vec<_>>();- let request = build_request(- SessionId::new(),- &observations,- &"x".repeat(50_000),- Some(&"preference ".repeat(500)),- tightened,- );- assert!(- estimated_input_chars::<ConsolidatedPage>(&request) <= tightened.max_input_chars,- "packed request must respect the tightened budget"- );- }-/// The batch path must include schema and dynamic slot snapshots in its/// envelope instead of assuming a fixed number of slots.#[test]diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rsindex a48b692c..32c3b6dc 100644--- a/crates/ai-memory-consolidate/src/lib.rs+++ b/crates/ai-memory-consolidate/src/lib.rs@@ -62,9 +62,8 @@ pub use cold_cluster::{adaptive_eps, cosine_distance, dbscan};pub use compaction::build_compacted_markdown;pub use consolidator::{BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult,- DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN, DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,- DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS, MIN_CONSOLIDATION_MAX_INPUT_TOKENS,- MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,+ DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS, DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,+ MIN_CONSOLIDATION_MAX_INPUT_TOKENS, MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,};pub use curator::{CuratorFinding, CuratorParams, CuratorReport, render_curator_report_markdown,diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 9a701f4a..267b117c 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -688,17 +688,9 @@ per_user = false # shared + own slots in agent context[consolidation] # LLM consolidation prompt sizingmax_input_tokens = 100000 # approximate whole-input target; min 6000- # a flat chars-per-token heuristic, so it- # UNDER-budgets denser corpora: pt-BR prose- # and source code tokenize at fewer chars per- # token than English and can overshoot the- # provider's real limit by ~40% — lower this- # (or input_token_safety_margin) for such a corpusmax_output_tokens = 32000 # provider generation limit; min 1000# their sum must fit the model context window;# leave headroom for tokenizer variance-input_token_safety_margin = 0.8 # scales the char budget, (0.0, 1.0]; the- # 0.8 default buys pt-BR/code headroom[auto_improve] # default-available learning reviewerrequire_approval = false # true leaves proposals pending for review