Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
Reapply "feat(consolidate): add input_token_safety_margin to bound approximate token budget (#884)"
AkitaOnRailsb3d8e99746a5
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ee4d85fc..8c44f92f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]### Added+- `[consolidation] input_token_safety_margin` (float, default `0.8`, validated+ to `(0.0, 1.0]`) scales the approximate char-count input budget. The+ `max_input_tokens` budget uses a flat chars-per-token heuristic that+ under-budgets denser corpora — pt-BR text and source code tokenize at fewer+ chars per token than English and could overshoot a provider's real input+ limit by ~40%. The default tightens the common case modestly while leaving+ such corpora headroom; lower it further for a mostly non-English or code+ corpus. `max_input_tokens` is now documented as an approximate heuristic in+ the config reference. (#884)- `install-skills --agent hermes` (and the `hermes` entry in`memory_install_self_routing`'s `target_hints`) installs the managed routingskills into Hermes Agent's `.hermes/skills` (project) or `~/.hermes/skills`diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex 3e8d47be..f219c45f 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -2599,6 +2599,7 @@ fn configure_consolidator(.with_prompt_limits(config.consolidation.max_input_tokens,config.consolidation.max_output_tokens,+ config.consolidation.input_token_safety_margin,),);server = server.with_consolidator_arc(wiki.clone(), llm.clone(), consolidator.clone());diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rsindex b4025bec..fe70c356 100644--- a/crates/ai-memory-cli/src/config.rs+++ b/crates/ai-memory-cli/src/config.rs@@ -996,6 +996,15 @@ pub struct ConsolidationSettings {/// Maximum tokens the provider may generate for a consolidation response./// Small-context models must lower this together with `max_input_tokens`.pub max_output_tokens: u32,+ /// Safety margin applied to the `max_input_tokens` budget (`0 < m <= 1`).+ ///+ /// `max_input_tokens` is an approximate char-count heuristic (a flat+ /// chars-per-token ratio), so it under-budgets denser corpora — pt-BR text+ /// and source code tokenize at fewer chars per token than English prose and+ /// can overshoot a provider's real input limit by ~40%. This margin shrinks+ /// the effective char budget (default 0.8); lower it further for a corpus+ /// that is mostly non-English or code. (#884)+ pub input_token_safety_margin: f64,}impl Default for ConsolidationSettings {@@ -1003,6 +1012,8 @@ impl Default for ConsolidationSettings {Self {max_input_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,max_output_tokens: ai_memory_consolidate::DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,+ input_token_safety_margin:+ ai_memory_consolidate::DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN,}}}@@ -1750,6 +1761,21 @@ impl Config {config.consolidation.max_output_tokens);}+ // The safety margin scales the input budget, so a non-positive value+ // would starve every prompt and one above 1.0 would loosen the budget+ // past the nominal token limit it is meant to tighten (#884). NaN also+ // fails every comparison below, so it is rejected here too.+ let safety_margin = config.consolidation.input_token_safety_margin;+ if !(safety_margin > 0.0 && safety_margin <= 1.0) {+ anyhow::bail!(+ "consolidation.input_token_safety_margin must be in (0.0, 1.0] \+ (got {safety_margin}); it scales the approximate input-token budget"+ );+ }+ // The safety margin scales the input budget, so a non-positive value+ // would starve every prompt and one above 1.0 would loosen the budget+ // past the nominal token limit it is meant to tighten (#884). NaN also+ // fails every comparison below, so it is rejected here too.// Zero (or a sub-second remainder rounded down) would cut every// provider request off before it is sent.if config.llm_timeout_secs == 0 {@@ -3345,6 +3371,49 @@ mod tests {}}+ /// #884: the input-token safety margin must stay in `(0.0, 1.0]` — a+ /// non-positive value starves every prompt and a value above 1.0 loosens+ /// the budget past the limit it exists to tighten.+ #[test]+ fn load_rejects_an_out_of_range_input_token_safety_margin() {+ for value in ["0.0", "-0.1", "1.5", "nan"] {+ let tmp = TempDir::new().unwrap();+ let config_path = tmp.path().join("config.toml");+ std::fs::write(+ &config_path,+ format!("[consolidation]\ninput_token_safety_margin = {value}\n"),+ )+ .unwrap();+ let error = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))+ .expect_err("an out-of-range safety margin must fail closed");+ assert!(+ error+ .to_string()+ .contains("consolidation.input_token_safety_margin"),+ "unexpected error for {value}: {error:#}"+ );+ }+ }++ /// A valid margin survives the config round-trip and the default is 0.8.+ #[test]+ fn load_accepts_a_valid_input_token_safety_margin() {+ let tmp = TempDir::new().unwrap();+ let config_path = tmp.path().join("config.toml");+ std::fs::write(+ &config_path,+ "[consolidation]\ninput_token_safety_margin = 0.6\n",+ )+ .unwrap();+ let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))+ .expect("a margin inside (0.0, 1.0] must load");+ assert_eq!(config.consolidation.input_token_safety_margin, 0.6);+ assert_eq!(+ ConsolidationSettings::default().input_token_safety_margin,+ 0.8+ );+ }+/// A small-context provider needs both sides of the context allocation to/// survive the config round-trip.#[test]diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rsindex d5b8b58a..116120f6 100644--- a/crates/ai-memory-consolidate/src/consolidator.rs+++ b/crates/ai-memory-consolidate/src/consolidator.rs@@ -155,9 +155,23 @@ impl Consolidator {////// `max_input_tokens + max_output_tokens` must fit the provider's context/// window. Callers validate the supported minimums when resolving config.+ ///+ /// `safety_margin` shrinks the char-count input budget so the flat+ /// chars-per-token heuristic does not over-admit on denser-than-English+ /// corpora (pt-BR, code); it is validated to `0 < margin <= 1` at config+ /// load. See [`DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN`] (#884).#[must_use]- pub fn with_prompt_limits(mut self, max_input_tokens: usize, max_output_tokens: u32) -> Self {- self.budgets = PromptBudgets::from_limits(max_input_tokens, max_output_tokens);+ pub fn with_prompt_limits(+ mut self,+ max_input_tokens: usize,+ max_output_tokens: u32,+ safety_margin: f64,+ ) -> Self {+ self.budgets = PromptBudgets::from_limits_with_margin(+ max_input_tokens,+ max_output_tokens,+ safety_margin,+ );self}@@ -1276,6 +1290,17 @@ pub const DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS: usize = 100_000;/// Default maximum generated tokens for a consolidation response.pub const DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS: u32 = 32_000;+/// Default multiplier applied to the char-count input budget (#884).+///+/// `max_input_tokens` is turned into a char budget with a flat+/// [`CHARS_PER_TOKEN`] heuristic. That ratio holds for English prose but+/// over-admits on denser corpora — pt-BR prose and source code tokenize at+/// closer to ~2.1 chars/token, so a 3:1 estimate overshot the real token+/// count by ~40% and tripped provider `max_input_tokens` limits. Shrinking+/// the effective char budget to 80% of the nominal value buys that headroom+/// back for the common case while leaving English budgets close to before.+pub const DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN: f64 = 0.8;+/// Conservative character-to-token estimate for provider-neutral budgeting./// The exact tokenizer is provider/model-specific, so this is a target rather/// than a hard token count. Three characters per token plus the default@@ -1329,8 +1354,26 @@ struct PromptBudgets {impl PromptBudgets {fn from_limits(max_input_tokens: usize, max_output_tokens: u32) -> Self {+ // A bare limit applies no safety margin (margin 1.0), preserving the+ // historical char budget; production tightens it via config (#884).+ Self::from_limits_with_margin(max_input_tokens, max_output_tokens, 1.0)+ }++ /// Derive budgets from the token limits, shrinking the char-count input+ /// budget by `safety_margin` (#884). The margin compensates for the flat+ /// [`CHARS_PER_TOKEN`] heuristic over-admitting on denser-than-English+ /// corpora (pt-BR, code). Callers pass a validated `0 < margin <= 1`.+ fn from_limits_with_margin(+ max_input_tokens: usize,+ max_output_tokens: u32,+ safety_margin: f64,+ ) -> Self {+ let nominal = max_input_tokens.saturating_mul(CHARS_PER_TOKEN);+ // `safety_margin` is validated to `0 < margin <= 1` at config load, so+ // the product never exceeds `nominal` and the cast cannot overflow.+ let max_input_chars = (nominal as f64 * safety_margin) as usize;Self {- max_input_chars: max_input_tokens.saturating_mul(CHARS_PER_TOKEN),+ max_input_chars,max_output_tokens,}}@@ -2345,6 +2388,54 @@ mod tests {);}+ /// #884: the safety margin scales the char budget by exactly the factor,+ /// and margin 1.0 reproduces the un-margined budget.+ #[test]+ fn safety_margin_scales_input_char_budget() {+ let full = PromptBudgets::from_limits(10_000, 1_000);+ assert_eq!(full.max_input_chars, 30_000);+ assert_eq!(+ PromptBudgets::from_limits_with_margin(10_000, 1_000, 1.0),+ full,+ "margin 1.0 must leave the budget unchanged"+ );+ let tightened = PromptBudgets::from_limits_with_margin(10_000, 1_000, 0.8);+ assert_eq!(+ tightened.max_input_chars, 24_000,+ "margin 0.8 must shrink the char budget to 0.8x"+ );+ // Output allowance is independent of the input margin.+ assert_eq!(tightened.max_output_tokens, 1_000);+ }++ /// #884: a large observation set is packed to the tightened budget, so a+ /// margin actually reduces how much prompt content is admitted.+ #[test]+ fn safety_margin_trims_packed_observations() {+ let tightened = PromptBudgets::from_limits_with_margin(+ DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,+ DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,+ 0.8,+ );+ let full = PromptBudgets::from_limits(+ DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,+ DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,+ );+ assert!(tightened.max_input_chars < full.max_input_chars);+ let observations = (0..256).map(|_| obs_of_size(4_000)).collect::<Vec<_>>();+ let request = build_request(+ SessionId::new(),+ &observations,+ &"x".repeat(50_000),+ Some(&"preference ".repeat(500)),+ tightened,+ );+ assert!(+ estimated_input_chars::<ConsolidatedPage>(&request) <= tightened.max_input_chars,+ "packed request must respect the tightened budget"+ );+ }+/// The batch path must include schema and dynamic slot snapshots in its/// envelope instead of assuming a fixed number of slots.#[test]diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rsindex 5ed15462..1156c977 100644--- a/crates/ai-memory-consolidate/src/lib.rs+++ b/crates/ai-memory-consolidate/src/lib.rs@@ -62,8 +62,9 @@ pub use cold_cluster::{adaptive_eps, cosine_distance, dbscan};pub use compaction::build_compacted_markdown;pub use consolidator::{BATCH_SYSTEM_PROMPT, Consolidator, ConsolidatorError, ConsolidatorResult,- DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS, DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS,- MIN_CONSOLIDATION_MAX_INPUT_TOKENS, MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,+ DEFAULT_CONSOLIDATION_INPUT_TOKEN_SAFETY_MARGIN, DEFAULT_CONSOLIDATION_MAX_INPUT_TOKENS,+ DEFAULT_CONSOLIDATION_MAX_OUTPUT_TOKENS, MIN_CONSOLIDATION_MAX_INPUT_TOKENS,+ MIN_CONSOLIDATION_MAX_OUTPUT_TOKENS, build_batch_request,};pub use curator::{CuratorFinding, CuratorParams, CuratorReport, render_curator_report_markdown,diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 3f256f76..73905768 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -723,9 +723,17 @@ per_user = false # shared + own slots in agent context[consolidation] # LLM consolidation prompt sizingmax_input_tokens = 100000 # approximate whole-input target; min 6000+ # a flat chars-per-token heuristic, so it+ # UNDER-budgets denser corpora: pt-BR prose+ # and source code tokenize at fewer chars per+ # token than English and can overshoot the+ # provider's real limit by ~40% — lower this+ # (or input_token_safety_margin) for such a corpusmax_output_tokens = 32000 # provider generation limit; min 1000# their sum must fit the model context window;# leave headroom for tokenizer variance+input_token_safety_margin = 0.8 # scales the char budget, (0.0, 1.0]; the+ # 0.8 default buys pt-BR/code headroom[auto_improve] # default-available learning reviewerrequire_approval = false # true leaves proposals pending for review