Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(hooks): sanitize observation titles before truncating them
Maxsuel Einstein55952bc46d99
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c376849a..99632f3a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -36,6 +36,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0(`docs/lifecycle-ops.md#backup`) is unchanged. (#950)### Fixed+- `observations.title` is now sanitized before it is truncated, not after.+ `title_hint` used to be cut to 80 chars in `ai-memory-hooks::payload`+ *before* the sanitizer ever ran, so a secret straddling that cutoff was+ often left as a fragment too short to match a built-in or `[sanitize]+ extra_patterns` rule — landing in the title, its FTS index, and every+ surface that renders titles (session pages, briefings, handoffs, search)+ unredacted, even though the same observation's body was correctly scrubbed+ first. `title_hint` extraction now keeps the full first line untruncated;+ `Sanitized::new` scrubs the title and only then applies the 80-char display+ cap (`ai_memory_core::sanitize::truncate_for_title`), mirroring the order+ the body already used. (#982)- The `/web` page view keeps a leading H1 that is not the page title. Itdropped the body's first H1 whatever it said, as a duplicate of the titlein the header, but a frontmatter `title:` outranks the H1 and a setext H1diff --git a/crates/ai-memory-core/src/lib.rs b/crates/ai-memory-core/src/lib.rsindex 5293a1d6..f7dcfabc 100644--- a/crates/ai-memory-core/src/lib.rs+++ b/crates/ai-memory-core/src/lib.rs@@ -75,8 +75,8 @@ pub use routing_snippet::{full_block,};pub use sanitize::{- OBSERVATION_BODY_MAX_BYTES, SanitizeConfig, Sanitized, Sanitizer, truncate_utf8_bytes,- truncate_utf8_bytes_head_tail,+ OBSERVATION_BODY_MAX_BYTES, SanitizeConfig, Sanitized, Sanitizer, truncate_for_title,+ truncate_utf8_bytes, truncate_utf8_bytes_head_tail,};pub use slots::{SLOT_PREFIX, SlotPlacement, SlotVisibility, is_slot_named, is_slot_path, slot_owner,diff --git a/crates/ai-memory-core/src/sanitize.rs b/crates/ai-memory-core/src/sanitize.rsindex 688a01c4..d2a5d179 100644--- a/crates/ai-memory-core/src/sanitize.rs+++ b/crates/ai-memory-core/src/sanitize.rs@@ -343,16 +343,54 @@ impl<T> Sanitized<T> {impl Sanitized<NewObservation> {/// Apply the privacy strip to an observation's title + body, then enforce- /// the universal durable-body ceiling after redaction.+ /// the universal durable-body ceiling and the title display cap after+ /// redaction.+ ///+ /// Scrub-then-truncate, for both fields, in that order. Truncating first+ /// (as `ai-memory-hooks::payload` used to do for `title_hint`, before+ /// #980) can cut a secret in half before the sanitizer ever sees it: the+ /// stored prefix is often too short to match a pattern, so the fragment+ /// lands in the title unredacted. Scrubbing first also means a+ /// `[REDACTED:…]` marker is already in place before the length cap runs,+ /// same as the body.#[must_use]pub fn new(mut obs: NewObservation, sanitizer: &Sanitizer) -> Self {- obs.title = sanitizer.scrub(&obs.title);+ obs.title = truncate_for_title(&sanitizer.scrub(&obs.title));obs.body =truncate_utf8_bytes_head_tail(&sanitizer.scrub(&obs.body), OBSERVATION_BODY_MAX_BYTES);Self(obs)}}+/// Cap a (single-line) title to at most 80 displayed characters, appending an+/// ellipsis when the input is longer. Applied by [`Sanitized::new`] to+/// `NewObservation::title` — after redaction, not before, so a secret that+/// straddles the cutoff is never stored half-redacted (see #980). Callers+/// that build a title hint upstream of sanitization (`ai-memory-hooks`) must+/// reduce it to a single line themselves; this function does not strip+/// newlines, mirroring [`truncate_utf8_bytes`], which does not either.+///+/// Known cosmetic gap: running after redaction fixes the security bug+/// (the secret itself is already gone by the time this runs), but a plain+/// char-count cutoff can still land inside a `[REDACTED:…]` marker the+/// scrubber just inserted, truncating it to something like `[REDACTE…`. A+/// title long enough to need this cap, ending in a marker, reproduces it+/// (see the `issue_980_user_prompt_title_is_sanitized_before_truncated` test+/// in `ai-memory-hooks::router`, which observes exactly this). Left+/// unhandled deliberately: no secret material leaks, and centering the cap+/// on marker boundaries would add real complexity for a display nicety.+#[must_use]+pub fn truncate_for_title(s: &str) -> String {+ const MAX: usize = 80;+ if s.chars().count() <= MAX {+ s.to_string()+ } else {+ let mut buf: String = s.chars().take(MAX - 1).collect();+ buf.push('…');+ buf+ }+}+/// Truncate text to at most `max` UTF-8 bytes, reserving the ellipsis inside/// the requested cap and never splitting a code point.#[must_use]@@ -851,6 +889,47 @@ mod tests {assert!(scrubbed.body.contains("[truncated"));}+ /// Regression for #980: a secret matched only by an operator's+ /// `[sanitize] extra_patterns` rule must be redacted in the title even+ /// when the raw title is long enough to need the 80-char cap — proving+ /// `Sanitized::new` scrubs `title` before truncating it, the same order+ /// the body already used. See `issue_980_user_prompt_title_is_sanitized_+ /// before_truncated` in `ai-memory-hooks::router` for the full pipeline+ /// (payload extraction -> router -> here) this guards end to end.+ #[test]+ fn title_is_scrubbed_before_the_80_char_cap_runs() {+ let sanitizer = Sanitizer::new(&SanitizeConfig {+ extra_patterns: vec![r"CANARY-[0-9]{20,}".to_string()],+ allowlist: Vec::new(),+ })+ .unwrap();+ let secret = format!("CANARY-{}", "9".repeat(25));+ let raw = NewObservation {+ occurred_at: None,+ session_id: SessionId::new(),+ workspace_id: WorkspaceId::new(),+ project_id: ProjectId::new(),+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: format!("{} {secret}", "x".repeat(70)),+ body: String::new(),+ importance: 5,+ };+ let scrubbed = Sanitized::new(raw, &sanitizer).into_inner();+ assert!(+ !scrubbed.title.contains(&secret),+ "raw secret survived scrub+cap: {:?}",+ scrubbed.title+ );+ assert!(+ scrubbed.title.contains("REDACT"),+ "title lost all trace of redaction: {:?}",+ scrubbed.title+ );+ assert!(scrubbed.title.chars().count() <= 80);+ }+#[test]fn utf8_truncation_reserves_the_ellipsis_inside_the_cap() {let truncated = truncate_utf8_bytes("abcééé", 7);@@ -860,6 +939,46 @@ mod tests {assert!(truncate_utf8_bytes("large", 2).is_empty());}+ /// Moved from `ai-memory-hooks::payload` (see #980): the char-count cap+ /// and ellipsis behaviour are unchanged from the original+ /// `payload::truncate_for_title`, only *where* and *when* it runs moved.+ #[test]+ fn title_truncation_caps_at_80_chars_with_ellipsis() {+ let short = "a short title";+ assert_eq!(truncate_for_title(short), short);++ let exactly_80 = "x".repeat(80);+ assert_eq!(truncate_for_title(&exactly_80), exactly_80);++ let long = "x".repeat(200);+ let truncated = truncate_for_title(&long);+ assert_eq!(truncated.chars().count(), 80);+ assert!(truncated.ends_with('…'));+ assert_eq!(&truncated[..79], &"x".repeat(79));+ }++ /// Char-count truncation must never split a multi-byte code point, even+ /// though it counts *characters*, not bytes, unlike [`truncate_utf8_bytes`].+ #[test]+ fn title_truncation_is_utf8_safe_on_multibyte_input() {+ // 100 é's (each 2 bytes in UTF-8): a byte-oriented cap at 80 would+ // risk landing mid-codepoint; a char-oriented cap never can.+ let input = "é".repeat(100);+ let truncated = truncate_for_title(&input);+ assert_eq!(truncated.chars().count(), 80);+ assert!(truncated.ends_with('…'));+ // Every remaining char before the ellipsis is a complete `é`.+ assert!(+ truncated[..truncated.len() - '…'.len_utf8()]+ .chars()+ .all(|c| c == 'é')+ );+ // The string is valid UTF-8 by construction (it's a `String`), but+ // assert the byte length is exactly what 79 `é`s + one `…` costs,+ // as a belt-and-suspenders check that nothing was sliced mid-byte.+ assert_eq!(truncated.len(), 79 * 'é'.len_utf8() + '…'.len_utf8());+ }+#[test]fn head_tail_truncation_preserves_head_and_tail() {let input = format!("HEAD{}TAIL", "x".repeat(200));diff --git a/crates/ai-memory-hooks/src/payload.rs b/crates/ai-memory-hooks/src/payload.rsindex e5a700e0..160f39c2 100644--- a/crates/ai-memory-hooks/src/payload.rs+++ b/crates/ai-memory-hooks/src/payload.rs@@ -916,7 +916,7 @@ fn best_title_hint(event: HookEvent, raw: &serde_json::Value) -> Option<String>// Kimi Code sends `prompt` as content blocks// (`[{"type":"text","text":...}]`); `extract_content` flattens// them and returns identical values for plain-string agents.- extract_content(raw, &["prompt", "message", "text"]).map(|s| truncate_for_title(&s))+ extract_content(raw, &["prompt", "message", "text"]).map(|s| first_line(&s))}HookEvent::PreToolUse | HookEvent::PostToolUse => {extract_string(raw, &["tool", "tool_name", "name"])@@ -933,7 +933,7 @@ fn best_title_hint(event: HookEvent, raw: &serde_json::Value) -> Option<String>fn extension_title_hint(raw: &serde_json::Value, source_event: &str) -> String {extract_string(raw, &["title", "summary", "subject", "name"])- .map(|s| truncate_for_title(&s))+ .map(|s| first_line(&s)).unwrap_or_else(|| source_event.to_string())}@@ -1026,16 +1026,18 @@ fn best_body_excerpt(event: HookEvent, raw: &serde_json::Value) -> Option<String}}-pub(crate) fn truncate_for_title(s: &str) -> String {- const MAX: usize = 80;- let one_line: String = s.chars().take_while(|c| *c != '\n').collect();- if one_line.chars().count() <= MAX {- one_line- } else {- let mut buf: String = one_line.chars().take(MAX - 1).collect();- buf.push('…');- buf- }+/// Reduce a hook-supplied string to its first line, discarding everything+/// after the first `\n`.+///+/// This is the only shaping `title_hint` gets before it reaches the+/// sanitizer: the 80-char display cap used to happen here too, but that ran+/// *before* redaction and could cut a secret in half — the truncated prefix+/// was then too short to match the sanitizer's pattern, so it was stored in+/// clear text (#980). The length cap now lives in+/// `ai_memory_core::sanitize::truncate_for_title`, applied by+/// `Sanitized::new` *after* `Sanitizer::scrub`.+fn first_line(s: &str) -> String {+ s.chars().take_while(|c| *c != '\n').collect()}fn truncate_excerpt(s: &str) -> String {@@ -2125,10 +2127,13 @@ mod tests {/// Title-hint extraction must truncate at the first newline (the/// "first line" rule used everywhere in the wiki log + handoff- /// surfaces) and cap at 80 chars to keep observation titles- /// scannable in the log.md heading.+ /// surfaces), but must NOT cap length here any more: the 80-char display+ /// cap moved to `ai_memory_core::sanitize::truncate_for_title`, applied+ /// by `Sanitized::new` after redaction (#980) — truncating a long+ /// `title_hint` in `payload.rs`, before the sanitizer ever runs, could+ /// cut a secret in half and leave the unmatched fragment unredacted.#[test]- fn user_prompt_title_truncates_at_newline_and_at_max_chars() {+ fn user_prompt_title_keeps_first_line_full_length_untruncated() {let q = HookQuery {event: "user-prompt".into(),agent: Some("claude-code".into()),@@ -2141,12 +2146,13 @@ mod tests {);assert_eq!(env.title_hint.as_deref(), Some("first line"));- // Very long single line → truncated with ellipsis.+ // Very long single line → title_hint keeps every character; the+ // 80-char cap now happens later, after sanitization.let long = "x".repeat(200);let env = HookEnvelope::from_query_and_body(q, serde_json::json!({ "prompt": long }));let title = env.title_hint.unwrap();- assert!(title.chars().count() <= 80);- assert!(title.ends_with('…'));+ assert_eq!(title.chars().count(), 200);+ assert!(!title.contains('…'));}#[test]diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rsindex f1ab52f5..02ecad19 100644--- a/crates/ai-memory-hooks/src/router.rs+++ b/crates/ai-memory-hooks/src/router.rs@@ -12161,6 +12161,70 @@ mod tests {}}+ /// Regression for #980: a secret straddling the old 80-char `title_hint`+ /// cutoff survived in `observations.title` unredacted, because+ /// `payload::truncate_for_title` used to cut the prompt to 80 chars+ /// *before* the sanitizer ever ran — often leaving too short a fragment+ /// to match a pattern (built-in or `[sanitize] extra_patterns`). The fix+ /// removed truncation from `title_hint` construction entirely; the+ /// 80-char cap now runs in `Sanitized::new`, after `Sanitizer::scrub`.+ ///+ /// Before the fix: the stored title contains a raw, unmatched fragment of+ /// the secret and no `[REDACTED:…]` marker at all, because the sanitizer+ /// only ever saw the truncated 8-character prefix `SECRETZZ` — too short+ /// for the `{20,}`-length pattern below. After the fix: the full secret+ /// reaches the sanitizer first, gets replaced with the marker, and the+ /// (much shorter) redacted title is what gets capped at 80 chars.+ #[tokio::test]+ async fn issue_980_user_prompt_title_is_sanitized_before_truncated() {+ let tmp = TempDir::new().unwrap();+ let mut state = make_state(&tmp).await;+ state.sanitizer = Sanitizer::new(&SanitizeConfig {+ extra_patterns: vec![r"SECRET[0-9A-Za-z]{20,}".into()],+ allowlist: Vec::new(),+ })+ .unwrap();+ let secret = format!("SECRET{}", "Z".repeat(30));+ let prompt = format!("{} {secret}", "x".repeat(70));+ let sid = SessionId::new();+ let env = HookEnvelope::from_query_and_body(+ HookQuery {+ event: "user-prompt".into(),+ agent: Some("claude-code".into()),+ ..Default::default()+ },+ serde_json::json!({+ "session_id": sid.to_string(),+ "cwd": "/repo",+ "prompt": prompt,+ }),+ );+ process(&state, env, None, Vec::new()).await.unwrap();++ let observations = state.reader.observations_for_session(sid).await.unwrap();+ let obs = observations+ .iter()+ .find(|o| o.kind == ObservationKind::UserPrompt)+ .expect("user prompt observation was recorded");++ assert!(+ !obs.title.contains(&secret),+ "full raw secret leaked into title: {:?}",+ obs.title+ );+ assert!(+ obs.title.contains("REDACT"),+ "title carries no trace of redaction \u{2014} the sanitizer never \+ saw enough of the secret to match it: {:?}",+ obs.title+ );+ assert!(+ obs.title.chars().count() <= 80,+ "title exceeded the 80-char display cap: {:?}",+ obs.title+ );+ }+#[test]fn codex_native_patch_capture_backstop_discards_unproven_output() {for protocol in [diff --git a/crates/ai-memory-hooks/src/synth.rs b/crates/ai-memory-hooks/src/synth.rsindex fc9eb2f3..1e85c516 100644--- a/crates/ai-memory-hooks/src/synth.rs+++ b/crates/ai-memory-hooks/src/synth.rs@@ -10,11 +10,11 @@ use std::collections::BTreeMap;use ai_memory_core::{AgentKind, NewPage, Observation, ObservationKind, PagePath, ProjectId, SessionId, Tier,- WorkspaceId, looks_like_scaffolding,+ WorkspaceId, looks_like_scaffolding, truncate_for_title,};use jiff::tz::TimeZone;-use crate::payload::{is_safe_tool_title, truncate_for_title};+use crate::payload::is_safe_tool_title;const RAW_OBSERVATION_MAX_LINES: usize = 500;const RAW_OBSERVATION_HEAD_LINES: usize = 250;