Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(store): take Sanitized<NewObservation> at the writer boundary
Samir Hanna Verza2a4638749982
diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rsindex 8675427b..2f2f02d3 100644--- a/crates/ai-memory-consolidate/src/auto_improve.rs+++ b/crates/ai-memory-consolidate/src/auto_improve.rs@@ -1808,7 +1808,9 @@ Every proposal must include bounded evidence quotes, confidence, rationale, and#[cfg(test)]mod tests {use super::*;- use ai_memory_core::{AgentKind, NewObservation, NewSession, ObservationId, ObservationKind};+ use ai_memory_core::{+ AgentKind, NewObservation, NewSession, ObservationId, ObservationKind, Sanitized, Sanitizer,+ };use ai_memory_llm::{ChatResponse, LlmResult};use ai_memory_store::Store;use jiff::Timestamp;@@ -2207,21 +2209,24 @@ mod tests {for i in 0..3 {store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: if i == 0 {- ObservationKind::SessionStart- } else {- ObservationKind::UserPrompt+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: if i == 0 {+ ObservationKind::SessionStart+ } else {+ ObservationKind::UserPrompt+ },+ extension: None,+ source_event: None,+ title: format!("event {i}"),+ body: "run the full gate before release".into(),+ importance: 5,},- extension: None,- source_event: None,- title: format!("event {i}"),- body: "run the full gate before release".into(),- importance: 5,- })+ &Sanitizer::builtin(),+ )).await.unwrap();}diff --git a/crates/ai-memory-consolidate/tests/recall_eval.rs b/crates/ai-memory-consolidate/tests/recall_eval.rsindex 55e0b993..15c3002b 100644--- a/crates/ai-memory-consolidate/tests/recall_eval.rs+++ b/crates/ai-memory-consolidate/tests/recall_eval.rs@@ -17,7 +17,8 @@use std::sync::Arc;use ai_memory_core::{- AgentKind, NewObservation, NewSession, ObservationKind, PagePath, SessionId, Tier,+ AgentKind, NewObservation, NewSession, ObservationKind, PagePath, Sanitized, Sanitizer,+ SessionId, Tier,};use ai_memory_llm::{Embedder, SyntheticEmbedder};use ai_memory_store::Store;@@ -277,17 +278,20 @@ async fn raw_observation_fallback_recovers_detail_when_wiki_misses() {.expect("begin session");store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "raw prompt".into(),- body: "raw fallback detail mentions capybara exactly once".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "raw prompt".into(),+ body: "raw fallback detail mentions capybara exactly once".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.expect("insert observation");diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rsindex 7f411999..dd661bbe 100644--- a/crates/ai-memory-hooks/src/router.rs+++ b/crates/ai-memory-hooks/src/router.rs@@ -1987,11 +1987,12 @@ async fn process(body,importance: importance_for(env.event),};+ // The store boundary takes `Sanitized<NewObservation>` directly — no+ // unwrap-and-clone; the type proves the scrub happened. The log line+ // below wants the scrubbed title, so keep a copy before the move.let sanitized = Sanitized::new(raw_obs, &state.sanitizer);- let _ = state- .writer- .insert_observation(sanitized.inner().clone())- .await?;+ let log_title = sanitized.inner().title.clone();+ let _ = state.writer.insert_observation(sanitized).await?;// Append the log line to the per-project log.md.if let Err(e) = log::append_event(@@ -2000,7 +2001,7 @@ async fn process(proj,Timestamp::now(),env.event,- sanitized.inner().title.as_str(),+ log_title.as_str(),) {warn!(error = %e, "log.md append failed");}diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex 9402ba6a..9256d87f 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -4972,6 +4972,7 @@ fn map_user_store_err(e: ai_memory_store::StoreError) -> (StatusCode, Json<serdemod tests {use super::*;use ai_memory_core::{AgentKind, NewObservation, NewSession, ObservationKind};+ use ai_memory_core::{Sanitized, Sanitizer};use ai_memory_llm::{ChatRequest, ChatResponse, LlmResult};use ai_memory_store::Store;use axum::body::to_bytes;@@ -5409,17 +5410,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "prompt".into(),- body: "focus changed and durable lesson".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "prompt".into(),+ body: "focus changed and durable lesson".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();@@ -5536,17 +5540,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "prompt".into(),- body: "focus changed and durable lesson".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "prompt".into(),+ body: "focus changed and durable lesson".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();let mut state =@@ -5631,17 +5638,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "prompt".into(),- body: "durable lesson".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "prompt".into(),+ body: "durable lesson".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();let mut state =diff --git a/crates/ai-memory-mcp/src/server.rs b/crates/ai-memory-mcp/src/server.rsindex 94eaf677..2f46e56b 100644--- a/crates/ai-memory-mcp/src/server.rs+++ b/crates/ai-memory-mcp/src/server.rs@@ -2799,6 +2799,7 @@ fn test_optional_parts() -> OptionalParts {#[cfg(test)]mod tests {use super::*;+ use ai_memory_core::{Sanitized, Sanitizer};use std::collections::BTreeSet;#[test]@@ -2973,17 +2974,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id,- project_id,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: title.into(),- body: body.into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id,+ project_id,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: title.into(),+ body: body.into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();}@@ -4075,17 +4079,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "raw prompt".into(),- body: "raw fallback contains quokka only detail".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "raw prompt".into(),+ body: "raw fallback contains quokka only detail".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();@@ -4144,17 +4151,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: scoped,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "raw prompt".into(),- body: "raw fallback contains quokka only detail".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: scoped,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "raw prompt".into(),+ body: "raw fallback contains quokka only detail".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();diff --git a/crates/ai-memory-mcp/tests/admin_move.rs b/crates/ai-memory-mcp/tests/admin_move.rsindex 41fa0c4a..52cc50ab 100644--- a/crates/ai-memory-mcp/tests/admin_move.rs+++ b/crates/ai-memory-mcp/tests/admin_move.rs@@ -13,7 +13,7 @@//! both versions survive), then purge the source — there the episodic rows//! (sessions/observations/handoffs) are dropped by the purge.-use ai_memory_core::{PagePath, Tier};+use ai_memory_core::{PagePath, Sanitized, Sanitizer, Tier};use ai_memory_mcp::{AdminState, admin_router};use ai_memory_store::{DecayParams, Store};use ai_memory_wiki::{@@ -573,17 +573,20 @@ async fn move_project_true_move_preserves_sessions_and_observations() {.unwrap();store.writer- .insert_observation(NewObservation {- session_id: sid,- workspace_id: src_ws,- project_id: src_proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "prompt".into(),- body: "do the thing".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id: sid,+ workspace_id: src_ws,+ project_id: src_proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "prompt".into(),+ body: "do the thing".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();diff --git a/crates/ai-memory-mcp/tests/admin_phase3.rs b/crates/ai-memory-mcp/tests/admin_phase3.rsindex c6d150e3..ba534cf4 100644--- a/crates/ai-memory-mcp/tests/admin_phase3.rs+++ b/crates/ai-memory-mcp/tests/admin_phase3.rs@@ -9,7 +9,7 @@use ai_memory_core::{ActorContext, AgentKind, NewObservation, NewPage, NewSession, ObservationKind, PagePath,- SessionId, Tier,+ Sanitized, Sanitizer, SessionId, Tier,};use ai_memory_llm::SyntheticEmbedder;use ai_memory_mcp::{AdminState, admin_router};@@ -346,17 +346,20 @@ async fn seed_sessions_for_reorg(store: &Store) -> (SessionId, SessionId) {.unwrap();store.writer- .insert_observation(NewObservation {- session_id: sid_a,- workspace_id: ws,- project_id: scratch,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "alpha prompt".into(),- body: "".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id: sid_a,+ workspace_id: ws,+ project_id: scratch,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "alpha prompt".into(),+ body: "".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();@@ -374,17 +377,20 @@ async fn seed_sessions_for_reorg(store: &Store) -> (SessionId, SessionId) {.unwrap();store.writer- .insert_observation(NewObservation {- session_id: sid_b,- workspace_id: ws,- project_id: scratch,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "beta prompt".into(),- body: "".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id: sid_b,+ workspace_id: ws,+ project_id: scratch,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "beta prompt".into(),+ body: "".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();diff --git a/crates/ai-memory-mcp/tests/admin_purge.rs b/crates/ai-memory-mcp/tests/admin_purge.rsindex 3ef2a230..598d401c 100644--- a/crates/ai-memory-mcp/tests/admin_purge.rs+++ b/crates/ai-memory-mcp/tests/admin_purge.rs@@ -6,7 +6,7 @@use ai_memory_core::{AgentKind, NewHandoff, NewObservation, NewSession, ObservationKind, PagePath, ProjectId,- SessionId, Tier, WorkspaceId,+ Sanitized, Sanitizer, SessionId, Tier, WorkspaceId,};use ai_memory_mcp::{AdminState, admin_router};use ai_memory_store::{DecayParams, Store};@@ -139,17 +139,20 @@ async fn seed_two_projects(store: &Store, wiki: &Wiki) -> (WorkspaceId, ProjectIfor i in 0..3u8 {store.writer- .insert_observation(NewObservation {- session_id: sid,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: format!("{label} obs {i}"),- body: "body".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id: sid,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: format!("{label} obs {i}"),+ body: "body".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();}diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 46713d87..1f277cb2 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -120,8 +120,8 @@ mod tests {use super::*;use ai_memory_core::{ActorContext, AgentKind, LinkTarget, NewObservation, NewPage, NewSession,- NewWorkstreamEvent, ObservationId, ObservationKind, PageId, PagePath, ProjectId, SessionId,- Tier, UserId, WorkspaceId, WorkstreamEventKind,+ NewWorkstreamEvent, ObservationId, ObservationKind, PageId, PagePath, ProjectId, Sanitized,+ Sanitizer, SessionId, Tier, UserId, WorkspaceId, WorkstreamEventKind,};use rusqlite::{Connection, params};use sha2::{Digest, Sha256};@@ -2319,17 +2319,20 @@ mod tests {.unwrap();store.writer- .insert_observation(NewObservation {- session_id,- workspace_id: ws,- project_id: proj,- kind: ObservationKind::UserPrompt,- extension: None,- source_event: None,- title: "prompt".into(),- body: "the raw-only zebra detail lives here".into(),- importance: 5,- })+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "prompt".into(),+ body: "the raw-only zebra detail lives here".into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ )).await.unwrap();@@ -2343,6 +2346,71 @@ mod tests {assert!(hits[0].snippet.contains("<mark>zebra</mark>"));}+ /// The public writer API only accepts `Sanitized<NewObservation>`, and+ /// `Sanitized::new` is the single constructor — so a secret passed in an+ /// observation body cannot reach disk unscrubbed. This locks the typed+ /// boundary end-to-end: through the writer, into the SQLite `body` column.+ #[tokio::test]+ async fn insert_observation_boundary_scrubs_before_disk() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "ai-memory", None)+ .await+ .unwrap();+ let session_id = SessionId::new();+ store+ .writer+ .begin_session(NewSession {+ id: session_id,+ workspace_id: ws,+ project_id: proj,+ agent_kind: AgentKind::ClaudeCode,+ cwd: None,+ })+ .await+ .unwrap();+ store+ .writer+ .insert_observation(Sanitized::new(+ NewObservation {+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "Authorization: Bearer abcdef0123456789ABCDEF0123456789".into(),+ body: "leaked Authorization: Bearer abcdef0123456789ABCDEF0123456789 in transcript"+ .into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ ))+ .await+ .unwrap();++ let conn = Connection::open(store.db_path()).unwrap();+ let (title, body): (String, String) = conn+ .query_row("SELECT title, body FROM observations", [], |r| {+ Ok((r.get(0)?, r.get(1)?))+ })+ .unwrap();+ for col in [&title, &body] {+ assert!(col.contains("[REDACTED]"), "expected scrub in: {col}");+ assert!(+ !col.contains("abcdef0123"),+ "secret reached disk unscrubbed: {col}"+ );+ }+ }+#[tokio::test]async fn latest_completed_session_for_project_ignores_open_sessions() {let tmp = TempDir::new().unwrap();diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex c1693ff6..bc22ac30 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -12,7 +12,7 @@ use std::thread::{self, JoinHandle};use ai_memory_core::{AgentKind, HandoffId, ManagedRunId, NewHandoff, NewObservation, NewPage, NewSession, NewUser,- ObservationId, PageId, PagePath, ProjectId, SessionId, UserId, WorkspaceId,+ ObservationId, PageId, PagePath, ProjectId, Sanitized, SessionId, UserId, WorkspaceId,};use rusqlite::Connection;use tokio::sync::{mpsc, oneshot};@@ -473,9 +473,18 @@ impl WriterHandle {/// Append an observation row.///+ /// Takes [`Sanitized<NewObservation>`] — the privacy strip is enforced by+ /// the type system at the store boundary, so an unsanitized observation+ /// cannot reach disk by construction ([`Sanitized::new`] is the single+ /// constructor and applies the scrub).+ ////// # Errors/// Returns [`StoreError::WriterClosed`] or propagates SQL errors.- pub async fn insert_observation(&self, obs: NewObservation) -> StoreResult<ObservationId> {+ pub async fn insert_observation(+ &self,+ obs: Sanitized<NewObservation>,+ ) -> StoreResult<ObservationId> {+ let obs = obs.into_inner();let (tx, rx) = oneshot::channel();self.send(WriteCmd::InsertObservation { obs, reply: tx }).await?;