Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(store): take Sanitized<NewObservation> at the writer boundary

Samir Hanna Verza2a4638749982
diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rs
index 8675427b..2f2f02d3 100644
--- a/crates/ai-memory-consolidate/src/auto_improve.rs
+++ b/crates/ai-memory-consolidate/src/auto_improve.rs
@@ -1808,7 +1808,9 @@ Every proposal must include bounded evidence quotes, confidence, rationale, and
#[cfg(test)]
mod tests {
use super::*;
- use ai_memory_core::{AgentKind, NewObservation, NewSession, ObservationId, ObservationKind};
+ use ai_memory_core::{
+ AgentKind, NewObservation, NewSession, ObservationId, ObservationKind, Sanitized, Sanitizer,
+ };
use ai_memory_llm::{ChatResponse, LlmResult};
use ai_memory_store::Store;
use jiff::Timestamp;
@@ -2207,21 +2209,24 @@ mod tests {
for i in 0..3 {
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: if i == 0 {
- ObservationKind::SessionStart
- } else {
- ObservationKind::UserPrompt
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: if i == 0 {
+ ObservationKind::SessionStart
+ } else {
+ ObservationKind::UserPrompt
+ },
+ extension: None,
+ source_event: None,
+ title: format!("event {i}"),
+ body: "run the full gate before release".into(),
+ importance: 5,
},
- extension: None,
- source_event: None,
- title: format!("event {i}"),
- body: "run the full gate before release".into(),
- importance: 5,
- })
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
}
diff --git a/crates/ai-memory-consolidate/tests/recall_eval.rs b/crates/ai-memory-consolidate/tests/recall_eval.rs
index 55e0b993..15c3002b 100644
--- a/crates/ai-memory-consolidate/tests/recall_eval.rs
+++ b/crates/ai-memory-consolidate/tests/recall_eval.rs
@@ -17,7 +17,8 @@
use std::sync::Arc;
use ai_memory_core::{
- AgentKind, NewObservation, NewSession, ObservationKind, PagePath, SessionId, Tier,
+ AgentKind, NewObservation, NewSession, ObservationKind, PagePath, Sanitized, Sanitizer,
+ SessionId, Tier,
};
use ai_memory_llm::{Embedder, SyntheticEmbedder};
use ai_memory_store::Store;
@@ -277,17 +278,20 @@ async fn raw_observation_fallback_recovers_detail_when_wiki_misses() {
.expect("begin session");
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "raw prompt".into(),
- body: "raw fallback detail mentions capybara exactly once".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "raw prompt".into(),
+ body: "raw fallback detail mentions capybara exactly once".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.expect("insert observation");
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index 7f411999..dd661bbe 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -1987,11 +1987,12 @@ async fn process(
body,
importance: importance_for(env.event),
};
+ // The store boundary takes `Sanitized<NewObservation>` directly — no
+ // unwrap-and-clone; the type proves the scrub happened. The log line
+ // below wants the scrubbed title, so keep a copy before the move.
let sanitized = Sanitized::new(raw_obs, &state.sanitizer);
- let _ = state
- .writer
- .insert_observation(sanitized.inner().clone())
- .await?;
+ let log_title = sanitized.inner().title.clone();
+ let _ = state.writer.insert_observation(sanitized).await?;
// Append the log line to the per-project log.md.
if let Err(e) = log::append_event(
@@ -2000,7 +2001,7 @@ async fn process(
proj,
Timestamp::now(),
env.event,
- sanitized.inner().title.as_str(),
+ log_title.as_str(),
) {
warn!(error = %e, "log.md append failed");
}
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 9402ba6a..9256d87f 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -4972,6 +4972,7 @@ fn map_user_store_err(e: ai_memory_store::StoreError) -> (StatusCode, Json<serde
mod tests {
use super::*;
use ai_memory_core::{AgentKind, NewObservation, NewSession, ObservationKind};
+ use ai_memory_core::{Sanitized, Sanitizer};
use ai_memory_llm::{ChatRequest, ChatResponse, LlmResult};
use ai_memory_store::Store;
use axum::body::to_bytes;
@@ -5409,17 +5410,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "prompt".into(),
- body: "focus changed and durable lesson".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "focus changed and durable lesson".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
@@ -5536,17 +5540,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "prompt".into(),
- body: "focus changed and durable lesson".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "focus changed and durable lesson".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
let mut state =
@@ -5631,17 +5638,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "prompt".into(),
- body: "durable lesson".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "durable lesson".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
let mut state =
diff --git a/crates/ai-memory-mcp/src/server.rs b/crates/ai-memory-mcp/src/server.rs
index 94eaf677..2f46e56b 100644
--- a/crates/ai-memory-mcp/src/server.rs
+++ b/crates/ai-memory-mcp/src/server.rs
@@ -2799,6 +2799,7 @@ fn test_optional_parts() -> OptionalParts {
#[cfg(test)]
mod tests {
use super::*;
+ use ai_memory_core::{Sanitized, Sanitizer};
use std::collections::BTreeSet;
#[test]
@@ -2973,17 +2974,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id,
- project_id,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: title.into(),
- body: body.into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id,
+ project_id,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: title.into(),
+ body: body.into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
}
@@ -4075,17 +4079,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "raw prompt".into(),
- body: "raw fallback contains quokka only detail".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "raw prompt".into(),
+ body: "raw fallback contains quokka only detail".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
@@ -4144,17 +4151,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: scoped,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "raw prompt".into(),
- body: "raw fallback contains quokka only detail".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: scoped,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "raw prompt".into(),
+ body: "raw fallback contains quokka only detail".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
diff --git a/crates/ai-memory-mcp/tests/admin_move.rs b/crates/ai-memory-mcp/tests/admin_move.rs
index 41fa0c4a..52cc50ab 100644
--- a/crates/ai-memory-mcp/tests/admin_move.rs
+++ b/crates/ai-memory-mcp/tests/admin_move.rs
@@ -13,7 +13,7 @@
//! both versions survive), then purge the source — there the episodic rows
//! (sessions/observations/handoffs) are dropped by the purge.
-use ai_memory_core::{PagePath, Tier};
+use ai_memory_core::{PagePath, Sanitized, Sanitizer, Tier};
use ai_memory_mcp::{AdminState, admin_router};
use ai_memory_store::{DecayParams, Store};
use ai_memory_wiki::{
@@ -573,17 +573,20 @@ async fn move_project_true_move_preserves_sessions_and_observations() {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id: sid,
- workspace_id: src_ws,
- project_id: src_proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "prompt".into(),
- body: "do the thing".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id: sid,
+ workspace_id: src_ws,
+ project_id: src_proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "do the thing".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
diff --git a/crates/ai-memory-mcp/tests/admin_phase3.rs b/crates/ai-memory-mcp/tests/admin_phase3.rs
index c6d150e3..ba534cf4 100644
--- a/crates/ai-memory-mcp/tests/admin_phase3.rs
+++ b/crates/ai-memory-mcp/tests/admin_phase3.rs
@@ -9,7 +9,7 @@
use ai_memory_core::{
ActorContext, AgentKind, NewObservation, NewPage, NewSession, ObservationKind, PagePath,
- SessionId, Tier,
+ Sanitized, Sanitizer, SessionId, Tier,
};
use ai_memory_llm::SyntheticEmbedder;
use ai_memory_mcp::{AdminState, admin_router};
@@ -346,17 +346,20 @@ async fn seed_sessions_for_reorg(store: &Store) -> (SessionId, SessionId) {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id: sid_a,
- workspace_id: ws,
- project_id: scratch,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "alpha prompt".into(),
- body: "".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id: sid_a,
+ workspace_id: ws,
+ project_id: scratch,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "alpha prompt".into(),
+ body: "".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
@@ -374,17 +377,20 @@ async fn seed_sessions_for_reorg(store: &Store) -> (SessionId, SessionId) {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id: sid_b,
- workspace_id: ws,
- project_id: scratch,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "beta prompt".into(),
- body: "".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id: sid_b,
+ workspace_id: ws,
+ project_id: scratch,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "beta prompt".into(),
+ body: "".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
diff --git a/crates/ai-memory-mcp/tests/admin_purge.rs b/crates/ai-memory-mcp/tests/admin_purge.rs
index 3ef2a230..598d401c 100644
--- a/crates/ai-memory-mcp/tests/admin_purge.rs
+++ b/crates/ai-memory-mcp/tests/admin_purge.rs
@@ -6,7 +6,7 @@
use ai_memory_core::{
AgentKind, NewHandoff, NewObservation, NewSession, ObservationKind, PagePath, ProjectId,
- SessionId, Tier, WorkspaceId,
+ Sanitized, Sanitizer, SessionId, Tier, WorkspaceId,
};
use ai_memory_mcp::{AdminState, admin_router};
use ai_memory_store::{DecayParams, Store};
@@ -139,17 +139,20 @@ async fn seed_two_projects(store: &Store, wiki: &Wiki) -> (WorkspaceId, ProjectI
for i in 0..3u8 {
store
.writer
- .insert_observation(NewObservation {
- session_id: sid,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: format!("{label} obs {i}"),
- body: "body".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id: sid,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: format!("{label} obs {i}"),
+ body: "body".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
}
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 46713d87..1f277cb2 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -120,8 +120,8 @@ mod tests {
use super::*;
use ai_memory_core::{
ActorContext, AgentKind, LinkTarget, NewObservation, NewPage, NewSession,
- NewWorkstreamEvent, ObservationId, ObservationKind, PageId, PagePath, ProjectId, SessionId,
- Tier, UserId, WorkspaceId, WorkstreamEventKind,
+ NewWorkstreamEvent, ObservationId, ObservationKind, PageId, PagePath, ProjectId, Sanitized,
+ Sanitizer, SessionId, Tier, UserId, WorkspaceId, WorkstreamEventKind,
};
use rusqlite::{Connection, params};
use sha2::{Digest, Sha256};
@@ -2319,17 +2319,20 @@ mod tests {
.unwrap();
store
.writer
- .insert_observation(NewObservation {
- session_id,
- workspace_id: ws,
- project_id: proj,
- kind: ObservationKind::UserPrompt,
- extension: None,
- source_event: None,
- title: "prompt".into(),
- body: "the raw-only zebra detail lives here".into(),
- importance: 5,
- })
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "the raw-only zebra detail lives here".into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
.await
.unwrap();
@@ -2343,6 +2346,71 @@ mod tests {
assert!(hits[0].snippet.contains("<mark>zebra</mark>"));
}
+ /// The public writer API only accepts `Sanitized<NewObservation>`, and
+ /// `Sanitized::new` is the single constructor — so a secret passed in an
+ /// observation body cannot reach disk unscrubbed. This locks the typed
+ /// boundary end-to-end: through the writer, into the SQLite `body` column.
+ #[tokio::test]
+ async fn insert_observation_boundary_scrubs_before_disk() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "ai-memory", None)
+ .await
+ .unwrap();
+ let session_id = SessionId::new();
+ store
+ .writer
+ .begin_session(NewSession {
+ id: session_id,
+ workspace_id: ws,
+ project_id: proj,
+ agent_kind: AgentKind::ClaudeCode,
+ cwd: None,
+ })
+ .await
+ .unwrap();
+ store
+ .writer
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "Authorization: Bearer abcdef0123456789ABCDEF0123456789".into(),
+ body: "leaked Authorization: Bearer abcdef0123456789ABCDEF0123456789 in transcript"
+ .into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
+ .await
+ .unwrap();
+
+ let conn = Connection::open(store.db_path()).unwrap();
+ let (title, body): (String, String) = conn
+ .query_row("SELECT title, body FROM observations", [], |r| {
+ Ok((r.get(0)?, r.get(1)?))
+ })
+ .unwrap();
+ for col in [&title, &body] {
+ assert!(col.contains("[REDACTED]"), "expected scrub in: {col}");
+ assert!(
+ !col.contains("abcdef0123"),
+ "secret reached disk unscrubbed: {col}"
+ );
+ }
+ }
+
#[tokio::test]
async fn latest_completed_session_for_project_ignores_open_sessions() {
let tmp = TempDir::new().unwrap();
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index c1693ff6..bc22ac30 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -12,7 +12,7 @@ use std::thread::{self, JoinHandle};
use ai_memory_core::{
AgentKind, HandoffId, ManagedRunId, NewHandoff, NewObservation, NewPage, NewSession, NewUser,
- ObservationId, PageId, PagePath, ProjectId, SessionId, UserId, WorkspaceId,
+ ObservationId, PageId, PagePath, ProjectId, Sanitized, SessionId, UserId, WorkspaceId,
};
use rusqlite::Connection;
use tokio::sync::{mpsc, oneshot};
@@ -473,9 +473,18 @@ impl WriterHandle {
/// Append an observation row.
///
+ /// Takes [`Sanitized<NewObservation>`] — the privacy strip is enforced by
+ /// the type system at the store boundary, so an unsanitized observation
+ /// cannot reach disk by construction ([`Sanitized::new`] is the single
+ /// constructor and applies the scrub).
+ ///
/// # Errors
/// Returns [`StoreError::WriterClosed`] or propagates SQL errors.
- pub async fn insert_observation(&self, obs: NewObservation) -> StoreResult<ObservationId> {
+ pub async fn insert_observation(
+ &self,
+ obs: Sanitized<NewObservation>,
+ ) -> StoreResult<ObservationId> {
+ let obs = obs.into_inner();
let (tx, rx) = oneshot::channel();
self.send(WriteCmd::InsertObservation { obs, reply: tx })
.await?;