Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(retrieval): add an opt-in observation-evidence stream to hybrid search
AkitaOnRails7e19e18d8147
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rsindex fff30358..e0f2f45c 100644--- a/crates/ai-memory-cli/src/config.rs+++ b/crates/ai-memory-cli/src/config.rs@@ -1641,7 +1641,8 @@ impl DreamSettings {/// that never heard of this section.////// Env form: `AI_MEMORY_RETRIEVAL__QUERY_INTENT=true`,-/// `AI_MEMORY_RETRIEVAL__ABSTRACT_VECTORS=true`.+/// `AI_MEMORY_RETRIEVAL__ABSTRACT_VECTORS=true`,+/// `AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM=true`.#[derive(Debug, Clone, Copy, Serialize, Deserialize)]#[serde(default)]pub struct RetrievalSettings {@@ -1663,6 +1664,11 @@ pub struct RetrievalSettings {/// `confidence` into its authority factor, inside the existing bounds./// OFF by default: enabling it is gated on a positive R2 delta.pub belief_authority_weight: f64,+ /// Add the observation-evidence stream (R1) to the RRF fusion: matching+ /// observation bodies vote, once per session, for that session's latest+ /// `sessions/<id>.md` page in the same project. Read-only; the wiki is+ /// not enlarged. OFF by default.+ pub observation_stream: bool,}impl Default for RetrievalSettings {@@ -1673,6 +1679,7 @@ impl Default for RetrievalSettings {session_recall_bonus: base.session_recall_bonus,abstract_vectors: base.abstract_vectors,belief_authority_weight: base.belief_authority_weight,+ observation_stream: base.observation_stream,}}}@@ -1689,6 +1696,7 @@ impl RetrievalSettings {// supported pages; clamp it out so misconfiguration is inert, not// inverted.belief_authority_weight: self.belief_authority_weight.max(0.0),+ observation_stream: self.observation_stream,}}}@@ -3306,6 +3314,53 @@ mod tests {}}+ #[test]+ fn retrieval_observation_stream_defaults_off_and_loads_from_toml() {+ assert!(!Config::default().retrieval.tuning().observation_stream);+ let tmp = TempDir::new().unwrap();+ let config_path = tmp.path().join("config.toml");+ std::fs::write(&config_path, "[retrieval]\nobservation_stream = true\n").unwrap();+ let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();+ assert!(config.retrieval.tuning().observation_stream);+ // The other [retrieval] signals keep their defaults.+ assert!(!config.retrieval.tuning().abstract_vectors);+ }++ #[test]+ fn retrieval_observation_stream_env_overrides_config() {+ const CHILD_MARKER: &str = "AI_MEMORY_TEST_OBSERVATION_STREAM_CHILD";+ if std::env::var_os(CHILD_MARKER).is_some() {+ let tmp = TempDir::new().unwrap();+ let config_path = tmp.path().join("config.toml");+ std::fs::write(&config_path, "[retrieval]\nobservation_stream = false\n").unwrap();+ let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();+ assert!(config.retrieval.tuning().observation_stream);+ println!("observation-stream-env-applied");+ return;+ }++ let output = std::process::Command::new(std::env::current_exe().unwrap())+ .arg("--exact")+ .arg("config::tests::retrieval_observation_stream_env_overrides_config")+ .arg("--test-threads=1")+ .arg("--nocapture")+ .env(CHILD_MARKER, "1")+ .env("AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM", "true")+ .output()+ .unwrap();+ assert!(+ output.status.success(),+ "child test failed:\nstdout: {}\nstderr: {}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ assert!(+ String::from_utf8_lossy(&output.stdout).contains("observation-stream-env-applied"),+ "child stdout: {}",+ String::from_utf8_lossy(&output.stdout)+ );+ }+#[test]fn load_rejects_destructive_invalid_breadth_weights() {for value in ["-0.1", "nan", "inf"] {diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex bc024085..0847e5df 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -328,6 +328,17 @@ const AUTHORITY_CANDIDATE_MULTIPLIER: usize = 4;const AUTHORITY_MIN_CANDIDATES: usize = 20;const AUTHORITY_MAX_EXTRA_CANDIDATES: usize = 300;+/// Observation rows the R1 observation stream reads per stream candidate.+/// Observations outnumber session pages by an order of magnitude and one+/// session's turns often match together, so the window is wider than the+/// page-candidate window it feeds; deduplication by session brings it back+/// down to at most `candidate_limit` pages.+const OBSERVATION_STREAM_ROWS_PER_CANDIDATE: usize = 4;+/// Hard cap on observation rows read per query, whatever the limit.+const OBSERVATION_STREAM_MAX_ROWS: usize = 1_000;+/// Matched observation ids reported per page in `explain`.+const OBSERVATION_STREAM_IDS_PER_PAGE: usize = 3;+#[derive(Debug, Clone, Copy)]struct PageAuthority {factor: f64,@@ -638,6 +649,20 @@ struct GraphNeighbor {edge: Option<String>,}+/// One session page voted for by the R1 observation stream. Internal to+/// hybrid search + explain.+struct ObservationVote {+ page_id: PageId,+ path: PagePath,+ title: String,+ /// FTS5 snippet of the session's best-ranked matching observation.+ /// Observations are sanitized and bounded on the way in, and the snippet+ /// is a bounded window over that stored text.+ snippet: String,+ /// Best-ranked first, at most [`OBSERVATION_STREAM_IDS_PER_PAGE`].+ observation_ids: Vec<ObservationId>,+}+/// Which seed page pulled a hit in via graph expansion, and the link/// direction. Part of [`SearchExplain`].#[derive(Debug, Clone, Serialize)]@@ -670,6 +695,10 @@ pub struct RrfContributions {/// `[retrieval] abstract_vectors`; 0.0 when off or missed).#[serde(rename = "abstract", skip_serializing_if = "is_zero")]pub abstract_vector: f64,+ /// Observation-evidence stream contribution (opt-in+ /// `[retrieval] observation_stream`; 0.0 when off or missed).+ #[serde(skip_serializing_if = "is_zero")]+ pub observation: f64,}fn is_zero(v: &f64) -> bool {@@ -718,6 +747,16 @@ pub struct SearchExplain {/// Entity names on this page that matched the query.#[serde(skip_serializing_if = "Vec::is_empty")]pub matched_entities: Vec<String>,+ /// 1-based rank in the observation-evidence stream (opt-in+ /// `[retrieval] observation_stream`): where this session page's+ /// best-matching observation placed among the matching sessions.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub observation_rank: Option<usize>,+ /// The matching observations that voted for this session page, best+ /// first and bounded. Only the best one contributes to `fused`; the+ /// session counts once however many of its observations match.+ #[serde(skip_serializing_if = "Vec::is_empty")]+ pub observation_ids: Vec<ObservationId>,/// Per-stream RRF contributions.pub rrf: RrfContributions,/// Total fused score (sum of the RRF contributions) — higher is@@ -2769,6 +2808,173 @@ impl ReaderPool {.await}+ /// The R1 observation stream: sessions whose observations match `query`,+ /// mapped to their latest `sessions/<id>.md` page in the same scope and+ /// ordered by each session's best-ranked observation.+ ///+ /// Both halves of the join are filtered to `(workspace_id, project_id)`:+ /// an observation from another project never votes, and a same-path page+ /// in another project never receives a vote. A session counts once+ /// however many of its observations match. A session with no latest,+ /// unexpired page in this scope contributes nothing; the raw-observation+ /// fallback still covers live sessions. The two reads share one read+ /// transaction so the mapping sees a single snapshot. Read-only.+ async fn observation_stream_votes(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ query: &str,+ candidate_limit: usize,+ cutoff: i64,+ ) -> StoreResult<Vec<ObservationVote>> {+ let fts_query = normalize_fts_query(query, &self.fts_stopwords);+ if fts_query.is_empty() || candidate_limit == 0 {+ return Ok(Vec::new());+ }+ let row_limit = candidate_limit+ .saturating_mul(OBSERVATION_STREAM_ROWS_PER_CANDIDATE)+ .min(OBSERVATION_STREAM_MAX_ROWS);+ self.with_conn(move |conn| {+ let tx = conn.unchecked_transaction()?;+ // `observations_fts` spans every project, and bm25 runs for every+ // match before the scope filter can drop it. The project's rowid+ // span (one covering-index probe) bounds the index scan to the+ // stretch of the log where this project wrote; the scope+ // predicates below still decide membership inside it.+ let span: Option<(i64, i64)> = tx+ .query_row(+ "SELECT MIN(rowid), MAX(rowid) FROM observations \+ WHERE workspace_id = ?1 AND project_id = ?2",+ params![workspace_id.as_bytes(), project_id.as_bytes()],+ |row| Ok((row.get::<_, Option<i64>>(0)?, row.get::<_, Option<i64>>(1)?)),+ )+ .map(|(lo, hi)| lo.zip(hi))?;+ let Some((first_rowid, last_rowid)) = span else {+ return Ok(Vec::new());+ };+ // Best-ranked observation first, grouped by session in first-seen+ // order: the session's stream rank is its best observation's.+ let mut sessions: Vec<(SessionId, String, Vec<ObservationId>)> = Vec::new();+ {+ // CROSS JOIN pins the FTS table as the outer loop: left to+ // itself the planner drives from the observations scope index+ // and re-enters FTS once per row, which is far slower.+ let mut stmt = tx.prepare(+ "SELECT observations.id, observations.session_id, \+ snippet(observations_fts, 1, '<mark>', '</mark>', '…', 24) \+ FROM observations_fts \+ CROSS JOIN observations ON observations.rowid = observations_fts.rowid \+ WHERE observations_fts MATCH ?1 \+ AND observations_fts.rowid >= ?5 \+ AND observations_fts.rowid <= ?6 \+ AND observations.workspace_id = ?2 \+ AND observations.project_id = ?3 \+ ORDER BY observations_fts.rank \+ LIMIT ?4",+ )?;+ #[allow(clippy::cast_possible_wrap)]+ let rows = stmt.query_map(+ params![+ fts_query,+ workspace_id.as_bytes(),+ project_id.as_bytes(),+ row_limit as i64,+ first_rowid,+ last_rowid,+ ],+ |row| {+ Ok((+ row.get::<_, Vec<u8>>(0)?,+ row.get::<_, Vec<u8>>(1)?,+ row.get::<_, String>(2)?,+ ))+ },+ )?;+ let mut index: std::collections::HashMap<SessionId, usize> =+ std::collections::HashMap::new();+ for row in rows {+ let (id_bytes, session_bytes, snippet) = row?;+ let id = ObservationId::from_slice(&id_bytes)?;+ let session_id = SessionId::from_slice(&session_bytes)?;+ match index.get(&session_id) {+ Some(&at) => {+ let ids = &mut sessions[at].2;+ if ids.len() < OBSERVATION_STREAM_IDS_PER_PAGE {+ ids.push(id);+ }+ }+ None => {+ index.insert(session_id, sessions.len());+ sessions.push((session_id, snippet, vec![id]));+ }+ }+ }+ }+ if sessions.is_empty() {+ return Ok(Vec::new());+ }++ let mut sql_params: Vec<Value> = Vec::with_capacity(sessions.len() + 3);+ sql_params.push(Value::Blob(workspace_id.as_bytes().to_vec()));+ sql_params.push(Value::Blob(project_id.as_bytes().to_vec()));+ sql_params.push(Value::Integer(cutoff));+ let mut placeholders = String::with_capacity(sessions.len() * 3);+ for (idx, (session_id, _, _)) in sessions.iter().enumerate() {+ if idx > 0 {+ placeholders.push_str(", ");+ }+ placeholders.push('?');+ sql_params.push(Value::Text(format!("sessions/{session_id}.md")));+ }+ let sql = format!(+ "SELECT pages.id, pages.path, pages.title \+ FROM pages \+ WHERE pages.workspace_id = ?1 \+ AND pages.project_id = ?2 \+ AND pages.is_latest = 1{not_expired} \+ AND pages.path IN ({placeholders})",+ not_expired = not_expired("pages", "?3"),+ );+ let mut pages: std::collections::HashMap<String, (PageId, String)> =+ std::collections::HashMap::new();+ {+ let mut stmt = tx.prepare(&sql)?;+ let rows = stmt.query_map(params_from_iter(sql_params.iter()), |row| {+ Ok((+ row.get::<_, Vec<u8>>(0)?,+ row.get::<_, String>(1)?,+ row.get::<_, String>(2)?,+ ))+ })?;+ for row in rows {+ let (id_bytes, path, title) = row?;+ pages.insert(path, (PageId::from_slice(&id_bytes)?, title));+ }+ }+ tx.finish()?;++ let mut votes = Vec::new();+ for (session_id, snippet, observation_ids) in sessions {+ let path = format!("sessions/{session_id}.md");+ let Some((page_id, title)) = pages.remove(&path) else {+ continue;+ };+ votes.push(ObservationVote {+ page_id,+ path: PagePath::new(path)?,+ title,+ snippet,+ observation_ids,+ });+ if votes.len() == candidate_limit {+ break;+ }+ }+ Ok(votes)+ })+ .await+ }+/// Return the N most-recently-updated `is_latest = 1` pages.////// # Errors@@ -5621,6 +5827,12 @@ impl ReaderPool {).await?};+ // The stream-off hot path keeps moving the query without a clone.+ let observation_query = if tuning.observation_stream {+ query.clone()+ } else {+ String::new()+ };let fts_candidates = self.search_page_candidates_for_project(workspace_id,@@ -5710,6 +5922,20 @@ impl ReaderPool {include_superseded,).await?;+ // R1: the observation-evidence stream, off by default. It votes only+ // for latest session pages, so it does not seed graph expansion.+ let observation_votes = if tuning.observation_stream {+ self.observation_stream_votes(+ workspace_id,+ project_id,+ &observation_query,+ candidate_limit,+ cutoff,+ )+ .await?+ } else {+ Vec::new()+ };// RRF fuse: score(d) = Σ 1/(k + rank_i(d)) over rankers.let k = 60.0_f64;@@ -5719,6 +5945,10 @@ impl ReaderPool {snippet: String,score: f64,explain: Option<SearchExplain>,+ /// 1-based FTS rank, kept to decide whose snippet explains the hit.+ fts_rank: Option<usize>,+ /// Observation-stream rank and the matching observation's snippet.+ observation: Option<(usize, String)>,}let mut fused: std::collections::HashMap<PageId, Fused> = std::collections::HashMap::new();@@ -5730,8 +5960,11 @@ impl ReaderPool {snippet: h.snippet.clone(),score: 0.0,explain: explain.then(SearchExplain::default),+ fts_rank: None,+ observation: None,});entry.score += contrib;+ entry.fts_rank = Some(rank + 1);if let Some(details) = &mut entry.explain {details.fts_rank = Some(rank + 1);details.fts_score = Some(h.rank);@@ -5747,6 +5980,8 @@ impl ReaderPool {snippet: String::new(),score: 0.0,explain: explain.then(SearchExplain::default),+ fts_rank: None,+ observation: None,});entry.score += contrib;if let Some(details) = &mut entry.explain {@@ -5764,6 +5999,8 @@ impl ReaderPool {snippet: String::new(),score: 0.0,explain: explain.then(SearchExplain::default),+ fts_rank: None,+ observation: None,});entry.score += contrib;if let Some(details) = &mut entry.explain {@@ -5781,6 +6018,8 @@ impl ReaderPool {snippet: e.hit.snippet.clone(),score: 0.0,explain: explain.then(SearchExplain::default),+ fts_rank: None,+ observation: None,});entry.score += contrib;if let Some(details) = &mut entry.explain {@@ -5799,6 +6038,8 @@ impl ReaderPool {snippet: n.hit.snippet.clone(),score: 0.0,explain: explain.then(SearchExplain::default),+ fts_rank: None,+ observation: None,});entry.score += contrib;if let Some(details) = &mut entry.explain {@@ -5817,15 +6058,50 @@ impl ReaderPool {}}+ for (rank, vote) in observation_votes.into_iter().enumerate() {+ let contrib = 1.0 / (k + (rank + 1) as f64);+ let entry = fused.entry(vote.page_id).or_insert_with(|| Fused {+ path: vote.path,+ title: String::new(),+ snippet: String::new(),+ score: 0.0,+ explain: explain.then(SearchExplain::default),+ fts_rank: None,+ observation: None,+ });+ entry.score += contrib;+ if entry.title.is_empty() {+ entry.title = vote.title;+ }+ entry.observation = Some((rank + 1, vote.snippet));+ if let Some(details) = &mut entry.explain {+ details.observation_rank = Some(rank + 1);+ details.observation_ids = vote.observation_ids;+ details.rrf.observation = contrib;+ details.fused += contrib;+ }+ }+let mut out: Vec<(PageHit, Option<SearchExplain>)> = fused.into_iter().map(|(id, entry)| {+ // The matching observation's excerpt explains the hit when+ // the observation stream ranked the page ahead of FTS (or FTS+ // missed it); otherwise the page keeps its own snippet.+ let snippet = match entry.observation {+ Some((observation_rank, observation_snippet))+ if entry.fts_rank.is_none_or(|fts| observation_rank < fts) =>+ {+ observation_snippet+ }+ _ => entry.snippet,+ };(PageHit {id,path: entry.path,title: entry.title,- snippet: entry.snippet,+ snippet,rank: -entry.score, // lower = better (matches FTS5 convention)superseded: false,pinned: false,diff --git a/crates/ai-memory-store/src/retrieval_tuning.rs b/crates/ai-memory-store/src/retrieval_tuning.rsindex fcc8dae7..b0d6871c 100644--- a/crates/ai-memory-store/src/retrieval_tuning.rs+++ b/crates/ai-memory-store/src/retrieval_tuning.rs@@ -1,6 +1,6 @@//! Opt-in post-fusion ranking signals for `hybrid_search`.//!-//! Two bounded, independently switchable signals, both off by default so a+//! Bounded, independently switchable signals, all off by default so a//! store that never configures them ranks exactly as before://!//! * **Session-recall routing** — a zero-LLM lexical check that recognises@@ -19,12 +19,19 @@//! a precise second vote per page. Contributes nothing while the table//! is empty.//!-//! Both signals are reported in `SearchExplain`, so an `explain=true`+//! * **Observation stream** — a further RRF stream over `observations_fts`.+//! The zero-LLM session page keeps only a short excerpt of each turn, so+//! the evidence a question needs often lives only in the session's+//! sanitized observation bodies. Each session's best-ranked matching+//! observation votes once for that session's latest page; a session with+//! no page contributes nothing (the raw fallback still covers it).+//!+//! Every signal is reported in `SearchExplain`, so an `explain=true`//! query can account for the returned rank.use serde::{Deserialize, Serialize};-/// Tunables for the opt-in ranking signals. `Default` disables both.+/// Tunables for the opt-in ranking signals. `Default` disables all of them.#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]#[serde(default)]pub struct RetrievalTuning {@@ -49,6 +56,12 @@ pub struct RetrievalTuning {/// default on without a positive R2 delta (retrieval-triple / QA), not yet/// performed.pub belief_authority_weight: f64,+ /// Add the observation-evidence stream to the RRF fusion (R1): search+ /// `observations_fts` in the same project and let each matching+ /// observation vote, once per session, for its session's latest+ /// `sessions/<id>.md` page. Nothing is written. Ships OFF: it changes+ /// ranking, and the maintainer decides default-on from measurements.+ pub observation_stream: bool,}impl Default for RetrievalTuning {@@ -66,6 +79,7 @@ impl Default for RetrievalTuning {// OFF by default: folding belief confidence into ranking is// R2-gated (see the field doc).belief_authority_weight: 0.0,+ observation_stream: false,}}}@@ -171,6 +185,7 @@ mod tests {assert!(!t.session_recall_routing);assert!(!t.abstract_vectors);assert_eq!(t.belief_authority_weight, 0.0);+ assert!(!t.observation_stream);}#[test]diff --git a/crates/ai-memory-store/tests/suite/mod.rs b/crates/ai-memory-store/tests/suite/mod.rsindex 8c01a37e..bb392812 100644--- a/crates/ai-memory-store/tests/suite/mod.rs+++ b/crates/ai-memory-store/tests/suite/mod.rs@@ -21,6 +21,7 @@ mod profile;mod project_authz;mod reclaim_ledger_versions;mod related_walk;+mod retrieval_observation_stream;mod retrieval_superseded;mod retrieval_tuning_streams;mod session_ids_touching_scope;diff --git a/crates/ai-memory-store/tests/suite/multi_session.rs b/crates/ai-memory-store/tests/suite/multi_session.rsindex 25d9e161..fb65be94 100644--- a/crates/ai-memory-store/tests/suite/multi_session.rs+++ b/crates/ai-memory-store/tests/suite/multi_session.rs@@ -3261,3 +3261,158 @@ async fn two_harnesses_of_one_operator_share_one_private_profile() {assert!(read.is_ok(), "an open project stays shared");}}++/// R1 observation stream, per-project isolation (row 1). The stream maps an+/// observation to `sessions/<id>.md` by session id, so the two lookups it+/// joins are each a place a foreign scope could leak in: an observation from+/// another project voting for this project's page, or this project's+/// observation voting for a same-path page in another project. Both must be+/// filtered to the queried `(workspace_id, project_id)`.+#[tokio::test]+async fn observation_stream_never_crosses_a_project_or_workspace() {+ use super::retrieval_observation_stream::{observe, search, session, session_page, stream_on};++ let tmp = tempfile::tempdir().unwrap();+ let mut store = Store::open(tmp.path()).unwrap();+ store.reader.set_retrieval_tuning(stream_on());+ let (ws, home) = scope(&store).await;+ let foreign = store+ .writer+ .get_or_create_project(ws, "foreign-app".to_string(), None)+ .await+ .unwrap();+ let other_ws = store+ .writer+ .get_or_create_workspace("other-ws".to_string())+ .await+ .unwrap();+ let twin = store+ .writer+ .get_or_create_project(other_ws, "shared-app".to_string(), None)+ .await+ .unwrap();++ // A home session whose evidence lives only in its observation. Pages at+ // its session path exist in the foreign project and the cross-workspace+ // twin, but not (yet) at home.+ let sid = session(&store, ws, home).await;+ // The foreign and twin scopes write before and after it, so home's+ // observation sits inside their slice of the shared log and only the+ // scope predicates can keep it out.+ let foreign_sid = session(&store, ws, foreign).await;+ let twin_sid = session(&store, other_ws, twin).await;+ observe(&store, ws, foreign, foreign_sid, "foreign chatter before").await;+ observe(&store, other_ws, twin, twin_sid, "twin chatter before").await;+ let evidence = observe(&store, ws, home, sid, "the lynx password rotation plan").await;+ observe(&store, ws, foreign, foreign_sid, "foreign chatter after").await;+ observe(&store, other_ws, twin, twin_sid, "twin chatter after").await;+ session_page(&store, ws, foreign, sid, "foreign copy").await;+ session_page(&store, other_ws, twin, sid, "twin copy").await;++ // Home's observation must not vote for a page outside home...+ assert!(+ search(&store, ws, home, "lynx").await.is_empty(),+ "a home observation lifted a foreign page into home results"+ );+ // ...and the foreign scopes must not be lifted by home's observation.+ assert!(+ search(&store, ws, foreign, "lynx").await.is_empty(),+ "another project's observation lifted this project's page"+ );+ assert!(+ search(&store, other_ws, twin, "lynx").await.is_empty(),+ "another workspace's observation lifted this workspace's page"+ );++ // A foreign session's own observation votes for its own page there, and+ // never surfaces at home.+ session_page(&store, ws, foreign, foreign_sid, "foreign session").await;+ observe(&store, ws, foreign, foreign_sid, "ibex budget numbers").await;+ assert!(search(&store, ws, home, "ibex").await.is_empty());+ assert_eq!(search(&store, ws, foreign, "ibex").await.len(), 1);++ // Control: once home has its own session page, home's observation lifts+ // exactly that page.+ session_page(&store, ws, home, sid, "home copy").await;+ let hits = search(&store, ws, home, "lynx").await;+ assert_eq!(hits.len(), 1);+ assert_eq!(hits[0].0.path.as_str(), format!("sessions/{sid}.md"));+ assert_eq!(hits[0].0.title, format!("sessions/{sid}.md"));+ assert_eq!(hits[0].1.observation_ids, vec![evidence]);+ let home_ids = store+ .reader+ .latest_page_ids_by_paths(ws, home, vec![format!("sessions/{sid}.md")])+ .await+ .unwrap();+ assert_eq!(vec![hits[0].0.id], home_ids, "the vote went to home's page");+}++/// R1 observation stream, pages shared (row 5a, invariant #16). Operator B's+/// session, page and observations in a shared project are evidence for+/// operator A's query: neither the session owner nor `author_id` becomes a+/// read filter on the stream.+#[tokio::test]+async fn another_operators_observations_lift_their_session_page_in_a_shared_project() {+ use super::retrieval_observation_stream::{observe, search, stream_on};++ let tmp = tempfile::tempdir().unwrap();+ let mut store = Store::open(tmp.path()).unwrap();+ store.reader.set_retrieval_tuning(stream_on());+ let (ws, proj) = scope(&store).await;+ let operator_b = store+ .writer+ .create_human_user(+ NewUser {+ username: "operator-b".into(),+ name: None,+ email: None,+ },+ UserRole::User,+ None,+ false,+ )+ .await+ .unwrap();+ let sid = SessionId::new();+ store+ .writer+ .begin_session(NewSession {+ occurred_at: None,+ id: sid,+ workspace_id: ws,+ project_id: proj,+ agent_kind: AgentKind::Codex,+ cwd: Some("/repo".into()),+ actor_user: Some(operator("operator-b")),+ })+ .await+ .unwrap();+ store+ .writer+ .upsert_page(NewPage {+ author_id: Some(operator_b),+ ..page(+ ws,+ proj,+ &format!("sessions/{sid}.md"),+ "B's session",+ "Operator B worked on the build.",+ )+ })+ .await+ .unwrap();+ let evidence = observe(+ &store,+ ws,+ proj,+ sid,+ "the walrus migration needs a backfill",+ )+ .await;++ // Operator A's query carries no owner coordinate: hybrid search has none.+ let hits = search(&store, ws, proj, "walrus backfill").await;+ assert_eq!(hits.len(), 1, "B's session evidence must be visible to A");+ assert_eq!(hits[0].0.path.as_str(), format!("sessions/{sid}.md"));+ assert_eq!(hits[0].1.observation_ids, vec![evidence]);+}diff --git a/crates/ai-memory-store/tests/suite/retrieval_observation_stream.rs b/crates/ai-memory-store/tests/suite/retrieval_observation_stream.rsnew file mode 100644index 00000000..3f66cdda--- /dev/null+++ b/crates/ai-memory-store/tests/suite/retrieval_observation_stream.rs@@ -0,0 +1,383 @@+//! `[retrieval] observation_stream` (R1): matching observation bodies vote,+//! once per session, for that session's latest `sessions/<id>.md` page as one+//! more RRF stream. Off by default and byte-identical when off.++use ai_memory_core::{+ AgentKind, NewObservation, NewPage, NewSession, ObservationId, ObservationKind, PagePath,+ ProjectId, Sanitized, Sanitizer, SessionId, Tier, WorkspaceId,+};+use ai_memory_store::{PageHit, RetrievalTuning, SearchExplain, Store};++pub(crate) fn stream_on() -> RetrievalTuning {+ RetrievalTuning {+ observation_stream: true,+ ..RetrievalTuning::default()+ }+}++pub(crate) async fn scope(store: &Store, ws: &str, proj: &str) -> (WorkspaceId, ProjectId) {+ let ws = store+ .writer+ .get_or_create_workspace(ws.to_string())+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, proj.to_string(), None)+ .await+ .unwrap();+ (ws, proj)+}++pub(crate) async fn session(store: &Store, ws: WorkspaceId, proj: ProjectId) -> SessionId {+ let id = SessionId::new();+ store+ .writer+ .begin_session(NewSession {+ occurred_at: None,+ id,+ workspace_id: ws,+ project_id: proj,+ agent_kind: AgentKind::ClaudeCode,+ cwd: Some("/repo".into()),+ actor_user: None,+ })+ .await+ .unwrap();+ id+}++pub(crate) async fn observe(+ store: &Store,+ ws: WorkspaceId,+ proj: ProjectId,+ session_id: SessionId,+ body: &str,+) -> ObservationId {+ store+ .writer+ .insert_observation(Sanitized::new(+ NewObservation {+ occurred_at: None,+ session_id,+ workspace_id: ws,+ project_id: proj,+ kind: ObservationKind::UserPrompt,+ extension: None,+ source_event: None,+ title: "prompt".into(),+ body: body.into(),+ importance: 5,+ },+ &Sanitizer::builtin(),+ ))+ .await+ .unwrap()+}++pub(crate) fn page(ws: WorkspaceId, proj: ProjectId, path: &str, body: &str) -> NewPage {+ NewPage {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new(path).unwrap(),+ title: path.into(),+ body: body.into(),+ tier: Tier::Episodic,+ frontmatter_json: serde_json::json!({}),+ pinned: false,+ links: Vec::new(),+ author_id: None,+ expires_at: None,+ entities: Vec::new(),+ evidence: Vec::new(),+ }+}++pub(crate) async fn session_page(+ store: &Store,+ ws: WorkspaceId,+ proj: ProjectId,+ session_id: SessionId,+ body: &str,+) {+ store+ .writer+ .upsert_page(page(ws, proj, &format!("sessions/{session_id}.md"), body))+ .await+ .unwrap();+}++pub(crate) async fn search(+ store: &Store,+ ws: WorkspaceId,+ proj: ProjectId,+ query: &str,+) -> Vec<(PageHit, SearchExplain)> {+ store+ .reader+ .hybrid_search_explained(+ ws,+ proj,+ query.to_string(),+ None,+ String::new(),+ String::new(),+ 0,+ 10,+ None,+ false,+ )+ .await+ .unwrap()+}++fn paths(hits: &[(PageHit, SearchExplain)]) -> Vec<String> {+ hits.iter()+ .map(|(h, _)| h.path.as_str().to_string())+ .collect()+}++const K: f64 = 60.0;++#[tokio::test]+async fn an_observation_hit_lifts_its_own_session_page() {+ let tmp = tempfile::tempdir().unwrap();+ let mut store = Store::open(tmp.path()).unwrap();+ let (ws, proj) = scope(&store, "default", "app").await;+ let target = session(&store, ws, proj).await;+ let other = session(&store, ws, proj).await;+ // The session pages keep only a short excerpt: neither mentions the+ // evidence words.+ session_page(&store, ws, proj, target, "User asked about weekend plans.").await;+ session_page(&store, ws, proj, other, "User asked about weekend plans.").await;+ let evidence = observe(+ &store,+ ws,+ proj,+ target,+ "I finally bought the quasar telescope from the camera shop downtown",+ )+ .await;+ observe(&store, ws, proj, other, "we talked about gardening instead").await;++ // Off: the evidence is unreachable from pages.+ assert!(+ search(&store, ws, proj, "quasar telescope")+ .await+ .is_empty()+ );++ store.reader.set_retrieval_tuning(stream_on());+ let hits = search(&store, ws, proj, "quasar telescope").await;+ let target_path = format!("sessions/{target}.md");+ assert_eq!(paths(&hits), vec![target_path], "only the right session");+ let (hit, details) = &hits[0];+ assert_eq!(details.observation_rank, Some(1));+ assert_eq!(details.observation_ids, vec![evidence]);+ assert!((details.rrf.observation - 1.0 / (K + 1.0)).abs() < 1e-12);+ assert!((details.fused - details.rrf.observation).abs() < 1e-12);+ // The page had no snippet of its own for these words, so the bounded,+ // already-sanitized observation excerpt explains the hit.+ assert!(hit.snippet.contains("<mark>"), "{}", hit.snippet);+ assert!(hit.snippet.contains("quasar"), "{}", hit.snippet);+ assert!(!hit.title.is_empty());+}++#[tokio::test]+async fn a_session_counts_once_however_many_observations_match() {+ let tmp = tempfile::tempdir().unwrap();+ let mut store = Store::open(tmp.path()).unwrap();+ store.reader.set_retrieval_tuning(stream_on());+ let (ws, proj) = scope(&store, "default", "app").await;+ let chatty = session(&store, ws, proj).await;+ let quiet = session(&store, ws, proj).await;+ session_page(&store, ws, proj, chatty, "chatty session").await;+ session_page(&store, ws, proj, quiet, "quiet session").await;+ let mut chatty_ids = Vec::new();+ for i in 0..6 {+ chatty_ids.push(+ observe(+ &store,+ ws,+ proj,+ chatty,+ &format!("marmoset mention number {i} with filler words"),+ )+ .await,+ );+ }+ observe(&store, ws, proj, quiet, "marmoset marmoset marmoset").await;++ let hits = search(&store, ws, proj, "marmoset").await;+ assert_eq!(hits.len(), 2, "{:?}", paths(&hits));+ let chatty_path = format!("sessions/{chatty}.md");+ let (_, chatty_details) = hits+ .iter()+ .find(|(h, _)| h.path.as_str() == chatty_path)+ .unwrap();+ let rank = chatty_details.observation_rank.unwrap();+ // One contribution, at the session's own stream rank — six matching+ // observations do not add up to six votes.+ assert!(rank <= 2);+ assert!((chatty_details.rrf.observation - 1.0 / (K + rank as f64)).abs() < 1e-12);+ assert!((chatty_details.fused - chatty_details.rrf.observation).abs() < 1e-12);+ // The matched ids are bounded and all belong to the chatty session.+ assert!(!chatty_details.observation_ids.is_empty());+ assert!(chatty_details.observation_ids.len() <= 3);+ assert!(+ chatty_details+ .observation_ids+ .iter()+ .all(|id| chatty_ids.contains(id))+ );+ // Stream ranks are distinct per page: the stream ranks sessions, not+ // observations.+ let ranks: Vec<usize> = hits+ .iter()+ .map(|(_, d)| d.observation_rank.unwrap())+ .collect();+ let mut sorted = ranks.clone();+ sorted.sort_unstable();+ assert_eq!(sorted, vec![1, 2]);+}++#[tokio::test]+async fn a_session_without_a_page_contributes_nothing_and_the_raw_fallback_still_sees_it() {+ let tmp = tempfile::tempdir().unwrap();+ let mut store = Store::open(tmp.path()).unwrap();+ store.reader.set_retrieval_tuning(stream_on());+ let (ws, proj) = scope(&store, "default", "app").await;+ let live = session(&store, ws, proj).await;+ observe(+ &store,+ ws,+ proj,+ live,+ "pangolin sighting in the live session",+ )+ .await;+ // A page at a different session's path, and an unrelated note, must not+ // absorb the vote.+ let unrelated = session(&store, ws, proj).await;+ session_page(&store, ws, proj, unrelated, "nothing relevant").await;+ store+ .writer+ .upsert_page(page(ws, proj, "notes/plain.md", "plain note"))+ .await+ .unwrap();++ assert!(search(&store, ws, proj, "pangolin").await.is_empty());+ let raw = store+ .reader+ .search_observations_for_project(ws, proj, "pangolin".into(), 5)+ .await+ .unwrap();+ assert_eq!(raw.len(), 1);+ assert_eq!(raw[0].session_id, live);+}++#[tokio::test]+async fn a_superseded_or_expired_session_page_takes_no_vote() {+ let tmp = tempfile::tempdir().unwrap();+ let mut store = Store::open(tmp.path()).unwrap();+ store.reader.set_retrieval_tuning(stream_on());+ let (ws, proj) = scope(&store, "default", "app").await;+ let expired = session(&store, ws, proj).await;+ let mut expiring = page(ws, proj, &format!("sessions/{expired}.md"), "old session");+ expiring.expires_at = Some(jiff::Timestamp::from_second(1_000).unwrap());+ store.writer.upsert_page(expiring).await.unwrap();+ observe(&store, ws, proj, expired, "ocelot evidence").await;+ assert!(search(&store, ws, proj, "ocelot").await.is_empty());++ // A rewritten session page: only the latest version takes the vote.+ let rewritten = session(&store, ws, proj).await;+ session_page(&store, ws, proj, rewritten, "first draft").await;+ session_page(&store, ws, proj, rewritten, "second draft").await;+ observe(&store, ws, proj, rewritten, "axolotl evidence").await;+ let hits = search(&store, ws, proj, "axolotl").await;+ assert_eq!(hits.len(), 1);+ let latest = store+ .reader+ .latest_page_ids_by_paths(ws, proj, vec![format!("sessions/{rewritten}.md")])+ .await+ .unwrap();+ assert_eq!(vec![hits[0].0.id], latest);+}++/// Stream off must rank exactly as a store that never heard of the stream:+/// the observations present must not move a single hit, rank, or snippet,+/// and the explain JSON must carry no observation keys.+#[tokio::test]+async fn stream_off_ranks_byte_identically_to_a_store_without_observations() {+ async fn seeded(with_observations: bool) -> (tempfile::TempDir, Store, WorkspaceId, ProjectId) {+ let tmp = tempfile::tempdir().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let (ws, proj) = scope(&store, "default", "app").await;+ for (i, body) in [+ "kestrel nest under the bridge",+ "the kestrel came back",+ "nothing here",+ ]+ .iter()+ .enumerate()+ {+ store+ .writer+ .upsert_page(page(ws, proj, &format!("notes/n{i}.md"), body))+ .await+ .unwrap();+ }+ // Deterministic session ids so both stores share page paths.+ for i in 0..3_u128 {+ let id: SessionId = format!("00000000-0000-7000-8000-{:012x}", 0x1000 + i)+ .parse()+ .unwrap();+ store+ .writer+ .begin_session(NewSession {+ occurred_at: None,+ id,+ workspace_id: ws,+ project_id: proj,+ agent_kind: AgentKind::ClaudeCode,+ cwd: None,+ actor_user: None,+ })+ .await+ .unwrap();+ session_page(&store, ws, proj, id, "session about birds").await;+ if with_observations {+ observe(&store, ws, proj, id, "a kestrel hovered over the field").await;+ }+ }+ (tmp, store, ws, proj)+ }+ let (_a, plain, pws, pproj) = seeded(false).await;+ let (_b, observed, ows, oproj) = seeded(true).await;+ for query in ["kestrel", "birds", "kestrel birds"] {+ let expected = search(&plain, pws, pproj, query).await;+ let actual = search(&observed, ows, oproj, query).await;+ let shape = |hits: &[(PageHit, SearchExplain)]| {+ hits.iter()+ .map(|(h, d)| {+ let mut json = serde_json::to_value(d).unwrap();+ // Ids differ between the two stores; everything else+ // must match.+ json.as_object_mut().unwrap().remove("graph_via");+ (+ h.path.as_str().to_string(),+ h.title.clone(),+ h.snippet.clone(),+ h.rank.to_bits(),+ json,+ )+ })+ .collect::<Vec<_>>()+ };+ assert_eq!(shape(&expected), shape(&actual), "query {query}");+ for (_, details) in &actual {+ let json = serde_json::to_string(details).unwrap();+ assert!(!json.contains("observation"), "{json}");+ }+ }+}diff --git a/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs b/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rsindex c16563a4..27e08b9a 100644--- a/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs+++ b/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs@@ -36,6 +36,7 @@ fn tuning(session_recall: bool, bonus: f64, abstract_vectors: bool) -> Retrievalsession_recall_bonus: bonus,abstract_vectors,belief_authority_weight: 0.0,+ observation_stream: false,}}