Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(retrieval): add an opt-in observation-evidence stream to hybrid search

AkitaOnRails7e19e18d8147
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index fff30358..e0f2f45c 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -1641,7 +1641,8 @@ impl DreamSettings {
/// that never heard of this section.
///
/// Env form: `AI_MEMORY_RETRIEVAL__QUERY_INTENT=true`,
-/// `AI_MEMORY_RETRIEVAL__ABSTRACT_VECTORS=true`.
+/// `AI_MEMORY_RETRIEVAL__ABSTRACT_VECTORS=true`,
+/// `AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM=true`.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(default)]
pub struct RetrievalSettings {
@@ -1663,6 +1664,11 @@ pub struct RetrievalSettings {
/// `confidence` into its authority factor, inside the existing bounds.
/// OFF by default: enabling it is gated on a positive R2 delta.
pub belief_authority_weight: f64,
+ /// Add the observation-evidence stream (R1) to the RRF fusion: matching
+ /// observation bodies vote, once per session, for that session's latest
+ /// `sessions/<id>.md` page in the same project. Read-only; the wiki is
+ /// not enlarged. OFF by default.
+ pub observation_stream: bool,
}
impl Default for RetrievalSettings {
@@ -1673,6 +1679,7 @@ impl Default for RetrievalSettings {
session_recall_bonus: base.session_recall_bonus,
abstract_vectors: base.abstract_vectors,
belief_authority_weight: base.belief_authority_weight,
+ observation_stream: base.observation_stream,
}
}
}
@@ -1689,6 +1696,7 @@ impl RetrievalSettings {
// supported pages; clamp it out so misconfiguration is inert, not
// inverted.
belief_authority_weight: self.belief_authority_weight.max(0.0),
+ observation_stream: self.observation_stream,
}
}
}
@@ -3306,6 +3314,53 @@ mod tests {
}
}
+ #[test]
+ fn retrieval_observation_stream_defaults_off_and_loads_from_toml() {
+ assert!(!Config::default().retrieval.tuning().observation_stream);
+ let tmp = TempDir::new().unwrap();
+ let config_path = tmp.path().join("config.toml");
+ std::fs::write(&config_path, "[retrieval]\nobservation_stream = true\n").unwrap();
+ let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();
+ assert!(config.retrieval.tuning().observation_stream);
+ // The other [retrieval] signals keep their defaults.
+ assert!(!config.retrieval.tuning().abstract_vectors);
+ }
+
+ #[test]
+ fn retrieval_observation_stream_env_overrides_config() {
+ const CHILD_MARKER: &str = "AI_MEMORY_TEST_OBSERVATION_STREAM_CHILD";
+ if std::env::var_os(CHILD_MARKER).is_some() {
+ let tmp = TempDir::new().unwrap();
+ let config_path = tmp.path().join("config.toml");
+ std::fs::write(&config_path, "[retrieval]\nobservation_stream = false\n").unwrap();
+ let config = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();
+ assert!(config.retrieval.tuning().observation_stream);
+ println!("observation-stream-env-applied");
+ return;
+ }
+
+ let output = std::process::Command::new(std::env::current_exe().unwrap())
+ .arg("--exact")
+ .arg("config::tests::retrieval_observation_stream_env_overrides_config")
+ .arg("--test-threads=1")
+ .arg("--nocapture")
+ .env(CHILD_MARKER, "1")
+ .env("AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM", "true")
+ .output()
+ .unwrap();
+ assert!(
+ output.status.success(),
+ "child test failed:\nstdout: {}\nstderr: {}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ assert!(
+ String::from_utf8_lossy(&output.stdout).contains("observation-stream-env-applied"),
+ "child stdout: {}",
+ String::from_utf8_lossy(&output.stdout)
+ );
+ }
+
#[test]
fn load_rejects_destructive_invalid_breadth_weights() {
for value in ["-0.1", "nan", "inf"] {
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index bc024085..0847e5df 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -328,6 +328,17 @@ const AUTHORITY_CANDIDATE_MULTIPLIER: usize = 4;
const AUTHORITY_MIN_CANDIDATES: usize = 20;
const AUTHORITY_MAX_EXTRA_CANDIDATES: usize = 300;
+/// Observation rows the R1 observation stream reads per stream candidate.
+/// Observations outnumber session pages by an order of magnitude and one
+/// session's turns often match together, so the window is wider than the
+/// page-candidate window it feeds; deduplication by session brings it back
+/// down to at most `candidate_limit` pages.
+const OBSERVATION_STREAM_ROWS_PER_CANDIDATE: usize = 4;
+/// Hard cap on observation rows read per query, whatever the limit.
+const OBSERVATION_STREAM_MAX_ROWS: usize = 1_000;
+/// Matched observation ids reported per page in `explain`.
+const OBSERVATION_STREAM_IDS_PER_PAGE: usize = 3;
+
#[derive(Debug, Clone, Copy)]
struct PageAuthority {
factor: f64,
@@ -638,6 +649,20 @@ struct GraphNeighbor {
edge: Option<String>,
}
+/// One session page voted for by the R1 observation stream. Internal to
+/// hybrid search + explain.
+struct ObservationVote {
+ page_id: PageId,
+ path: PagePath,
+ title: String,
+ /// FTS5 snippet of the session's best-ranked matching observation.
+ /// Observations are sanitized and bounded on the way in, and the snippet
+ /// is a bounded window over that stored text.
+ snippet: String,
+ /// Best-ranked first, at most [`OBSERVATION_STREAM_IDS_PER_PAGE`].
+ observation_ids: Vec<ObservationId>,
+}
+
/// Which seed page pulled a hit in via graph expansion, and the link
/// direction. Part of [`SearchExplain`].
#[derive(Debug, Clone, Serialize)]
@@ -670,6 +695,10 @@ pub struct RrfContributions {
/// `[retrieval] abstract_vectors`; 0.0 when off or missed).
#[serde(rename = "abstract", skip_serializing_if = "is_zero")]
pub abstract_vector: f64,
+ /// Observation-evidence stream contribution (opt-in
+ /// `[retrieval] observation_stream`; 0.0 when off or missed).
+ #[serde(skip_serializing_if = "is_zero")]
+ pub observation: f64,
}
fn is_zero(v: &f64) -> bool {
@@ -718,6 +747,16 @@ pub struct SearchExplain {
/// Entity names on this page that matched the query.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub matched_entities: Vec<String>,
+ /// 1-based rank in the observation-evidence stream (opt-in
+ /// `[retrieval] observation_stream`): where this session page's
+ /// best-matching observation placed among the matching sessions.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub observation_rank: Option<usize>,
+ /// The matching observations that voted for this session page, best
+ /// first and bounded. Only the best one contributes to `fused`; the
+ /// session counts once however many of its observations match.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub observation_ids: Vec<ObservationId>,
/// Per-stream RRF contributions.
pub rrf: RrfContributions,
/// Total fused score (sum of the RRF contributions) — higher is
@@ -2769,6 +2808,173 @@ impl ReaderPool {
.await
}
+ /// The R1 observation stream: sessions whose observations match `query`,
+ /// mapped to their latest `sessions/<id>.md` page in the same scope and
+ /// ordered by each session's best-ranked observation.
+ ///
+ /// Both halves of the join are filtered to `(workspace_id, project_id)`:
+ /// an observation from another project never votes, and a same-path page
+ /// in another project never receives a vote. A session counts once
+ /// however many of its observations match. A session with no latest,
+ /// unexpired page in this scope contributes nothing; the raw-observation
+ /// fallback still covers live sessions. The two reads share one read
+ /// transaction so the mapping sees a single snapshot. Read-only.
+ async fn observation_stream_votes(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ query: &str,
+ candidate_limit: usize,
+ cutoff: i64,
+ ) -> StoreResult<Vec<ObservationVote>> {
+ let fts_query = normalize_fts_query(query, &self.fts_stopwords);
+ if fts_query.is_empty() || candidate_limit == 0 {
+ return Ok(Vec::new());
+ }
+ let row_limit = candidate_limit
+ .saturating_mul(OBSERVATION_STREAM_ROWS_PER_CANDIDATE)
+ .min(OBSERVATION_STREAM_MAX_ROWS);
+ self.with_conn(move |conn| {
+ let tx = conn.unchecked_transaction()?;
+ // `observations_fts` spans every project, and bm25 runs for every
+ // match before the scope filter can drop it. The project's rowid
+ // span (one covering-index probe) bounds the index scan to the
+ // stretch of the log where this project wrote; the scope
+ // predicates below still decide membership inside it.
+ let span: Option<(i64, i64)> = tx
+ .query_row(
+ "SELECT MIN(rowid), MAX(rowid) FROM observations \
+ WHERE workspace_id = ?1 AND project_id = ?2",
+ params![workspace_id.as_bytes(), project_id.as_bytes()],
+ |row| Ok((row.get::<_, Option<i64>>(0)?, row.get::<_, Option<i64>>(1)?)),
+ )
+ .map(|(lo, hi)| lo.zip(hi))?;
+ let Some((first_rowid, last_rowid)) = span else {
+ return Ok(Vec::new());
+ };
+ // Best-ranked observation first, grouped by session in first-seen
+ // order: the session's stream rank is its best observation's.
+ let mut sessions: Vec<(SessionId, String, Vec<ObservationId>)> = Vec::new();
+ {
+ // CROSS JOIN pins the FTS table as the outer loop: left to
+ // itself the planner drives from the observations scope index
+ // and re-enters FTS once per row, which is far slower.
+ let mut stmt = tx.prepare(
+ "SELECT observations.id, observations.session_id, \
+ snippet(observations_fts, 1, '<mark>', '</mark>', '…', 24) \
+ FROM observations_fts \
+ CROSS JOIN observations ON observations.rowid = observations_fts.rowid \
+ WHERE observations_fts MATCH ?1 \
+ AND observations_fts.rowid >= ?5 \
+ AND observations_fts.rowid <= ?6 \
+ AND observations.workspace_id = ?2 \
+ AND observations.project_id = ?3 \
+ ORDER BY observations_fts.rank \
+ LIMIT ?4",
+ )?;
+ #[allow(clippy::cast_possible_wrap)]
+ let rows = stmt.query_map(
+ params![
+ fts_query,
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ row_limit as i64,
+ first_rowid,
+ last_rowid,
+ ],
+ |row| {
+ Ok((
+ row.get::<_, Vec<u8>>(0)?,
+ row.get::<_, Vec<u8>>(1)?,
+ row.get::<_, String>(2)?,
+ ))
+ },
+ )?;
+ let mut index: std::collections::HashMap<SessionId, usize> =
+ std::collections::HashMap::new();
+ for row in rows {
+ let (id_bytes, session_bytes, snippet) = row?;
+ let id = ObservationId::from_slice(&id_bytes)?;
+ let session_id = SessionId::from_slice(&session_bytes)?;
+ match index.get(&session_id) {
+ Some(&at) => {
+ let ids = &mut sessions[at].2;
+ if ids.len() < OBSERVATION_STREAM_IDS_PER_PAGE {
+ ids.push(id);
+ }
+ }
+ None => {
+ index.insert(session_id, sessions.len());
+ sessions.push((session_id, snippet, vec![id]));
+ }
+ }
+ }
+ }
+ if sessions.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let mut sql_params: Vec<Value> = Vec::with_capacity(sessions.len() + 3);
+ sql_params.push(Value::Blob(workspace_id.as_bytes().to_vec()));
+ sql_params.push(Value::Blob(project_id.as_bytes().to_vec()));
+ sql_params.push(Value::Integer(cutoff));
+ let mut placeholders = String::with_capacity(sessions.len() * 3);
+ for (idx, (session_id, _, _)) in sessions.iter().enumerate() {
+ if idx > 0 {
+ placeholders.push_str(", ");
+ }
+ placeholders.push('?');
+ sql_params.push(Value::Text(format!("sessions/{session_id}.md")));
+ }
+ let sql = format!(
+ "SELECT pages.id, pages.path, pages.title \
+ FROM pages \
+ WHERE pages.workspace_id = ?1 \
+ AND pages.project_id = ?2 \
+ AND pages.is_latest = 1{not_expired} \
+ AND pages.path IN ({placeholders})",
+ not_expired = not_expired("pages", "?3"),
+ );
+ let mut pages: std::collections::HashMap<String, (PageId, String)> =
+ std::collections::HashMap::new();
+ {
+ let mut stmt = tx.prepare(&sql)?;
+ let rows = stmt.query_map(params_from_iter(sql_params.iter()), |row| {
+ Ok((
+ row.get::<_, Vec<u8>>(0)?,
+ row.get::<_, String>(1)?,
+ row.get::<_, String>(2)?,
+ ))
+ })?;
+ for row in rows {
+ let (id_bytes, path, title) = row?;
+ pages.insert(path, (PageId::from_slice(&id_bytes)?, title));
+ }
+ }
+ tx.finish()?;
+
+ let mut votes = Vec::new();
+ for (session_id, snippet, observation_ids) in sessions {
+ let path = format!("sessions/{session_id}.md");
+ let Some((page_id, title)) = pages.remove(&path) else {
+ continue;
+ };
+ votes.push(ObservationVote {
+ page_id,
+ path: PagePath::new(path)?,
+ title,
+ snippet,
+ observation_ids,
+ });
+ if votes.len() == candidate_limit {
+ break;
+ }
+ }
+ Ok(votes)
+ })
+ .await
+ }
+
/// Return the N most-recently-updated `is_latest = 1` pages.
///
/// # Errors
@@ -5621,6 +5827,12 @@ impl ReaderPool {
)
.await?
};
+ // The stream-off hot path keeps moving the query without a clone.
+ let observation_query = if tuning.observation_stream {
+ query.clone()
+ } else {
+ String::new()
+ };
let fts_candidates = self
.search_page_candidates_for_project(
workspace_id,
@@ -5710,6 +5922,20 @@ impl ReaderPool {
include_superseded,
)
.await?;
+ // R1: the observation-evidence stream, off by default. It votes only
+ // for latest session pages, so it does not seed graph expansion.
+ let observation_votes = if tuning.observation_stream {
+ self.observation_stream_votes(
+ workspace_id,
+ project_id,
+ &observation_query,
+ candidate_limit,
+ cutoff,
+ )
+ .await?
+ } else {
+ Vec::new()
+ };
// RRF fuse: score(d) = Σ 1/(k + rank_i(d)) over rankers.
let k = 60.0_f64;
@@ -5719,6 +5945,10 @@ impl ReaderPool {
snippet: String,
score: f64,
explain: Option<SearchExplain>,
+ /// 1-based FTS rank, kept to decide whose snippet explains the hit.
+ fts_rank: Option<usize>,
+ /// Observation-stream rank and the matching observation's snippet.
+ observation: Option<(usize, String)>,
}
let mut fused: std::collections::HashMap<PageId, Fused> = std::collections::HashMap::new();
@@ -5730,8 +5960,11 @@ impl ReaderPool {
snippet: h.snippet.clone(),
score: 0.0,
explain: explain.then(SearchExplain::default),
+ fts_rank: None,
+ observation: None,
});
entry.score += contrib;
+ entry.fts_rank = Some(rank + 1);
if let Some(details) = &mut entry.explain {
details.fts_rank = Some(rank + 1);
details.fts_score = Some(h.rank);
@@ -5747,6 +5980,8 @@ impl ReaderPool {
snippet: String::new(),
score: 0.0,
explain: explain.then(SearchExplain::default),
+ fts_rank: None,
+ observation: None,
});
entry.score += contrib;
if let Some(details) = &mut entry.explain {
@@ -5764,6 +5999,8 @@ impl ReaderPool {
snippet: String::new(),
score: 0.0,
explain: explain.then(SearchExplain::default),
+ fts_rank: None,
+ observation: None,
});
entry.score += contrib;
if let Some(details) = &mut entry.explain {
@@ -5781,6 +6018,8 @@ impl ReaderPool {
snippet: e.hit.snippet.clone(),
score: 0.0,
explain: explain.then(SearchExplain::default),
+ fts_rank: None,
+ observation: None,
});
entry.score += contrib;
if let Some(details) = &mut entry.explain {
@@ -5799,6 +6038,8 @@ impl ReaderPool {
snippet: n.hit.snippet.clone(),
score: 0.0,
explain: explain.then(SearchExplain::default),
+ fts_rank: None,
+ observation: None,
});
entry.score += contrib;
if let Some(details) = &mut entry.explain {
@@ -5817,15 +6058,50 @@ impl ReaderPool {
}
}
+ for (rank, vote) in observation_votes.into_iter().enumerate() {
+ let contrib = 1.0 / (k + (rank + 1) as f64);
+ let entry = fused.entry(vote.page_id).or_insert_with(|| Fused {
+ path: vote.path,
+ title: String::new(),
+ snippet: String::new(),
+ score: 0.0,
+ explain: explain.then(SearchExplain::default),
+ fts_rank: None,
+ observation: None,
+ });
+ entry.score += contrib;
+ if entry.title.is_empty() {
+ entry.title = vote.title;
+ }
+ entry.observation = Some((rank + 1, vote.snippet));
+ if let Some(details) = &mut entry.explain {
+ details.observation_rank = Some(rank + 1);
+ details.observation_ids = vote.observation_ids;
+ details.rrf.observation = contrib;
+ details.fused += contrib;
+ }
+ }
+
let mut out: Vec<(PageHit, Option<SearchExplain>)> = fused
.into_iter()
.map(|(id, entry)| {
+ // The matching observation's excerpt explains the hit when
+ // the observation stream ranked the page ahead of FTS (or FTS
+ // missed it); otherwise the page keeps its own snippet.
+ let snippet = match entry.observation {
+ Some((observation_rank, observation_snippet))
+ if entry.fts_rank.is_none_or(|fts| observation_rank < fts) =>
+ {
+ observation_snippet
+ }
+ _ => entry.snippet,
+ };
(
PageHit {
id,
path: entry.path,
title: entry.title,
- snippet: entry.snippet,
+ snippet,
rank: -entry.score, // lower = better (matches FTS5 convention)
superseded: false,
pinned: false,
diff --git a/crates/ai-memory-store/src/retrieval_tuning.rs b/crates/ai-memory-store/src/retrieval_tuning.rs
index fcc8dae7..b0d6871c 100644
--- a/crates/ai-memory-store/src/retrieval_tuning.rs
+++ b/crates/ai-memory-store/src/retrieval_tuning.rs
@@ -1,6 +1,6 @@
//! Opt-in post-fusion ranking signals for `hybrid_search`.
//!
-//! Two bounded, independently switchable signals, both off by default so a
+//! Bounded, independently switchable signals, all off by default so a
//! store that never configures them ranks exactly as before:
//!
//! * **Session-recall routing** — a zero-LLM lexical check that recognises
@@ -19,12 +19,19 @@
//! a precise second vote per page. Contributes nothing while the table
//! is empty.
//!
-//! Both signals are reported in `SearchExplain`, so an `explain=true`
+//! * **Observation stream** — a further RRF stream over `observations_fts`.
+//! The zero-LLM session page keeps only a short excerpt of each turn, so
+//! the evidence a question needs often lives only in the session's
+//! sanitized observation bodies. Each session's best-ranked matching
+//! observation votes once for that session's latest page; a session with
+//! no page contributes nothing (the raw fallback still covers it).
+//!
+//! Every signal is reported in `SearchExplain`, so an `explain=true`
//! query can account for the returned rank.
use serde::{Deserialize, Serialize};
-/// Tunables for the opt-in ranking signals. `Default` disables both.
+/// Tunables for the opt-in ranking signals. `Default` disables all of them.
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct RetrievalTuning {
@@ -49,6 +56,12 @@ pub struct RetrievalTuning {
/// default on without a positive R2 delta (retrieval-triple / QA), not yet
/// performed.
pub belief_authority_weight: f64,
+ /// Add the observation-evidence stream to the RRF fusion (R1): search
+ /// `observations_fts` in the same project and let each matching
+ /// observation vote, once per session, for its session's latest
+ /// `sessions/<id>.md` page. Nothing is written. Ships OFF: it changes
+ /// ranking, and the maintainer decides default-on from measurements.
+ pub observation_stream: bool,
}
impl Default for RetrievalTuning {
@@ -66,6 +79,7 @@ impl Default for RetrievalTuning {
// OFF by default: folding belief confidence into ranking is
// R2-gated (see the field doc).
belief_authority_weight: 0.0,
+ observation_stream: false,
}
}
}
@@ -171,6 +185,7 @@ mod tests {
assert!(!t.session_recall_routing);
assert!(!t.abstract_vectors);
assert_eq!(t.belief_authority_weight, 0.0);
+ assert!(!t.observation_stream);
}
#[test]
diff --git a/crates/ai-memory-store/tests/suite/mod.rs b/crates/ai-memory-store/tests/suite/mod.rs
index 8c01a37e..bb392812 100644
--- a/crates/ai-memory-store/tests/suite/mod.rs
+++ b/crates/ai-memory-store/tests/suite/mod.rs
@@ -21,6 +21,7 @@ mod profile;
mod project_authz;
mod reclaim_ledger_versions;
mod related_walk;
+mod retrieval_observation_stream;
mod retrieval_superseded;
mod retrieval_tuning_streams;
mod session_ids_touching_scope;
diff --git a/crates/ai-memory-store/tests/suite/multi_session.rs b/crates/ai-memory-store/tests/suite/multi_session.rs
index 25d9e161..fb65be94 100644
--- a/crates/ai-memory-store/tests/suite/multi_session.rs
+++ b/crates/ai-memory-store/tests/suite/multi_session.rs
@@ -3261,3 +3261,158 @@ async fn two_harnesses_of_one_operator_share_one_private_profile() {
assert!(read.is_ok(), "an open project stays shared");
}
}
+
+/// R1 observation stream, per-project isolation (row 1). The stream maps an
+/// observation to `sessions/<id>.md` by session id, so the two lookups it
+/// joins are each a place a foreign scope could leak in: an observation from
+/// another project voting for this project's page, or this project's
+/// observation voting for a same-path page in another project. Both must be
+/// filtered to the queried `(workspace_id, project_id)`.
+#[tokio::test]
+async fn observation_stream_never_crosses_a_project_or_workspace() {
+ use super::retrieval_observation_stream::{observe, search, session, session_page, stream_on};
+
+ let tmp = tempfile::tempdir().unwrap();
+ let mut store = Store::open(tmp.path()).unwrap();
+ store.reader.set_retrieval_tuning(stream_on());
+ let (ws, home) = scope(&store).await;
+ let foreign = store
+ .writer
+ .get_or_create_project(ws, "foreign-app".to_string(), None)
+ .await
+ .unwrap();
+ let other_ws = store
+ .writer
+ .get_or_create_workspace("other-ws".to_string())
+ .await
+ .unwrap();
+ let twin = store
+ .writer
+ .get_or_create_project(other_ws, "shared-app".to_string(), None)
+ .await
+ .unwrap();
+
+ // A home session whose evidence lives only in its observation. Pages at
+ // its session path exist in the foreign project and the cross-workspace
+ // twin, but not (yet) at home.
+ let sid = session(&store, ws, home).await;
+ // The foreign and twin scopes write before and after it, so home's
+ // observation sits inside their slice of the shared log and only the
+ // scope predicates can keep it out.
+ let foreign_sid = session(&store, ws, foreign).await;
+ let twin_sid = session(&store, other_ws, twin).await;
+ observe(&store, ws, foreign, foreign_sid, "foreign chatter before").await;
+ observe(&store, other_ws, twin, twin_sid, "twin chatter before").await;
+ let evidence = observe(&store, ws, home, sid, "the lynx password rotation plan").await;
+ observe(&store, ws, foreign, foreign_sid, "foreign chatter after").await;
+ observe(&store, other_ws, twin, twin_sid, "twin chatter after").await;
+ session_page(&store, ws, foreign, sid, "foreign copy").await;
+ session_page(&store, other_ws, twin, sid, "twin copy").await;
+
+ // Home's observation must not vote for a page outside home...
+ assert!(
+ search(&store, ws, home, "lynx").await.is_empty(),
+ "a home observation lifted a foreign page into home results"
+ );
+ // ...and the foreign scopes must not be lifted by home's observation.
+ assert!(
+ search(&store, ws, foreign, "lynx").await.is_empty(),
+ "another project's observation lifted this project's page"
+ );
+ assert!(
+ search(&store, other_ws, twin, "lynx").await.is_empty(),
+ "another workspace's observation lifted this workspace's page"
+ );
+
+ // A foreign session's own observation votes for its own page there, and
+ // never surfaces at home.
+ session_page(&store, ws, foreign, foreign_sid, "foreign session").await;
+ observe(&store, ws, foreign, foreign_sid, "ibex budget numbers").await;
+ assert!(search(&store, ws, home, "ibex").await.is_empty());
+ assert_eq!(search(&store, ws, foreign, "ibex").await.len(), 1);
+
+ // Control: once home has its own session page, home's observation lifts
+ // exactly that page.
+ session_page(&store, ws, home, sid, "home copy").await;
+ let hits = search(&store, ws, home, "lynx").await;
+ assert_eq!(hits.len(), 1);
+ assert_eq!(hits[0].0.path.as_str(), format!("sessions/{sid}.md"));
+ assert_eq!(hits[0].0.title, format!("sessions/{sid}.md"));
+ assert_eq!(hits[0].1.observation_ids, vec![evidence]);
+ let home_ids = store
+ .reader
+ .latest_page_ids_by_paths(ws, home, vec![format!("sessions/{sid}.md")])
+ .await
+ .unwrap();
+ assert_eq!(vec![hits[0].0.id], home_ids, "the vote went to home's page");
+}
+
+/// R1 observation stream, pages shared (row 5a, invariant #16). Operator B's
+/// session, page and observations in a shared project are evidence for
+/// operator A's query: neither the session owner nor `author_id` becomes a
+/// read filter on the stream.
+#[tokio::test]
+async fn another_operators_observations_lift_their_session_page_in_a_shared_project() {
+ use super::retrieval_observation_stream::{observe, search, stream_on};
+
+ let tmp = tempfile::tempdir().unwrap();
+ let mut store = Store::open(tmp.path()).unwrap();
+ store.reader.set_retrieval_tuning(stream_on());
+ let (ws, proj) = scope(&store).await;
+ let operator_b = store
+ .writer
+ .create_human_user(
+ NewUser {
+ username: "operator-b".into(),
+ name: None,
+ email: None,
+ },
+ UserRole::User,
+ None,
+ false,
+ )
+ .await
+ .unwrap();
+ let sid = SessionId::new();
+ store
+ .writer
+ .begin_session(NewSession {
+ occurred_at: None,
+ id: sid,
+ workspace_id: ws,
+ project_id: proj,
+ agent_kind: AgentKind::Codex,
+ cwd: Some("/repo".into()),
+ actor_user: Some(operator("operator-b")),
+ })
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(NewPage {
+ author_id: Some(operator_b),
+ ..page(
+ ws,
+ proj,
+ &format!("sessions/{sid}.md"),
+ "B's session",
+ "Operator B worked on the build.",
+ )
+ })
+ .await
+ .unwrap();
+ let evidence = observe(
+ &store,
+ ws,
+ proj,
+ sid,
+ "the walrus migration needs a backfill",
+ )
+ .await;
+
+ // Operator A's query carries no owner coordinate: hybrid search has none.
+ let hits = search(&store, ws, proj, "walrus backfill").await;
+ assert_eq!(hits.len(), 1, "B's session evidence must be visible to A");
+ assert_eq!(hits[0].0.path.as_str(), format!("sessions/{sid}.md"));
+ assert_eq!(hits[0].1.observation_ids, vec![evidence]);
+}
diff --git a/crates/ai-memory-store/tests/suite/retrieval_observation_stream.rs b/crates/ai-memory-store/tests/suite/retrieval_observation_stream.rs
new file mode 100644
index 00000000..3f66cdda
--- /dev/null
+++ b/crates/ai-memory-store/tests/suite/retrieval_observation_stream.rs
@@ -0,0 +1,383 @@
+//! `[retrieval] observation_stream` (R1): matching observation bodies vote,
+//! once per session, for that session's latest `sessions/<id>.md` page as one
+//! more RRF stream. Off by default and byte-identical when off.
+
+use ai_memory_core::{
+ AgentKind, NewObservation, NewPage, NewSession, ObservationId, ObservationKind, PagePath,
+ ProjectId, Sanitized, Sanitizer, SessionId, Tier, WorkspaceId,
+};
+use ai_memory_store::{PageHit, RetrievalTuning, SearchExplain, Store};
+
+pub(crate) fn stream_on() -> RetrievalTuning {
+ RetrievalTuning {
+ observation_stream: true,
+ ..RetrievalTuning::default()
+ }
+}
+
+pub(crate) async fn scope(store: &Store, ws: &str, proj: &str) -> (WorkspaceId, ProjectId) {
+ let ws = store
+ .writer
+ .get_or_create_workspace(ws.to_string())
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, proj.to_string(), None)
+ .await
+ .unwrap();
+ (ws, proj)
+}
+
+pub(crate) async fn session(store: &Store, ws: WorkspaceId, proj: ProjectId) -> SessionId {
+ let id = SessionId::new();
+ store
+ .writer
+ .begin_session(NewSession {
+ occurred_at: None,
+ id,
+ workspace_id: ws,
+ project_id: proj,
+ agent_kind: AgentKind::ClaudeCode,
+ cwd: Some("/repo".into()),
+ actor_user: None,
+ })
+ .await
+ .unwrap();
+ id
+}
+
+pub(crate) async fn observe(
+ store: &Store,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ session_id: SessionId,
+ body: &str,
+) -> ObservationId {
+ store
+ .writer
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ occurred_at: None,
+ session_id,
+ workspace_id: ws,
+ project_id: proj,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: body.into(),
+ importance: 5,
+ },
+ &Sanitizer::builtin(),
+ ))
+ .await
+ .unwrap()
+}
+
+pub(crate) fn page(ws: WorkspaceId, proj: ProjectId, path: &str, body: &str) -> NewPage {
+ NewPage {
+ workspace_id: ws,
+ project_id: proj,
+ path: PagePath::new(path).unwrap(),
+ title: path.into(),
+ body: body.into(),
+ tier: Tier::Episodic,
+ frontmatter_json: serde_json::json!({}),
+ pinned: false,
+ links: Vec::new(),
+ author_id: None,
+ expires_at: None,
+ entities: Vec::new(),
+ evidence: Vec::new(),
+ }
+}
+
+pub(crate) async fn session_page(
+ store: &Store,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ session_id: SessionId,
+ body: &str,
+) {
+ store
+ .writer
+ .upsert_page(page(ws, proj, &format!("sessions/{session_id}.md"), body))
+ .await
+ .unwrap();
+}
+
+pub(crate) async fn search(
+ store: &Store,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ query: &str,
+) -> Vec<(PageHit, SearchExplain)> {
+ store
+ .reader
+ .hybrid_search_explained(
+ ws,
+ proj,
+ query.to_string(),
+ None,
+ String::new(),
+ String::new(),
+ 0,
+ 10,
+ None,
+ false,
+ )
+ .await
+ .unwrap()
+}
+
+fn paths(hits: &[(PageHit, SearchExplain)]) -> Vec<String> {
+ hits.iter()
+ .map(|(h, _)| h.path.as_str().to_string())
+ .collect()
+}
+
+const K: f64 = 60.0;
+
+#[tokio::test]
+async fn an_observation_hit_lifts_its_own_session_page() {
+ let tmp = tempfile::tempdir().unwrap();
+ let mut store = Store::open(tmp.path()).unwrap();
+ let (ws, proj) = scope(&store, "default", "app").await;
+ let target = session(&store, ws, proj).await;
+ let other = session(&store, ws, proj).await;
+ // The session pages keep only a short excerpt: neither mentions the
+ // evidence words.
+ session_page(&store, ws, proj, target, "User asked about weekend plans.").await;
+ session_page(&store, ws, proj, other, "User asked about weekend plans.").await;
+ let evidence = observe(
+ &store,
+ ws,
+ proj,
+ target,
+ "I finally bought the quasar telescope from the camera shop downtown",
+ )
+ .await;
+ observe(&store, ws, proj, other, "we talked about gardening instead").await;
+
+ // Off: the evidence is unreachable from pages.
+ assert!(
+ search(&store, ws, proj, "quasar telescope")
+ .await
+ .is_empty()
+ );
+
+ store.reader.set_retrieval_tuning(stream_on());
+ let hits = search(&store, ws, proj, "quasar telescope").await;
+ let target_path = format!("sessions/{target}.md");
+ assert_eq!(paths(&hits), vec![target_path], "only the right session");
+ let (hit, details) = &hits[0];
+ assert_eq!(details.observation_rank, Some(1));
+ assert_eq!(details.observation_ids, vec![evidence]);
+ assert!((details.rrf.observation - 1.0 / (K + 1.0)).abs() < 1e-12);
+ assert!((details.fused - details.rrf.observation).abs() < 1e-12);
+ // The page had no snippet of its own for these words, so the bounded,
+ // already-sanitized observation excerpt explains the hit.
+ assert!(hit.snippet.contains("<mark>"), "{}", hit.snippet);
+ assert!(hit.snippet.contains("quasar"), "{}", hit.snippet);
+ assert!(!hit.title.is_empty());
+}
+
+#[tokio::test]
+async fn a_session_counts_once_however_many_observations_match() {
+ let tmp = tempfile::tempdir().unwrap();
+ let mut store = Store::open(tmp.path()).unwrap();
+ store.reader.set_retrieval_tuning(stream_on());
+ let (ws, proj) = scope(&store, "default", "app").await;
+ let chatty = session(&store, ws, proj).await;
+ let quiet = session(&store, ws, proj).await;
+ session_page(&store, ws, proj, chatty, "chatty session").await;
+ session_page(&store, ws, proj, quiet, "quiet session").await;
+ let mut chatty_ids = Vec::new();
+ for i in 0..6 {
+ chatty_ids.push(
+ observe(
+ &store,
+ ws,
+ proj,
+ chatty,
+ &format!("marmoset mention number {i} with filler words"),
+ )
+ .await,
+ );
+ }
+ observe(&store, ws, proj, quiet, "marmoset marmoset marmoset").await;
+
+ let hits = search(&store, ws, proj, "marmoset").await;
+ assert_eq!(hits.len(), 2, "{:?}", paths(&hits));
+ let chatty_path = format!("sessions/{chatty}.md");
+ let (_, chatty_details) = hits
+ .iter()
+ .find(|(h, _)| h.path.as_str() == chatty_path)
+ .unwrap();
+ let rank = chatty_details.observation_rank.unwrap();
+ // One contribution, at the session's own stream rank — six matching
+ // observations do not add up to six votes.
+ assert!(rank <= 2);
+ assert!((chatty_details.rrf.observation - 1.0 / (K + rank as f64)).abs() < 1e-12);
+ assert!((chatty_details.fused - chatty_details.rrf.observation).abs() < 1e-12);
+ // The matched ids are bounded and all belong to the chatty session.
+ assert!(!chatty_details.observation_ids.is_empty());
+ assert!(chatty_details.observation_ids.len() <= 3);
+ assert!(
+ chatty_details
+ .observation_ids
+ .iter()
+ .all(|id| chatty_ids.contains(id))
+ );
+ // Stream ranks are distinct per page: the stream ranks sessions, not
+ // observations.
+ let ranks: Vec<usize> = hits
+ .iter()
+ .map(|(_, d)| d.observation_rank.unwrap())
+ .collect();
+ let mut sorted = ranks.clone();
+ sorted.sort_unstable();
+ assert_eq!(sorted, vec![1, 2]);
+}
+
+#[tokio::test]
+async fn a_session_without_a_page_contributes_nothing_and_the_raw_fallback_still_sees_it() {
+ let tmp = tempfile::tempdir().unwrap();
+ let mut store = Store::open(tmp.path()).unwrap();
+ store.reader.set_retrieval_tuning(stream_on());
+ let (ws, proj) = scope(&store, "default", "app").await;
+ let live = session(&store, ws, proj).await;
+ observe(
+ &store,
+ ws,
+ proj,
+ live,
+ "pangolin sighting in the live session",
+ )
+ .await;
+ // A page at a different session's path, and an unrelated note, must not
+ // absorb the vote.
+ let unrelated = session(&store, ws, proj).await;
+ session_page(&store, ws, proj, unrelated, "nothing relevant").await;
+ store
+ .writer
+ .upsert_page(page(ws, proj, "notes/plain.md", "plain note"))
+ .await
+ .unwrap();
+
+ assert!(search(&store, ws, proj, "pangolin").await.is_empty());
+ let raw = store
+ .reader
+ .search_observations_for_project(ws, proj, "pangolin".into(), 5)
+ .await
+ .unwrap();
+ assert_eq!(raw.len(), 1);
+ assert_eq!(raw[0].session_id, live);
+}
+
+#[tokio::test]
+async fn a_superseded_or_expired_session_page_takes_no_vote() {
+ let tmp = tempfile::tempdir().unwrap();
+ let mut store = Store::open(tmp.path()).unwrap();
+ store.reader.set_retrieval_tuning(stream_on());
+ let (ws, proj) = scope(&store, "default", "app").await;
+ let expired = session(&store, ws, proj).await;
+ let mut expiring = page(ws, proj, &format!("sessions/{expired}.md"), "old session");
+ expiring.expires_at = Some(jiff::Timestamp::from_second(1_000).unwrap());
+ store.writer.upsert_page(expiring).await.unwrap();
+ observe(&store, ws, proj, expired, "ocelot evidence").await;
+ assert!(search(&store, ws, proj, "ocelot").await.is_empty());
+
+ // A rewritten session page: only the latest version takes the vote.
+ let rewritten = session(&store, ws, proj).await;
+ session_page(&store, ws, proj, rewritten, "first draft").await;
+ session_page(&store, ws, proj, rewritten, "second draft").await;
+ observe(&store, ws, proj, rewritten, "axolotl evidence").await;
+ let hits = search(&store, ws, proj, "axolotl").await;
+ assert_eq!(hits.len(), 1);
+ let latest = store
+ .reader
+ .latest_page_ids_by_paths(ws, proj, vec![format!("sessions/{rewritten}.md")])
+ .await
+ .unwrap();
+ assert_eq!(vec![hits[0].0.id], latest);
+}
+
+/// Stream off must rank exactly as a store that never heard of the stream:
+/// the observations present must not move a single hit, rank, or snippet,
+/// and the explain JSON must carry no observation keys.
+#[tokio::test]
+async fn stream_off_ranks_byte_identically_to_a_store_without_observations() {
+ async fn seeded(with_observations: bool) -> (tempfile::TempDir, Store, WorkspaceId, ProjectId) {
+ let tmp = tempfile::tempdir().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (ws, proj) = scope(&store, "default", "app").await;
+ for (i, body) in [
+ "kestrel nest under the bridge",
+ "the kestrel came back",
+ "nothing here",
+ ]
+ .iter()
+ .enumerate()
+ {
+ store
+ .writer
+ .upsert_page(page(ws, proj, &format!("notes/n{i}.md"), body))
+ .await
+ .unwrap();
+ }
+ // Deterministic session ids so both stores share page paths.
+ for i in 0..3_u128 {
+ let id: SessionId = format!("00000000-0000-7000-8000-{:012x}", 0x1000 + i)
+ .parse()
+ .unwrap();
+ store
+ .writer
+ .begin_session(NewSession {
+ occurred_at: None,
+ id,
+ workspace_id: ws,
+ project_id: proj,
+ agent_kind: AgentKind::ClaudeCode,
+ cwd: None,
+ actor_user: None,
+ })
+ .await
+ .unwrap();
+ session_page(&store, ws, proj, id, "session about birds").await;
+ if with_observations {
+ observe(&store, ws, proj, id, "a kestrel hovered over the field").await;
+ }
+ }
+ (tmp, store, ws, proj)
+ }
+ let (_a, plain, pws, pproj) = seeded(false).await;
+ let (_b, observed, ows, oproj) = seeded(true).await;
+ for query in ["kestrel", "birds", "kestrel birds"] {
+ let expected = search(&plain, pws, pproj, query).await;
+ let actual = search(&observed, ows, oproj, query).await;
+ let shape = |hits: &[(PageHit, SearchExplain)]| {
+ hits.iter()
+ .map(|(h, d)| {
+ let mut json = serde_json::to_value(d).unwrap();
+ // Ids differ between the two stores; everything else
+ // must match.
+ json.as_object_mut().unwrap().remove("graph_via");
+ (
+ h.path.as_str().to_string(),
+ h.title.clone(),
+ h.snippet.clone(),
+ h.rank.to_bits(),
+ json,
+ )
+ })
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(shape(&expected), shape(&actual), "query {query}");
+ for (_, details) in &actual {
+ let json = serde_json::to_string(details).unwrap();
+ assert!(!json.contains("observation"), "{json}");
+ }
+ }
+}
diff --git a/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs b/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs
index c16563a4..27e08b9a 100644
--- a/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs
+++ b/crates/ai-memory-store/tests/suite/retrieval_tuning_streams.rs
@@ -36,6 +36,7 @@ fn tuning(session_recall: bool, bonus: f64, abstract_vectors: bool) -> Retrieval
session_recall_bonus: bonus,
abstract_vectors,
belief_authority_weight: 0.0,
+ observation_stream: false,
}
}