Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(consolidate): drop the merged-away page's summary from its merge stub
Rudy Mizrahi Celekli92729e058046
diff --git a/crates/ai-memory-consolidate/src/dream.rs b/crates/ai-memory-consolidate/src/dream.rsindex bb668386..6e9ac7c7 100644--- a/crates/ai-memory-consolidate/src/dream.rs+++ b/crates/ai-memory-consolidate/src/dream.rs@@ -705,6 +705,17 @@ async fn merge_one_cluster(survivor.path.as_str());let mut stub_fm = frontmatter_object(&loser_md.frontmatter);+ // The stub's body is now a two-line merge note, so the loser's own+ // `summary:` would describe content that moved to the survivor:+ // `page_descriptor_expr` prefers a non-blank frontmatter summary+ // over the body on every non-FTS retrieval path, and+ // `abstract_bearing_page_ids` would keep re-embedding the abstract.+ // Removing both rather than rewriting `summary` to the merge note:+ // the descriptor then falls back to the body prefix, which already+ // IS that note. The pre-merge frontmatter stays reachable through+ // the supersession chain and git.+ stub_fm.remove("summary");+ stub_fm.remove("abstract");stub_fm.insert("compacted".to_string(), serde_json::Value::Bool(true));stub_fm.insert("merged_into".to_string(),diff --git a/crates/ai-memory-consolidate/src/sweep.rs b/crates/ai-memory-consolidate/src/sweep.rsindex 622332ba..4d07e273 100644--- a/crates/ai-memory-consolidate/src/sweep.rs+++ b/crates/ai-memory-consolidate/src/sweep.rs@@ -1374,6 +1374,17 @@ async fn plan_cold_cluster_dedup(MERGE_STUB_FOOTER);let mut stub_fm = frontmatter_object(&md.frontmatter);+ // The stub's body is now a two-line merge note, so the loser's own+ // `summary:` would describe content that moved to the survivor:+ // `page_descriptor_expr` prefers a non-blank frontmatter summary+ // over the body on every non-FTS retrieval path, and+ // `abstract_bearing_page_ids` would keep re-embedding the abstract.+ // Removing both rather than rewriting `summary` to the merge note:+ // the descriptor then falls back to the body prefix, which already+ // IS that note. The pre-merge frontmatter stays reachable through+ // the supersession chain and git.+ stub_fm.remove("summary");+ stub_fm.remove("abstract");stub_fm.insert("compacted".to_string(), serde_json::Value::Bool(true));stub_fm.insert("merged_into".to_string(),diff --git a/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs b/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rsindex fb877469..f9703a42 100644--- a/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs+++ b/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs@@ -86,6 +86,86 @@ async fn write(fx: &Fixture, path: &str, body: &str) -> PageId {.unwrap()}+/// `write`, plus a frontmatter `summary:` and `abstract:` — the shape a page+/// has when something upstream (or an operator) gave it a descriptor.+async fn write_with_summary(fx: &Fixture, path: &str, body: &str, summary: &str) -> PageId {+ fx.wiki+ .write_page(WritePageRequest {+ workspace_id: fx.ws,+ project_id: fx.proj,+ path: PagePath::new(path).unwrap(),+ frontmatter: serde_json::json!({+ "title": path,+ "summary": summary,+ "abstract": summary,+ }),+ body: body.to_string(),+ tier: Tier::Episodic,+ pinned: false,+ title: Some(path.to_string()),+ admission_ctx: None,+ author_id: None,+ actor: ActorContext::anonymous(),+ evidence: Vec::new(),+ })+ .await+ .unwrap()+}++fn frontmatter_map(raw: &str) -> serde_json::Map<String, serde_json::Value> {+ match serde_json::from_str(raw).unwrap() {+ serde_json::Value::Object(m) => m,+ other => panic!("frontmatter_json is not an object: {other}"),+ }+}++fn latest_frontmatter(fx: &Fixture, path: &str) -> serde_json::Map<String, serde_json::Value> {+ let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();+ let raw: String = conn+ .query_row(+ "SELECT frontmatter_json FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND path = ?3 AND is_latest = 1",+ params![fx.ws.as_bytes(), fx.proj.as_bytes(), path],+ |row| row.get(0),+ )+ .unwrap();+ frontmatter_map(&raw)+}++fn superseded_frontmatter(+ fx: &Fixture,+ path: &str,+) -> Vec<serde_json::Map<String, serde_json::Value>> {+ let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();+ let mut stmt = conn+ .prepare(+ "SELECT frontmatter_json FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND path = ?3 AND is_latest = 0 \+ ORDER BY created_at",+ )+ .unwrap();+ let rows = stmt+ .query_map(params![fx.ws.as_bytes(), fx.proj.as_bytes(), path], |row| {+ row.get::<_, String>(0)+ })+ .unwrap();+ rows.map(|r| frontmatter_map(&r.unwrap())).collect()+}++/// What a non-FTS retrieval path says about a page: the recency stream's+/// descriptor, built by `page_descriptor_expr` like every other such stream.+async fn project_descriptor(fx: &Fixture, path: &str) -> String {+ fx.store+ .reader+ .recent_pages_for_project(fx.ws, fx.proj, 20)+ .await+ .unwrap()+ .into_iter()+ .find(|hit| hit.path.as_str() == path)+ .unwrap_or_else(|| panic!("{path} is not in the recency stream"))+ .snippet+}+fn backdate_latest(fx: &Fixture, path: &str, days: i64) {let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();conn.pragma_update(None, "busy_timeout", 5_000).unwrap();@@ -369,3 +449,85 @@ async fn no_embeddings_is_a_clean_no_op() {);assert_eq!(report.evicted.len(), 2, "cold pages follow the normal path");}++/// The summary a merged-away page carried described the body that moved to the+/// survivor. `page_descriptor_expr` prefers a non-blank frontmatter `summary:`+/// over the body on every non-FTS retrieval path, so leaving it on the stub+/// makes the brief and the recency / vector / entity / graph hit lists promise+/// content a reader will not find when they open the page.+#[tokio::test]+async fn merge_stub_drops_the_loser_summary_and_abstract() {+ let fx = seed_fixture(true).await;+ for path in ["sessions/a.md", "sessions/b.md"] {+ write_with_summary(+ &fx,+ path,+ if path.ends_with("a.md") { DUP_A } else { DUP_B },+ "Fixed the flaky sweep test by guarding the upsert in ops.rs.",+ )+ .await;+ }+ write(&fx, "sessions/far.md", FAR).await;+ for p in ["sessions/a.md", "sessions/b.md", "sessions/far.md"] {+ backdate_latest(&fx, p, COLD_DAYS);+ }++ let report = run_sweep_with_hygiene(+ &fx.store.reader,+ &fx.store.writer,+ Some(&fx.wiki),+ fx.ws,+ fx.proj,+ &DecayParams::default(),+ 0.0,+ ObservationRetention::default(),+ false,+ dedup_on(),+ false,+ )+ .await+ .unwrap();+ assert_eq!(report.merged.len(), 1, "one cluster: {:?}", report.merged);+ let cluster = &report.merged[0];+ assert!(cluster.applied, "the collapse landed");+ let loser_path = cluster.merged[0].clone();++ let fm = latest_frontmatter(&fx, &loser_path);+ assert!(+ !fm.contains_key("summary"),+ "the stub carries no summary of content that moved: {fm:?}"+ );+ assert!(+ !fm.contains_key("abstract"),+ "and no abstract, so it is not re-embedded into the abstract stream: {fm:?}"+ );+ // Control: the merge markers are still written, so the removal above did+ // not simply discard the stub's whole frontmatter.+ assert_eq!(fm.get("compacted"), Some(&serde_json::Value::Bool(true)));+ assert_eq!(+ fm.get("merged_into").and_then(serde_json::Value::as_str),+ Some(cluster.survivor.as_str())+ );++ // Control: the pre-merge summary is still reachable through supersession,+ // so the test cannot pass because the summary was never written at all.+ let history = superseded_frontmatter(&fx, &loser_path);+ assert!(+ history.iter().any(|fm| fm+ .get("summary")+ .and_then(serde_json::Value::as_str)+ .is_some_and(|s| s.contains("guarding the upsert"))),+ "the pre-merge frontmatter stays in the chain: {history:?}"+ );++ // What retrieval now says about the stub: the merge note, not the fix.+ let descriptor = project_descriptor(&fx, &loser_path).await;+ assert!(+ descriptor.contains("merged into"),+ "the stub describes itself as a merge note: {descriptor}"+ );+ assert!(+ !descriptor.contains("guarding the upsert"),+ "and no longer promises the fix that moved to the survivor: {descriptor}"+ );+}diff --git a/crates/ai-memory-consolidate/tests/suite/dream_pass.rs b/crates/ai-memory-consolidate/tests/suite/dream_pass.rsindex fe4aa6e8..fd3b6229 100644--- a/crates/ai-memory-consolidate/tests/suite/dream_pass.rs+++ b/crates/ai-memory-consolidate/tests/suite/dream_pass.rs@@ -181,6 +181,86 @@ async fn write(fx: &Fixture, path: &str, body: &str) -> PageId {.unwrap()}+/// `write`, plus a frontmatter `summary:` and `abstract:` — the shape a page+/// has when something upstream (or an operator) gave it a descriptor.+async fn write_with_summary(fx: &Fixture, path: &str, body: &str, summary: &str) -> PageId {+ fx.wiki+ .write_page(WritePageRequest {+ workspace_id: fx.ws,+ project_id: fx.proj,+ path: PagePath::new(path).unwrap(),+ frontmatter: serde_json::json!({+ "title": path,+ "summary": summary,+ "abstract": summary,+ }),+ body: body.to_string(),+ tier: Tier::Episodic,+ pinned: false,+ title: Some(path.to_string()),+ admission_ctx: None,+ author_id: None,+ actor: ActorContext::anonymous(),+ evidence: Vec::new(),+ })+ .await+ .unwrap()+}++fn frontmatter_map(raw: &str) -> serde_json::Map<String, serde_json::Value> {+ match serde_json::from_str(raw).unwrap() {+ serde_json::Value::Object(m) => m,+ other => panic!("frontmatter_json is not an object: {other}"),+ }+}++fn latest_frontmatter(fx: &Fixture, path: &str) -> serde_json::Map<String, serde_json::Value> {+ let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();+ let raw: String = conn+ .query_row(+ "SELECT frontmatter_json FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND path = ?3 AND is_latest = 1",+ params![fx.ws.as_bytes(), fx.proj.as_bytes(), path],+ |row| row.get(0),+ )+ .unwrap();+ frontmatter_map(&raw)+}++fn superseded_frontmatter(+ fx: &Fixture,+ path: &str,+) -> Vec<serde_json::Map<String, serde_json::Value>> {+ let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();+ let mut stmt = conn+ .prepare(+ "SELECT frontmatter_json FROM pages \+ WHERE workspace_id = ?1 AND project_id = ?2 AND path = ?3 AND is_latest = 0 \+ ORDER BY created_at",+ )+ .unwrap();+ let rows = stmt+ .query_map(params![fx.ws.as_bytes(), fx.proj.as_bytes(), path], |row| {+ row.get::<_, String>(0)+ })+ .unwrap();+ rows.map(|r| frontmatter_map(&r.unwrap())).collect()+}++/// What a non-FTS retrieval path says about a page: the recency stream's+/// descriptor, built by `page_descriptor_expr` like every other such stream.+async fn project_descriptor(fx: &Fixture, path: &str) -> String {+ fx.store+ .reader+ .recent_pages_for_project(fx.ws, fx.proj, 20)+ .await+ .unwrap()+ .into_iter()+ .find(|hit| hit.path.as_str() == path)+ .unwrap_or_else(|| panic!("{path} is not in the recency stream"))+ .snippet+}+fn backdate_latest(fx: &Fixture, path: &str, days: i64) {let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();conn.pragma_update(None, "busy_timeout", 5_000).unwrap();@@ -573,3 +653,89 @@ async fn cancel_stops_before_next_cluster() {"only the first cluster was written (survivor + 1 loser); the rest is untouched");}++/// The dream pass builds the same merge stub as A3 and had the same defect:+/// the merged-away page's `summary:` rode onto a page whose body is now a+/// two-line merge note. `page_descriptor_expr` prefers that summary over the+/// body on every non-FTS retrieval path, so the stub advertised a fix that had+/// moved to the survivor.+#[tokio::test]+async fn merge_stub_drops_the_loser_summary_and_abstract() {+ let fx = seed_fixture(true).await;+ write_with_summary(+ &fx,+ "sessions/a.md",+ DUP_A,+ "Fixed the flaky sweep test by guarding the upsert in ops.rs.",+ )+ .await;+ write_with_summary(+ &fx,+ "sessions/b.md",+ DUP_B,+ "Fixed the flaky sweep test by guarding the upsert in ops.rs.",+ )+ .await;+ write(&fx, "sessions/far.md", K8S_A).await;+ for p in ["sessions/a.md", "sessions/b.md", "sessions/far.md"] {+ backdate_latest(&fx, p, COLD_DAYS);+ }++ let llm = FakeMergeLlm::new();+ let report = run_dream_pass(+ &fx.store.reader,+ &fx.wiki,+ Some(&llm),+ fx.ws,+ fx.proj,+ &DecayParams::default(),+ 0.0,+ &dream_on(),+ &DreamCancel::new(),+ false,+ )+ .await+ .unwrap();+ assert_eq!(report.clusters_merged, 1, "the merge applied: {report:?}");+ let merge = &report.merges[0];+ let loser_path = merge.merged[0].clone();++ let fm = latest_frontmatter(&fx, &loser_path);+ assert!(+ !fm.contains_key("summary"),+ "the stub carries no summary of content that moved: {fm:?}"+ );+ assert!(+ !fm.contains_key("abstract"),+ "and no abstract, so it is not re-embedded into the abstract stream: {fm:?}"+ );+ // Control: the merge markers are still written, so the removal above did+ // not simply discard the stub's whole frontmatter.+ assert_eq!(fm.get("compacted"), Some(&serde_json::Value::Bool(true)));+ assert_eq!(+ fm.get("merged_into").and_then(serde_json::Value::as_str),+ Some(merge.survivor.as_str())+ );++ // Control: the pre-merge summary is still reachable through supersession,+ // so the test cannot pass because the summary was never written at all.+ let history = superseded_frontmatter(&fx, &loser_path);+ assert!(+ history.iter().any(|fm| fm+ .get("summary")+ .and_then(serde_json::Value::as_str)+ .is_some_and(|s| s.contains("guarding the upsert"))),+ "the pre-merge frontmatter stays in the chain: {history:?}"+ );++ // What retrieval now says about the stub: the merge note, not the fix.+ let descriptor = project_descriptor(&fx, &loser_path).await;+ assert!(+ descriptor.contains("merged into"),+ "the stub describes itself as a merge note: {descriptor}"+ );+ assert!(+ !descriptor.contains("guarding the upsert"),+ "and no longer promises the fix that moved to the survivor: {descriptor}"+ );+}