Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(store): describe the page, not its heading, on non-FTS hit descriptors
Samir Hanna Verzaad0a64d02ec6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6f86e6e2..1b227ea0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- Made non-FTS search hits describe the page instead of repeating its+ heading. Only the FTS5 path built a real excerpt (`snippet(pages_fts, ...)`,+ centred on the matched terms); the vector, entity-match, graph-neighbour and+ recency paths had no matched term to centre on and returned+ `substr(body, 1, 240)` — on a compiled page that is the `# Title` line plus+ the following structural heading, which duplicates the `title` field already+ present on the hit and gives an agent nothing to decide a follow-up+ `memory_read_page` on. Measured against a live instance over 100 real pages+ (55 of them substantive, the rest near-empty session stubs), 53% of+ substantive descriptors opened by repeating the title verbatim, and the+ median page spent its 240-character budget on 35 characters of prose. Those+ six queries now prefer the page's own frontmatter `summary`, and otherwise+ fill the budget from the page's own text, skipping structural lines,+ `- **key:** value` metadata bullets, and any line that merely repeats the+ title. On the same sample title repetition drops to 0%, and 71% of session+ pages land on the prompts after the first — the part the title does not+ already show. The FTS path is unchanged, response shape is unchanged, and no+ schema migration is involved.+## [1.31.0] - 2026-08-22### Fixeddiff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 7376989c..b5b3adb5 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -2466,6 +2466,128 @@ mod tests {assert_eq!(hits[0].path.as_str(), "hyphen.md");}+ #[tokio::test]+ async fn linked_neighbor_hit_describes_the_page_not_its_heading() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "ai-memory", None)+ .await+ .unwrap();++ // A compiled page opens with its title and a structural heading. The+ // graph-neighbour path has no matched term to centre an excerpt on,+ // so before the descriptor fix its snippet was exactly this preamble.+ let body = concat!(+ "# Session 2026-04-20\n",+ "\n",+ "## Decisions\n",+ "\n",+ "Dropped the retry queue because the writer actor already serialises.\n",+ );+ store+ .writer+ .upsert_page(sample_page(ws, proj, "target.md", body))+ .await+ .unwrap();+ let mut source = sample_page(ws, proj, "source.md", "needle source content");+ source.links = vec![PagePath::new("target.md").unwrap().into()];+ store.writer.upsert_page(source).await.unwrap();++ let hits = store+ .reader+ .hybrid_search(+ ws,+ proj,+ "needle".into(),+ None,+ String::new(),+ String::new(),+ 0,+ 10,+ None,+ )+ .await+ .unwrap();+ let neighbor = hits+ .iter()+ .find(|h| h.path.as_str() == "target.md")+ .expect("linked neighbor should be included");+ assert_eq!(+ neighbor.snippet,+ "Dropped the retry queue because the writer actor already serialises.",+ "neighbour snippet should describe the page, not repeat its heading"+ );+ }++ #[tokio::test]+ async fn frontmatter_summary_wins_over_the_body_lede() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "ai-memory", None)+ .await+ .unwrap();++ let mut page = sample_page(ws, proj, "curated.md", "Body prose that loses.");+ page.frontmatter_json = serde_json::json!({ "summary": "Curated one-liner." });+ store.writer.upsert_page(page).await.unwrap();++ let hits = store+ .reader+ .recent_pages_for_project(ws, proj, 10)+ .await+ .unwrap();+ let hit = hits+ .iter()+ .find(|h| h.path.as_str() == "curated.md")+ .expect("page should be listed");+ assert_eq!(hit.snippet, "Curated one-liner.");+ }++ #[tokio::test]+ async fn blank_frontmatter_summary_falls_back_to_the_body_lede() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "ai-memory", None)+ .await+ .unwrap();++ let mut page = sample_page(ws, proj, "blank.md", "Body prose that wins.");+ page.frontmatter_json = serde_json::json!({ "summary": " " });+ store.writer.upsert_page(page).await.unwrap();++ let hits = store+ .reader+ .recent_pages_for_project(ws, proj, 10)+ .await+ .unwrap();+ let hit = hits+ .iter()+ .find(|h| h.path.as_str() == "blank.md")+ .expect("page should be listed");+ assert_eq!(hit.snippet, "Body prose that wins.");+ }+#[tokio::test]async fn hybrid_search_includes_linked_neighbors() {let tmp = TempDir::new().unwrap();diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex e918f9aa..7152122f 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -76,6 +76,112 @@ fn page_kind_expr(path_column: &str, frontmatter_column: &str) -> String {)}+/// Leading body characters scanned when a page carries no frontmatter+/// `summary`. Wide enough to step past the `# Title` line and a structural+/// heading or two before the first real prose.+const DESCRIPTOR_SCAN_CHARS: usize = 600;++/// Maximum length of a synthesised page descriptor, in characters.+const DESCRIPTOR_MAX_CHARS: usize = 240;++/// SQL expression yielding the raw material for [`page_descriptor`]: the+/// page's own frontmatter `summary` when it has a non-blank one, otherwise a+/// prefix of the body. `NULLIF(TRIM(...), '')` keeps an empty summary from+/// winning the `COALESCE` over real body text.+fn page_descriptor_expr(body_column: &str, frontmatter_column: &str) -> String {+ format!(+ "COALESCE( \+ NULLIF(TRIM(json_extract({frontmatter_column}, '$.summary')), ''), \+ substr({body_column}, 1, {DESCRIPTOR_SCAN_CHARS}) \+ )"+ )+}++/// Strip a leading markdown list marker (`- `, `* `, `+ `, `1. `) so a kept+/// line reads as prose in the descriptor.+fn strip_list_marker(line: &str) -> &str {+ if let Some(rest) = line+ .strip_prefix("- ")+ .or_else(|| line.strip_prefix("* "))+ .or_else(|| line.strip_prefix("+ "))+ {+ return rest.trim_start();+ }+ let digits = line.chars().take_while(char::is_ascii_digit).count();+ if digits > 0 && line[digits..].starts_with(". ") {+ return line[digits + 2..].trim_start();+ }+ line+}++/// `true` for the `- **key:** value` bullets that open a compiled session+/// page's metadata block — session id, timestamps, observation count. They+/// are addressing, not content.+fn is_metadata_bullet(line: &str) -> bool {+ line.starts_with("- **") && line.contains(":**")+}++/// Page-intrinsic descriptor for a retrieval hit.+///+/// The FTS path centres its excerpt on the matched terms through+/// `snippet(pages_fts, ...)`. The vector, entity-match, graph-neighbour and+/// recency paths have no matched term to centre on. They previously returned+/// `substr(body, 1, 240)`, which on a compiled page is the `# Title` line plus+/// the `## Session metadata` block under it — the title the hit already+/// carries, followed by a session id and three timestamps.+///+/// Fills the [`DESCRIPTOR_MAX_CHARS`] budget from the page's own text,+/// skipping four things that carry no signal for the reader deciding whether+/// to open the page: structural lines, metadata bullets, and any line that+/// merely repeats `title`. On a session page that lands on the prompts after+/// the first, which is the part `title` does not already show.+fn page_descriptor(raw: &str, title: &str) -> String {+ let title_key = title.trim().trim_end_matches('\u{2026}').trim();+ let mut out = String::new();+ for line in raw.lines().map(str::trim) {+ if line.is_empty()+ || line.starts_with('#')+ || line.starts_with("---")+ || line.starts_with("___")+ || line.starts_with("***")+ || is_metadata_bullet(line)+ {+ continue;+ }+ let content = strip_list_marker(line).trim();+ if content.is_empty() {+ continue;+ }+ if !title_key.is_empty() && content.starts_with(title_key) {+ continue;+ }+ if !out.is_empty() {+ if out.chars().count() + 1 + content.chars().count() > DESCRIPTOR_MAX_CHARS {+ break;+ }+ out.push(' ');+ }+ out.push_str(content);+ if out.chars().count() >= DESCRIPTOR_MAX_CHARS {+ break;+ }+ }+ if out.is_empty() {+ return truncate_chars(raw.trim(), DESCRIPTOR_MAX_CHARS);+ }+ truncate_chars(&out, DESCRIPTOR_MAX_CHARS)+}++/// Truncate on a character boundary, marking elision with an ellipsis.+fn truncate_chars(text: &str, max: usize) -> String {+ if text.chars().count() <= max {+ return text.to_owned();+ }+ let mut out: String = text.chars().take(max.saturating_sub(1)).collect();+ out.push('\u{2026}');+ out+}+const AUTHORITY_CANDIDATE_MULTIPLIER: usize = 4;const AUTHORITY_MIN_CANDIDATES: usize = 20;const AUTHORITY_MAX_EXTRA_CANDIDATES: usize = 300;@@ -1586,12 +1692,13 @@ impl ReaderPool {self.with_conn(move |conn| {let sql = format!("SELECT id, path, title, \- substr(body, 1, 240) AS snip, \+ {descriptor} AS snip, \CAST(updated_at AS REAL) AS rank \FROM pages \WHERE is_latest = 1{not_expired} \ORDER BY updated_at DESC \LIMIT ?1",+ descriptor = page_descriptor_expr("body", "frontmatter_json"),not_expired = not_expired("pages", "?2"),);let mut stmt = conn.prepare_cached(&sql)?;@@ -1600,7 +1707,7 @@ impl ReaderPool {let id_bytes: Vec<u8> = row.get(0)?;let path: String = row.get(1)?;let title: String = row.get(2)?;- let snippet: String = row.get(3)?;+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);let rank: f64 = row.get(4)?;Ok((id_bytes, path, title, snippet, rank))})?;@@ -1633,12 +1740,13 @@ impl ReaderPool {self.with_conn(move |conn| {let sql = format!("SELECT id, path, title, \- substr(body, 1, 240) AS snip, \+ {descriptor} AS snip, \CAST(updated_at AS REAL) AS rank \FROM pages \WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 1{not_expired} \ORDER BY updated_at DESC \LIMIT ?3",+ descriptor = page_descriptor_expr("body", "frontmatter_json"),not_expired = not_expired("pages", "?4"),);let mut stmt = conn.prepare_cached(&sql)?;@@ -1654,7 +1762,7 @@ impl ReaderPool {let id_bytes: Vec<u8> = row.get(0)?;let path: String = row.get(1)?;let title: String = row.get(2)?;- let snippet: String = row.get(3)?;+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);let rank: f64 = row.get(4)?;Ok((id_bytes, path, title, snippet, rank))},@@ -1691,7 +1799,7 @@ impl ReaderPool {self.with_conn(move |conn| {let sql = format!("SELECT workspaces.name, projects.name, pages.path, pages.title, \- substr(pages.body, 1, 240) AS snip, \+ {descriptor} AS snip, \CAST(pages.updated_at AS REAL) AS rank \FROM pages \JOIN projects ON projects.id = pages.project_id \@@ -1699,6 +1807,7 @@ impl ReaderPool {WHERE pages.is_latest = 1{not_expired} \ORDER BY pages.updated_at DESC \LIMIT ?1",+ descriptor = page_descriptor_expr("pages.body", "pages.frontmatter_json"),not_expired = not_expired("pages", "?2"),);let mut stmt = conn.prepare_cached(&sql)?;@@ -1708,7 +1817,7 @@ impl ReaderPool {let project_name: String = row.get(1)?;let path: String = row.get(2)?;let title: String = row.get(3)?;- let snippet: String = row.get(4)?;+ let snippet = page_descriptor(&row.get::<_, String>(4)?, &title);let rank: f64 = row.get(5)?;Ok((workspace_name, project_name, path, title, snippet, rank))})?;@@ -3158,7 +3267,7 @@ impl ReaderPool {WHERE 1 = 1{freq_not_expired} \GROUP BY m.entity_id, m.name \) \- SELECT pg.id, pg.path, pg.title, substr(pg.body, 1, 240) AS snippet, \+ SELECT pg.id, pg.path, pg.title, {descriptor} AS snippet, \SUM(1.0 / f.pages) AS weight, \COUNT(*) AS matches, \json_group_array(f.name) AS names \@@ -3169,6 +3278,7 @@ impl ReaderPool {GROUP BY pg.id, pg.path, pg.title \ORDER BY weight DESC, matches DESC, pg.path ASC \LIMIT ?",+ descriptor = page_descriptor_expr("pg.body", "pg.frontmatter_json"),not_expired = not_expired("pg", "?"),freq_not_expired = not_expired("p", "?"),)@@ -3179,7 +3289,7 @@ impl ReaderPool {let id_bytes: Vec<u8> = row.get(0)?;let path: String = row.get(1)?;let title: String = row.get(2)?;- let snippet: String = row.get(3)?;+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);let weight: f64 = row.get(4)?;let names_json: String = row.get(6)?;Ok((id_bytes, path, title, snippet, weight, names_json))@@ -3331,6 +3441,8 @@ impl ReaderPool {sql_params.push(Value::Blob(project_id.as_bytes().to_vec()));sql_params.push(Value::Integer(now));+ let out_descriptor = page_descriptor_expr("tp.body", "tp.frontmatter_json");+ let in_descriptor = page_descriptor_expr("fp.body", "fp.frontmatter_json");let out_not_expired = not_expired("tp", "?");let in_not_expired = not_expired("fp", "?");let mut sql = String::with_capacity(values_clause.len() + 1_500);@@ -3339,7 +3451,7 @@ impl ReaderPool {"WITH seeds(seed_id, seed_ord) AS (VALUES {values_clause}), \neighbors AS ( \SELECT tp.id AS id, tp.path AS path, tp.title AS title, \- substr(tp.body, 1, 240) AS snippet, \+ {out_descriptor} AS snippet, \seeds.seed_ord * 2 AS stream_ord, tp.updated_at AS updated_at \FROM seeds \JOIN links l ON l.from_page_id = seeds.seed_id \@@ -3347,7 +3459,7 @@ impl ReaderPool {WHERE tp.workspace_id = ? AND tp.project_id = ? AND tp.is_latest = 1{out_not_expired} \UNION ALL \SELECT fp.id AS id, fp.path AS path, fp.title AS title, \- substr(fp.body, 1, 240) AS snippet, \+ {in_descriptor} AS snippet, \seeds.seed_ord * 2 + 1 AS stream_ord, fp.updated_at AS updated_at \FROM seeds \JOIN links l ON l.to_page_id = seeds.seed_id \@@ -3366,7 +3478,7 @@ impl ReaderPool {let id_bytes: Vec<u8> = row.get(0)?;let path: String = row.get(1)?;let title: String = row.get(2)?;- let snippet: String = row.get(3)?;+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);let stream_ord: i64 = row.get(4)?;Ok((id_bytes, path, title, snippet, stream_ord))})?;@@ -7598,7 +7710,10 @@ fn open_read_only(path: &Path) -> StoreResult<Connection> {#[cfg(test)]mod tests {- use super::{entity_query_tokens, handoff_listing_sql, like_escape};+ use super::{+ DESCRIPTOR_MAX_CHARS, entity_query_tokens, handoff_listing_sql, like_escape,+ page_descriptor, page_descriptor_expr,+ };use crate::Store;use ai_memory_core::{@@ -8293,4 +8408,97 @@ mod tests {assert_eq!(like_escape("back\\slash"), "back\\\\slash");assert_eq!(like_escape("plain-name"), "plain-name");}++ #[test]+ fn page_descriptor_skips_heading_and_rule_lines() {+ let body = concat!(+ "# Session 2026-04-20\n",+ "\n",+ "---\n",+ "\n",+ "## Summary\n",+ "\n",+ "Switched the scheduler to a single writer actor.\n",+ );+ assert_eq!(+ page_descriptor(body, ""),+ "Switched the scheduler to a single writer actor."+ );+ }++ #[test]+ fn page_descriptor_fills_the_budget_across_prose_lines() {+ // A short opening line must not cost the rest of the budget: the+ // `_lint` pages open with "N finding(s)." and everything that tells+ // you what the findings were comes after it.+ let body = concat!(+ "# Lint findings\n",+ "\n",+ "501 finding(s).\n",+ "\n",+ "## 1 - stale (info)\n",+ "\n",+ "Episodic page sessions/e24f6fbc.md is 30 days old with zero accesses.\n",+ );+ assert_eq!(+ page_descriptor(body, ""),+ "501 finding(s). Episodic page sessions/e24f6fbc.md is 30 days old \+ with zero accesses."+ );+ }++ #[test]+ fn page_descriptor_falls_back_to_raw_text_when_all_lines_are_structural() {+ assert_eq!(page_descriptor("# Only a title\n", ""), "# Only a title");+ assert_eq!(page_descriptor(" ", ""), "");+ }++ #[test]+ fn page_descriptor_skips_metadata_bullets_and_the_repeated_title() {+ // The compiled session-page shape: the title again as prompt 1, with+ // the metadata block in between. What the reader does not already+ // have is prompt 2 onward.+ let title = "vamos revisar o deploy";+ let body = concat!(+ "# vamos revisar o deploy\n",+ "\n",+ "## Session metadata\n",+ "\n",+ "- **session_id:** `4e79a7ce-55e9-44c1-bcb5-74a6fe740900`\n",+ "- **started_at:** 2026-08-22T19:26:07Z\n",+ "- **observations:** 59\n",+ "\n",+ "## Prompts\n",+ "\n",+ "1. vamos revisar o deploy\n",+ "2. o rollback falhou no passo do migrate\n",+ );+ assert_eq!(+ page_descriptor(body, title),+ "o rollback falhou no passo do migrate"+ );+ }++ #[test]+ fn page_descriptor_matches_a_title_truncated_with_an_ellipsis() {+ let title = "vamos revisar o deploy do SHVIA-WEB e o rollback\u{2026}";+ let body = "# x\n\n1. vamos revisar o deploy do SHVIA-WEB e o rollback do migrate\n2. depois o resto\n";+ assert_eq!(page_descriptor(body, title), "depois o resto");+ }++ #[test]+ fn page_descriptor_truncates_on_a_character_boundary() {+ let body = "\u{e7}".repeat(DESCRIPTOR_MAX_CHARS + 10);+ let out = page_descriptor(&body, "");+ assert_eq!(out.chars().count(), DESCRIPTOR_MAX_CHARS);+ assert!(out.ends_with('\u{2026}'));+ }++ #[test]+ fn page_descriptor_expr_prefers_a_non_blank_frontmatter_summary() {+ let sql = page_descriptor_expr("pages.body", "pages.frontmatter_json");+ assert!(sql.contains("json_extract(pages.frontmatter_json, '$.summary')"));+ assert!(sql.contains("NULLIF(TRIM("));+ assert!(sql.contains("substr(pages.body, 1, 600)"));+ }}