Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(store): describe the page, not its heading, on non-FTS hit descriptors

Samir Hanna Verzaad0a64d02ec6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f86e6e2..1b227ea0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+- Made non-FTS search hits describe the page instead of repeating its
+ heading. Only the FTS5 path built a real excerpt (`snippet(pages_fts, ...)`,
+ centred on the matched terms); the vector, entity-match, graph-neighbour and
+ recency paths had no matched term to centre on and returned
+ `substr(body, 1, 240)` — on a compiled page that is the `# Title` line plus
+ the following structural heading, which duplicates the `title` field already
+ present on the hit and gives an agent nothing to decide a follow-up
+ `memory_read_page` on. Measured against a live instance over 100 real pages
+ (55 of them substantive, the rest near-empty session stubs), 53% of
+ substantive descriptors opened by repeating the title verbatim, and the
+ median page spent its 240-character budget on 35 characters of prose. Those
+ six queries now prefer the page's own frontmatter `summary`, and otherwise
+ fill the budget from the page's own text, skipping structural lines,
+ `- **key:** value` metadata bullets, and any line that merely repeats the
+ title. On the same sample title repetition drops to 0%, and 71% of session
+ pages land on the prompts after the first — the part the title does not
+ already show. The FTS path is unchanged, response shape is unchanged, and no
+ schema migration is involved.
+
## [1.31.0] - 2026-08-22
### Fixed
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 7376989c..b5b3adb5 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -2466,6 +2466,128 @@ mod tests {
assert_eq!(hits[0].path.as_str(), "hyphen.md");
}
+ #[tokio::test]
+ async fn linked_neighbor_hit_describes_the_page_not_its_heading() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "ai-memory", None)
+ .await
+ .unwrap();
+
+ // A compiled page opens with its title and a structural heading. The
+ // graph-neighbour path has no matched term to centre an excerpt on,
+ // so before the descriptor fix its snippet was exactly this preamble.
+ let body = concat!(
+ "# Session 2026-04-20\n",
+ "\n",
+ "## Decisions\n",
+ "\n",
+ "Dropped the retry queue because the writer actor already serialises.\n",
+ );
+ store
+ .writer
+ .upsert_page(sample_page(ws, proj, "target.md", body))
+ .await
+ .unwrap();
+ let mut source = sample_page(ws, proj, "source.md", "needle source content");
+ source.links = vec![PagePath::new("target.md").unwrap().into()];
+ store.writer.upsert_page(source).await.unwrap();
+
+ let hits = store
+ .reader
+ .hybrid_search(
+ ws,
+ proj,
+ "needle".into(),
+ None,
+ String::new(),
+ String::new(),
+ 0,
+ 10,
+ None,
+ )
+ .await
+ .unwrap();
+ let neighbor = hits
+ .iter()
+ .find(|h| h.path.as_str() == "target.md")
+ .expect("linked neighbor should be included");
+ assert_eq!(
+ neighbor.snippet,
+ "Dropped the retry queue because the writer actor already serialises.",
+ "neighbour snippet should describe the page, not repeat its heading"
+ );
+ }
+
+ #[tokio::test]
+ async fn frontmatter_summary_wins_over_the_body_lede() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "ai-memory", None)
+ .await
+ .unwrap();
+
+ let mut page = sample_page(ws, proj, "curated.md", "Body prose that loses.");
+ page.frontmatter_json = serde_json::json!({ "summary": "Curated one-liner." });
+ store.writer.upsert_page(page).await.unwrap();
+
+ let hits = store
+ .reader
+ .recent_pages_for_project(ws, proj, 10)
+ .await
+ .unwrap();
+ let hit = hits
+ .iter()
+ .find(|h| h.path.as_str() == "curated.md")
+ .expect("page should be listed");
+ assert_eq!(hit.snippet, "Curated one-liner.");
+ }
+
+ #[tokio::test]
+ async fn blank_frontmatter_summary_falls_back_to_the_body_lede() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "ai-memory", None)
+ .await
+ .unwrap();
+
+ let mut page = sample_page(ws, proj, "blank.md", "Body prose that wins.");
+ page.frontmatter_json = serde_json::json!({ "summary": " " });
+ store.writer.upsert_page(page).await.unwrap();
+
+ let hits = store
+ .reader
+ .recent_pages_for_project(ws, proj, 10)
+ .await
+ .unwrap();
+ let hit = hits
+ .iter()
+ .find(|h| h.path.as_str() == "blank.md")
+ .expect("page should be listed");
+ assert_eq!(hit.snippet, "Body prose that wins.");
+ }
+
#[tokio::test]
async fn hybrid_search_includes_linked_neighbors() {
let tmp = TempDir::new().unwrap();
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index e918f9aa..7152122f 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -76,6 +76,112 @@ fn page_kind_expr(path_column: &str, frontmatter_column: &str) -> String {
)
}
+/// Leading body characters scanned when a page carries no frontmatter
+/// `summary`. Wide enough to step past the `# Title` line and a structural
+/// heading or two before the first real prose.
+const DESCRIPTOR_SCAN_CHARS: usize = 600;
+
+/// Maximum length of a synthesised page descriptor, in characters.
+const DESCRIPTOR_MAX_CHARS: usize = 240;
+
+/// SQL expression yielding the raw material for [`page_descriptor`]: the
+/// page's own frontmatter `summary` when it has a non-blank one, otherwise a
+/// prefix of the body. `NULLIF(TRIM(...), '')` keeps an empty summary from
+/// winning the `COALESCE` over real body text.
+fn page_descriptor_expr(body_column: &str, frontmatter_column: &str) -> String {
+ format!(
+ "COALESCE( \
+ NULLIF(TRIM(json_extract({frontmatter_column}, '$.summary')), ''), \
+ substr({body_column}, 1, {DESCRIPTOR_SCAN_CHARS}) \
+ )"
+ )
+}
+
+/// Strip a leading markdown list marker (`- `, `* `, `+ `, `1. `) so a kept
+/// line reads as prose in the descriptor.
+fn strip_list_marker(line: &str) -> &str {
+ if let Some(rest) = line
+ .strip_prefix("- ")
+ .or_else(|| line.strip_prefix("* "))
+ .or_else(|| line.strip_prefix("+ "))
+ {
+ return rest.trim_start();
+ }
+ let digits = line.chars().take_while(char::is_ascii_digit).count();
+ if digits > 0 && line[digits..].starts_with(". ") {
+ return line[digits + 2..].trim_start();
+ }
+ line
+}
+
+/// `true` for the `- **key:** value` bullets that open a compiled session
+/// page's metadata block — session id, timestamps, observation count. They
+/// are addressing, not content.
+fn is_metadata_bullet(line: &str) -> bool {
+ line.starts_with("- **") && line.contains(":**")
+}
+
+/// Page-intrinsic descriptor for a retrieval hit.
+///
+/// The FTS path centres its excerpt on the matched terms through
+/// `snippet(pages_fts, ...)`. The vector, entity-match, graph-neighbour and
+/// recency paths have no matched term to centre on. They previously returned
+/// `substr(body, 1, 240)`, which on a compiled page is the `# Title` line plus
+/// the `## Session metadata` block under it — the title the hit already
+/// carries, followed by a session id and three timestamps.
+///
+/// Fills the [`DESCRIPTOR_MAX_CHARS`] budget from the page's own text,
+/// skipping four things that carry no signal for the reader deciding whether
+/// to open the page: structural lines, metadata bullets, and any line that
+/// merely repeats `title`. On a session page that lands on the prompts after
+/// the first, which is the part `title` does not already show.
+fn page_descriptor(raw: &str, title: &str) -> String {
+ let title_key = title.trim().trim_end_matches('\u{2026}').trim();
+ let mut out = String::new();
+ for line in raw.lines().map(str::trim) {
+ if line.is_empty()
+ || line.starts_with('#')
+ || line.starts_with("---")
+ || line.starts_with("___")
+ || line.starts_with("***")
+ || is_metadata_bullet(line)
+ {
+ continue;
+ }
+ let content = strip_list_marker(line).trim();
+ if content.is_empty() {
+ continue;
+ }
+ if !title_key.is_empty() && content.starts_with(title_key) {
+ continue;
+ }
+ if !out.is_empty() {
+ if out.chars().count() + 1 + content.chars().count() > DESCRIPTOR_MAX_CHARS {
+ break;
+ }
+ out.push(' ');
+ }
+ out.push_str(content);
+ if out.chars().count() >= DESCRIPTOR_MAX_CHARS {
+ break;
+ }
+ }
+ if out.is_empty() {
+ return truncate_chars(raw.trim(), DESCRIPTOR_MAX_CHARS);
+ }
+ truncate_chars(&out, DESCRIPTOR_MAX_CHARS)
+}
+
+/// Truncate on a character boundary, marking elision with an ellipsis.
+fn truncate_chars(text: &str, max: usize) -> String {
+ if text.chars().count() <= max {
+ return text.to_owned();
+ }
+ let mut out: String = text.chars().take(max.saturating_sub(1)).collect();
+ out.push('\u{2026}');
+ out
+}
+
const AUTHORITY_CANDIDATE_MULTIPLIER: usize = 4;
const AUTHORITY_MIN_CANDIDATES: usize = 20;
const AUTHORITY_MAX_EXTRA_CANDIDATES: usize = 300;
@@ -1586,12 +1692,13 @@ impl ReaderPool {
self.with_conn(move |conn| {
let sql = format!(
"SELECT id, path, title, \
- substr(body, 1, 240) AS snip, \
+ {descriptor} AS snip, \
CAST(updated_at AS REAL) AS rank \
FROM pages \
WHERE is_latest = 1{not_expired} \
ORDER BY updated_at DESC \
LIMIT ?1",
+ descriptor = page_descriptor_expr("body", "frontmatter_json"),
not_expired = not_expired("pages", "?2"),
);
let mut stmt = conn.prepare_cached(&sql)?;
@@ -1600,7 +1707,7 @@ impl ReaderPool {
let id_bytes: Vec<u8> = row.get(0)?;
let path: String = row.get(1)?;
let title: String = row.get(2)?;
- let snippet: String = row.get(3)?;
+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);
let rank: f64 = row.get(4)?;
Ok((id_bytes, path, title, snippet, rank))
})?;
@@ -1633,12 +1740,13 @@ impl ReaderPool {
self.with_conn(move |conn| {
let sql = format!(
"SELECT id, path, title, \
- substr(body, 1, 240) AS snip, \
+ {descriptor} AS snip, \
CAST(updated_at AS REAL) AS rank \
FROM pages \
WHERE workspace_id = ?1 AND project_id = ?2 AND is_latest = 1{not_expired} \
ORDER BY updated_at DESC \
LIMIT ?3",
+ descriptor = page_descriptor_expr("body", "frontmatter_json"),
not_expired = not_expired("pages", "?4"),
);
let mut stmt = conn.prepare_cached(&sql)?;
@@ -1654,7 +1762,7 @@ impl ReaderPool {
let id_bytes: Vec<u8> = row.get(0)?;
let path: String = row.get(1)?;
let title: String = row.get(2)?;
- let snippet: String = row.get(3)?;
+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);
let rank: f64 = row.get(4)?;
Ok((id_bytes, path, title, snippet, rank))
},
@@ -1691,7 +1799,7 @@ impl ReaderPool {
self.with_conn(move |conn| {
let sql = format!(
"SELECT workspaces.name, projects.name, pages.path, pages.title, \
- substr(pages.body, 1, 240) AS snip, \
+ {descriptor} AS snip, \
CAST(pages.updated_at AS REAL) AS rank \
FROM pages \
JOIN projects ON projects.id = pages.project_id \
@@ -1699,6 +1807,7 @@ impl ReaderPool {
WHERE pages.is_latest = 1{not_expired} \
ORDER BY pages.updated_at DESC \
LIMIT ?1",
+ descriptor = page_descriptor_expr("pages.body", "pages.frontmatter_json"),
not_expired = not_expired("pages", "?2"),
);
let mut stmt = conn.prepare_cached(&sql)?;
@@ -1708,7 +1817,7 @@ impl ReaderPool {
let project_name: String = row.get(1)?;
let path: String = row.get(2)?;
let title: String = row.get(3)?;
- let snippet: String = row.get(4)?;
+ let snippet = page_descriptor(&row.get::<_, String>(4)?, &title);
let rank: f64 = row.get(5)?;
Ok((workspace_name, project_name, path, title, snippet, rank))
})?;
@@ -3158,7 +3267,7 @@ impl ReaderPool {
WHERE 1 = 1{freq_not_expired} \
GROUP BY m.entity_id, m.name \
) \
- SELECT pg.id, pg.path, pg.title, substr(pg.body, 1, 240) AS snippet, \
+ SELECT pg.id, pg.path, pg.title, {descriptor} AS snippet, \
SUM(1.0 / f.pages) AS weight, \
COUNT(*) AS matches, \
json_group_array(f.name) AS names \
@@ -3169,6 +3278,7 @@ impl ReaderPool {
GROUP BY pg.id, pg.path, pg.title \
ORDER BY weight DESC, matches DESC, pg.path ASC \
LIMIT ?",
+ descriptor = page_descriptor_expr("pg.body", "pg.frontmatter_json"),
not_expired = not_expired("pg", "?"),
freq_not_expired = not_expired("p", "?"),
)
@@ -3179,7 +3289,7 @@ impl ReaderPool {
let id_bytes: Vec<u8> = row.get(0)?;
let path: String = row.get(1)?;
let title: String = row.get(2)?;
- let snippet: String = row.get(3)?;
+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);
let weight: f64 = row.get(4)?;
let names_json: String = row.get(6)?;
Ok((id_bytes, path, title, snippet, weight, names_json))
@@ -3331,6 +3441,8 @@ impl ReaderPool {
sql_params.push(Value::Blob(project_id.as_bytes().to_vec()));
sql_params.push(Value::Integer(now));
+ let out_descriptor = page_descriptor_expr("tp.body", "tp.frontmatter_json");
+ let in_descriptor = page_descriptor_expr("fp.body", "fp.frontmatter_json");
let out_not_expired = not_expired("tp", "?");
let in_not_expired = not_expired("fp", "?");
let mut sql = String::with_capacity(values_clause.len() + 1_500);
@@ -3339,7 +3451,7 @@ impl ReaderPool {
"WITH seeds(seed_id, seed_ord) AS (VALUES {values_clause}), \
neighbors AS ( \
SELECT tp.id AS id, tp.path AS path, tp.title AS title, \
- substr(tp.body, 1, 240) AS snippet, \
+ {out_descriptor} AS snippet, \
seeds.seed_ord * 2 AS stream_ord, tp.updated_at AS updated_at \
FROM seeds \
JOIN links l ON l.from_page_id = seeds.seed_id \
@@ -3347,7 +3459,7 @@ impl ReaderPool {
WHERE tp.workspace_id = ? AND tp.project_id = ? AND tp.is_latest = 1{out_not_expired} \
UNION ALL \
SELECT fp.id AS id, fp.path AS path, fp.title AS title, \
- substr(fp.body, 1, 240) AS snippet, \
+ {in_descriptor} AS snippet, \
seeds.seed_ord * 2 + 1 AS stream_ord, fp.updated_at AS updated_at \
FROM seeds \
JOIN links l ON l.to_page_id = seeds.seed_id \
@@ -3366,7 +3478,7 @@ impl ReaderPool {
let id_bytes: Vec<u8> = row.get(0)?;
let path: String = row.get(1)?;
let title: String = row.get(2)?;
- let snippet: String = row.get(3)?;
+ let snippet = page_descriptor(&row.get::<_, String>(3)?, &title);
let stream_ord: i64 = row.get(4)?;
Ok((id_bytes, path, title, snippet, stream_ord))
})?;
@@ -7598,7 +7710,10 @@ fn open_read_only(path: &Path) -> StoreResult<Connection> {
#[cfg(test)]
mod tests {
- use super::{entity_query_tokens, handoff_listing_sql, like_escape};
+ use super::{
+ DESCRIPTOR_MAX_CHARS, entity_query_tokens, handoff_listing_sql, like_escape,
+ page_descriptor, page_descriptor_expr,
+ };
use crate::Store;
use ai_memory_core::{
@@ -8293,4 +8408,97 @@ mod tests {
assert_eq!(like_escape("back\\slash"), "back\\\\slash");
assert_eq!(like_escape("plain-name"), "plain-name");
}
+
+ #[test]
+ fn page_descriptor_skips_heading_and_rule_lines() {
+ let body = concat!(
+ "# Session 2026-04-20\n",
+ "\n",
+ "---\n",
+ "\n",
+ "## Summary\n",
+ "\n",
+ "Switched the scheduler to a single writer actor.\n",
+ );
+ assert_eq!(
+ page_descriptor(body, ""),
+ "Switched the scheduler to a single writer actor."
+ );
+ }
+
+ #[test]
+ fn page_descriptor_fills_the_budget_across_prose_lines() {
+ // A short opening line must not cost the rest of the budget: the
+ // `_lint` pages open with "N finding(s)." and everything that tells
+ // you what the findings were comes after it.
+ let body = concat!(
+ "# Lint findings\n",
+ "\n",
+ "501 finding(s).\n",
+ "\n",
+ "## 1 - stale (info)\n",
+ "\n",
+ "Episodic page sessions/e24f6fbc.md is 30 days old with zero accesses.\n",
+ );
+ assert_eq!(
+ page_descriptor(body, ""),
+ "501 finding(s). Episodic page sessions/e24f6fbc.md is 30 days old \
+ with zero accesses."
+ );
+ }
+
+ #[test]
+ fn page_descriptor_falls_back_to_raw_text_when_all_lines_are_structural() {
+ assert_eq!(page_descriptor("# Only a title\n", ""), "# Only a title");
+ assert_eq!(page_descriptor(" ", ""), "");
+ }
+
+ #[test]
+ fn page_descriptor_skips_metadata_bullets_and_the_repeated_title() {
+ // The compiled session-page shape: the title again as prompt 1, with
+ // the metadata block in between. What the reader does not already
+ // have is prompt 2 onward.
+ let title = "vamos revisar o deploy";
+ let body = concat!(
+ "# vamos revisar o deploy\n",
+ "\n",
+ "## Session metadata\n",
+ "\n",
+ "- **session_id:** `4e79a7ce-55e9-44c1-bcb5-74a6fe740900`\n",
+ "- **started_at:** 2026-08-22T19:26:07Z\n",
+ "- **observations:** 59\n",
+ "\n",
+ "## Prompts\n",
+ "\n",
+ "1. vamos revisar o deploy\n",
+ "2. o rollback falhou no passo do migrate\n",
+ );
+ assert_eq!(
+ page_descriptor(body, title),
+ "o rollback falhou no passo do migrate"
+ );
+ }
+
+ #[test]
+ fn page_descriptor_matches_a_title_truncated_with_an_ellipsis() {
+ let title = "vamos revisar o deploy do SHVIA-WEB e o rollback\u{2026}";
+ let body = "# x\n\n1. vamos revisar o deploy do SHVIA-WEB e o rollback do migrate\n2. depois o resto\n";
+ assert_eq!(page_descriptor(body, title), "depois o resto");
+ }
+
+ #[test]
+ fn page_descriptor_truncates_on_a_character_boundary() {
+ let body = "\u{e7}".repeat(DESCRIPTOR_MAX_CHARS + 10);
+ let out = page_descriptor(&body, "");
+ assert_eq!(out.chars().count(), DESCRIPTOR_MAX_CHARS);
+ assert!(out.ends_with('\u{2026}'));
+ }
+
+ #[test]
+ fn page_descriptor_expr_prefers_a_non_blank_frontmatter_summary() {
+ let sql = page_descriptor_expr("pages.body", "pages.frontmatter_json");
+ assert!(sql.contains("json_extract(pages.frontmatter_json, '$.summary')"));
+ assert!(sql.contains("NULLIF(TRIM("));
+ assert!(sql.contains("substr(pages.body, 1, 600)"));
+ }
}