Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(web): make the search kind and tier filter honest about its pool
Peterson Salme0946118d83f0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ea538f03..bbc933d8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,8 +16,10 @@ and [0.x](docs/changelog/0.x.md).has "Kind" and "Tier" selects, and `/web/search` accepts `kind` and `tier`for both a project search and a cross-project one. The filter runs afterranking, so the hits that remain keep the order `memory_query` gives them,- and a project search looks through its top 100 matches rather than the- first page of results. An unknown `tier` is a 400. (#1248)+ and a filtered search looks through its top 500 matches rather than the+ first page of results, saying so when the ranker had more. The kind match+ ignores case and a nonstandard kind is flagged on an empty result. An+ unknown `tier` is a 400. (#1248)- Added backlinks, outgoing links and an outline to the web page view. A pagein `/web` now has a side column (below the text on narrow screens) with "Onthis page", the h2 and h3 headings linked to their anchors (shown from twodiff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 419f45c8..8f7e357f 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -88,8 +88,8 @@ pub use reader::{DecayCandidate, DecayTombstone, DerivedIndexStatus, EmbeddingTripleCount, FeedbackFinding,GraphVia, HealthDetail, HealthPage, HybridSearch, InboxCount, InboxEntry, ObservationHit,ObservationOrder, ObservationPage, ObservationPageResult, ObservationRecord, OpenSession,- PAGE_VERSIONS_MAX_LIMIT, PageAuthor, PageHit, PageHitWithMeta, PageLinks, PageLinksPreview,- PageMeta, PageSummary, PageVersion, PageVersionBody, PageVersions,+ PAGE_KINDS, PAGE_VERSIONS_MAX_LIMIT, PageAuthor, PageHit, PageHitWithMeta, PageLinks,+ PageLinksPreview, PageMeta, PageSummary, PageVersion, PageVersionBody, PageVersions,ProjectCoordinateCollisionReason, ProjectCoordinateDiagnostic, ProjectCoordinateStatus,ProjectSummary, RELATED_WALK_MAX_DEPTH, RELATED_WALK_MAX_NODES, ReaderPool,ReindexTargetStatus, RelatedNode, RelatedPage, RrfContributions, ScopeRow, SearchExplain,diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex 1a4c6f8d..581787cb 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -201,26 +201,87 @@ fn readable_repository_sql()}+/// Path prefix to kind, in match order: the kind a page has when its+/// frontmatter names none.+const PATH_KINDS: [(&str, &str); 9] = [+ ("_rules/", "rule"),+ ("_slots/", "slot"),+ ("sessions/", "session"),+ ("decisions/", "decision"),+ ("gotchas/", "gotcha"),+ ("concepts/", "concept"),+ ("procedures/", "procedure"),+ ("notes/", "note"),+ ("profile/", "preference"),+];++/// Kind of a page whose frontmatter names none and whose path matches no+/// [`PATH_KINDS`] prefix.+const DEFAULT_PAGE_KIND: &str = "fact";++/// Every kind a page can have without custom frontmatter, as+/// `page_kind_expr` derives it.+pub const PAGE_KINDS: [&str; 10] = [+ "rule",+ "slot",+ "session",+ "decision",+ "gotcha",+ "concept",+ "procedure",+ "note",+ "preference",+ DEFAULT_PAGE_KIND,+];+fn page_kind_expr(path_column: &str, frontmatter_column: &str) -> String {+ let arms: String = PATH_KINDS+ .iter()+ .map(|(prefix, kind)| {+ // `_` is a LIKE wildcard; escape it so `_rules/` matches only itself.+ let prefix = prefix.replace('_', "\\_");+ format!("WHEN {path_column} LIKE '{prefix}%' ESCAPE '\\' THEN '{kind}' ")+ })+ .collect();+ // The cast keeps a non-string frontmatter `kind` (`kind: 5`) readable as+ // text instead of failing the row.format!("COALESCE( \- json_extract({frontmatter_column}, '$.kind'), \- CASE \- WHEN {path_column} LIKE '\\_rules/%' ESCAPE '\\' THEN 'rule' \- WHEN {path_column} LIKE '\\_slots/%' ESCAPE '\\' THEN 'slot' \- WHEN {path_column} LIKE 'sessions/%' THEN 'session' \- WHEN {path_column} LIKE 'decisions/%' THEN 'decision' \- WHEN {path_column} LIKE 'gotchas/%' THEN 'gotcha' \- WHEN {path_column} LIKE 'concepts/%' THEN 'concept' \- WHEN {path_column} LIKE 'procedures/%' THEN 'procedure' \- WHEN {path_column} LIKE 'notes/%' THEN 'note' \- WHEN {path_column} LIKE 'profile/%' THEN 'preference' \- ELSE 'fact' \- END \+ CAST(json_extract({frontmatter_column}, '$.kind') AS TEXT), \+ CASE {arms}ELSE '{DEFAULT_PAGE_KIND}' END \)")}+#[cfg(test)]+mod page_kind_tests {+ use super::*;++ #[test]+ fn page_kinds_lists_every_path_kind_and_the_default() {+ let mut expected: Vec<&str> = PATH_KINDS.iter().map(|(_, kind)| *kind).collect();+ expected.push(DEFAULT_PAGE_KIND);+ assert_eq!(PAGE_KINDS.to_vec(), expected);+ }++ #[test]+ fn page_kind_expr_resolves_paths_and_non_string_kinds() {+ let conn = rusqlite::Connection::open_in_memory().unwrap();+ let sql = format!(+ "SELECT {} FROM (SELECT ?1 AS path, ?2 AS fm)",+ page_kind_expr("path", "fm")+ );+ let kind = |path: &str, fm: &str| -> String {+ conn.query_row(&sql, [path, fm], |row| row.get(0)).unwrap()+ };+ assert_eq!(kind("_rules/a.md", "{}"), "rule");+ assert_eq!(kind("xrules/a.md", "{}"), "fact");+ assert_eq!(kind("profile/a.md", "{}"), "preference");+ assert_eq!(kind("sessions/a.md", r#"{"kind":"custom"}"#), "custom");+ assert_eq!(kind("sessions/a.md", r#"{"kind":5}"#), "5");+ }+}+/// Leading body characters scanned when a page carries no frontmatter/// `summary`. Wide enough to step past the `# Title` line and a structural/// heading or two before the first real prose.diff --git a/crates/ai-memory-web/src/routes/search.rs b/crates/ai-memory-web/src/routes/search.rsindex 399b5848..acc494e1 100644--- a/crates/ai-memory-web/src/routes/search.rs+++ b/crates/ai-memory-web/src/routes/search.rs@@ -20,7 +20,7 @@ use std::str::FromStr;use std::sync::Arc;use ai_memory_core::Tier;-use ai_memory_store::{SearchExplain, StreamTimings};+use ai_memory_store::{PAGE_KINDS, SearchExplain, StreamTimings};use askama::Template;use axum::extract::{Query, State};use axum::http::StatusCode;@@ -40,20 +40,9 @@ const GLOBAL_DEFAULT_LIMIT: usize = 50;const SCOPED_DEFAULT_LIMIT: usize = 20;/// Upper bound on `limit`, the same clamp `memory_query` applies.const MAX_LIMIT: usize = 100;-/// Page kinds the filter offers: the kinds a path convention implies, which-/// are also the ones the consolidator writes.-const KINDS: [&str; 10] = [- "rule",- "slot",- "session",- "decision",- "gotcha",- "concept",- "procedure",- "note",- "preference",- "fact",-];+/// Matches a filtered search ranks before it filters. The ranker has no+/// kind or tier predicate, so a filter only sees this many top hits.+const FILTER_POOL: usize = 500;/// Memory tiers the filter offers, in `Tier` declaration order.const TIERS: [&str; 4] = ["working", "episodic", "semantic", "procedural"];@@ -150,12 +139,13 @@ pub(crate) async fn handler(};let filtered = kind.is_some() || tier.is_some();let keeps = |hit_kind: &str, hit_tier: &str| {- kind.as_deref().is_none_or(|k| k == hit_kind)+ kind.as_deref()+ .is_none_or(|k| k.eq_ignore_ascii_case(hit_kind))&& tier.as_deref().is_none_or(|t| t == hit_tier)};- // A filter runs after ranking, so the ranker is asked for the most it can- // return and the limit is applied to what survives.- let fetch_limit = |limit: usize| if filtered { MAX_LIMIT + 1 } else { limit + 1 };+ // A filter runs after ranking, so the ranker is asked for a wide pool+ // and the limit is applied to what survives.+ let fetch_limit = |limit: usize| if filtered { FILTER_POOL } else { limit + 1 };let internal = |error: ai_memory_store::StoreError| {tracing::error!(%error, "web search failed");StatusCode::INTERNAL_SERVER_ERROR.into_response()@@ -166,6 +156,8 @@ pub(crate) async fn handler(// apart from one that has exactly `limit` matches.let mut limit_used = 0;let mut truncated = false;+ // The filter saw a full pool, so matches ranked past it were not looked at.+ let mut pool_capped = false;let mut timings = None;let mut streams_active: Vec<&'static str> = Vec::new();match &scope {@@ -198,6 +190,7 @@ pub(crate) async fn handler(streams_active = result.streams_active;// The trace position is the rank before filtering.let mut ranked: Vec<_> = result.hits.into_iter().enumerate().collect();+ pool_capped = filtered && ranked.len() >= FILTER_POOL;if filtered {let ids: Vec<_> = ranked.iter().map(|(_, (hit, _))| hit.id).collect();let facets = match state.reader.page_kinds_tiers_by_ids(&ids).await {@@ -249,6 +242,7 @@ pub(crate) async fn handler(});streams_active = vec!["fts"];}+ pool_capped = filtered && raw.len() >= FILTER_POOL;let raw: Vec<_> = raw.into_iter().filter(|h| keeps(&h.kind, &h.tier))@@ -302,7 +296,12 @@ pub(crate) async fn handler(}).collect();let hit_count = hits.len();- let kind_options = filter_options(&KINDS, kind.as_deref());+ let unknown_kind = kind+ .as_deref()+ .filter(|k| !PAGE_KINDS.iter().any(|known| known.eq_ignore_ascii_case(k)))+ .unwrap_or_default()+ .to_owned();+ let kind_options = filter_options(&PAGE_KINDS, kind.as_deref());let tier_options = filter_options(&TIERS, tier.as_deref());let global_href = format!("search?q={}", encode_segment(&query));let view = SearchView {@@ -313,6 +312,9 @@ pub(crate) async fn handler(hits,hit_count,filtered,+ pool_capped,+ filter_pool: FILTER_POOL,+ unknown_kind,kind_options,tier_options,workspace,@@ -341,14 +343,14 @@ pub(crate) async fn handler(/// outside `known` (a custom frontmatter kind) is kept so the form round-trips.fn filter_options(known: &[&str], current: Option<&str>) -> Vec<FilterOption> {let mut values: Vec<&str> = known.to_vec();- if let Some(current) = current.filter(|c| !known.contains(c)) {+ if let Some(current) = current.filter(|c| !known.iter().any(|k| k.eq_ignore_ascii_case(c))) {values.push(current);}values.into_iter().map(|value| FilterOption {value: value.to_owned(),- selected: current == Some(value),+ selected: current.is_some_and(|c| c.eq_ignore_ascii_case(value)),}).collect()}diff --git a/crates/ai-memory-web/src/templates.rs b/crates/ai-memory-web/src/templates.rsindex a3ce8613..ba723210 100644--- a/crates/ai-memory-web/src/templates.rs+++ b/crates/ai-memory-web/src/templates.rs@@ -688,6 +688,14 @@ pub(crate) struct SearchView {pub project_href: String,/// A kind or tier filter narrowed the hits.pub filtered: bool,+ /// The filter looked only at the top `filter_pool` matches, and the+ /// ranker had more.+ pub pool_capped: bool,+ /// Matches a filtered search ranks before it filters.+ pub filter_pool: usize,+ /// The kind filter names no standard kind (empty when it does, or when+ /// there is none); a custom frontmatter kind or a typo.+ pub unknown_kind: String,/// Kinds the filter offers.pub kind_options: Vec<FilterOption>,/// Tiers the filter offers.diff --git a/crates/ai-memory-web/templates/search.html b/crates/ai-memory-web/templates/search.htmlindex 667c153d..67c22792 100644--- a/crates/ai-memory-web/templates/search.html+++ b/crates/ai-memory-web/templates/search.html@@ -113,6 +113,8 @@<p class="text-sm text-slate-500 dark:text-slate-400">Enter a search term above.</p>{% else if hits.is_empty() %}<p class="text-sm text-slate-500 dark:text-slate-400">No results for <strong>{{ query }}</strong>{% if filtered %} with these filters{% endif %}.+ {% if pool_capped %}Only the top {{ filter_pool }} matches were searched.{% endif %}+ {% if unknown_kind != "" %}<code>{{ unknown_kind }}</code> is not a standard page kind; check the spelling.{% endif %}{% if self.scoped() %}<a href="{{ global_href }}" class="ml-2 inline-block rounded-md border border-slate-300 dark:border-slate-700 px-3 py-1 text-slate-700 dark:text-slate-300 hover:bg-slate-50 dark:hover:bg-slate-900">Search all projects</a>{% endif %}@@ -123,6 +125,9 @@{% else %}<p class="text-sm text-slate-500 dark:text-slate-400 mb-4">{{ hit_count }} result{% if hit_count != 1 %}s{% endif %}</p>{% endif %}+{% if pool_capped %}+<p class="text-sm text-slate-500 dark:text-slate-400 mb-4">The filters were applied to the top {{ filter_pool }} matches; refine the query to reach lower-ranked pages.</p>+{% endif %}<ul class="divide-y divide-slate-100 dark:divide-slate-800">{% for hit in hits %}<li class="py-4" data-hit-path="{{ hit.path }}">diff --git a/crates/ai-memory-web/tests/suite/recall_trace.rs b/crates/ai-memory-web/tests/suite/recall_trace.rsindex ec0990e3..c46ef7b1 100644--- a/crates/ai-memory-web/tests/suite/recall_trace.rs+++ b/crates/ai-memory-web/tests/suite/recall_trace.rs@@ -674,3 +674,42 @@ async fn kind_filter_does_not_widen_what_a_viewer_can_read() {let (_, body) = get(&app, "/web/search?q=token%20rotation&kind=fact", None).await;assert!(body.contains("notes/beta-rotation.md"), "{body}");}++#[tokio::test]+async fn kind_filter_is_case_insensitive_and_flags_a_nonstandard_kind() {+ let f = fixture(false).await;+ add_procedure(&f).await;+ let app = web(&f, Recall::default());+ let base = "/web/search?q=token%20rotation&workspace=default&project=alpha";++ let (_, body) = get(&app, &format!("{base}&kind=Procedure"), None).await;+ assert_eq!(hit_paths(&body), ["procedures/rotate.md"], "{body}");+ assert!(!body.contains("is not a standard page kind"), "{body}");++ // A typo matches nothing and says so, instead of looking like no results.+ let (status, body) = get(&app, &format!("{base}&kind=procedures"), None).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert!(hit_paths(&body).is_empty(), "{body}");+ assert!(body.contains("is not a standard page kind"), "{body}");+}++#[test]+fn tier_filter_offers_every_tier() {+ use ai_memory_core::Tier;+ // An exhaustive match: a new `Tier` variant fails to compile here until+ // the filter's list is revisited.+ for tier in [+ Tier::Working,+ Tier::Episodic,+ Tier::Semantic,+ Tier::Procedural,+ ] {+ let name = match tier {+ Tier::Working => "working",+ Tier::Episodic => "episodic",+ Tier::Semantic => "semantic",+ Tier::Procedural => "procedural",+ };+ assert_eq!(name.parse::<Tier>().unwrap(), tier);+ }+}