Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(consolidate): cluster a cold set of exactly min_pts near-duplicates
Rudy Mizrahi Celeklie085cfc7a786
diff --git a/crates/ai-memory-consolidate/src/cold_cluster.rs b/crates/ai-memory-consolidate/src/cold_cluster.rsindex 27aa7af5..602f7b46 100644--- a/crates/ai-memory-consolidate/src/cold_cluster.rs+++ b/crates/ai-memory-consolidate/src/cold_cluster.rs@@ -49,12 +49,17 @@ pub fn cosine_distance(a: &[f32], b: &[f32]) -> f32 {/// kneedle construction). The result is then clamped to `max_eps` so an operator/// keeps a hard conservative ceiling: A3 errs toward NOT merging.///-/// Returns `None` when there are too few points to form a `k`-distance-/// (`points.len() <= k`), which the caller treats as "nothing to cluster".+/// Returns `None` only when there is nothing a distance can be measured+/// across (`k == 0` or fewer than two points), which the caller treats as+/// "nothing to cluster". A set of exactly `min_pts` points is a cluster the+/// density floor admits by definition — with the default `min_pts = 2`, "a+/// pair of near-duplicates is the smallest thing worth collapsing" — so it+/// must still yield an eps. The `max_eps` clamp below is what keeps A3+/// conservative, not a refusal to measure.#[must_use]pub fn adaptive_eps(points: &[Vec<f32>], k: usize, max_eps: f32) -> Option<f32> {let n = points.len();- if k == 0 || n <= k {+ if k == 0 || n < 2 {return None;}// k-th nearest-neighbour distance for every point.@@ -68,7 +73,10 @@ pub fn adaptive_eps(points: &[Vec<f32>], k: usize, max_eps: f32) -> Option<f32>.collect();dists.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));// `dists` excludes self, so the k-th nearest neighbour is index k-1.- k_distances.push(dists[k - 1]);+ // With fewer than `k` neighbours available, the farthest one is the+ // densest honest estimate; clamping here is what lets a set of exactly+ // `k` points (a pair, by default) produce an eps at all.+ k_distances.push(dists[(k - 1).min(dists.len() - 1)]);}k_distances.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));@@ -212,6 +220,52 @@ mod tests {);}+ /// A cold set of exactly `min_pts` near-duplicates is the smallest thing+ /// the default density floor is meant to collapse, so it must yield an eps+ /// and cluster. Before this, `adaptive_eps` returned `None` for `n <= k`,+ /// so a pair alone was silently never merged — while the same pair with+ /// any third page present collapsed fine.+ #[test]+ fn a_pair_of_near_duplicates_yields_an_eps_and_clusters() {+ let points = vec![vec![1.0, 0.0, 0.0], vec![0.99, 0.01, 0.0]];++ let eps = adaptive_eps(&points, 2, 0.5).expect("a pair must produce an eps");+ assert!(eps <= 0.5, "the conservative ceiling still applies: {eps}");++ let clusters = dbscan(&points, eps, 2);+ assert_eq!(clusters.len(), 1, "the pair collapses: {clusters:?}");+ let mut members = clusters[0].clone();+ members.sort_unstable();+ assert_eq!(members, vec![0, 1]);+ }++ /// The ceiling, not the point count, is what keeps A3 from over-merging: a+ /// distant pair still must not cluster under a tight `max_eps`.+ #[test]+ fn a_distant_pair_stays_unclustered_under_a_tight_ceiling() {+ let points = vec![vec![1.0, 0.0], vec![0.0, 1.0]];++ let eps = adaptive_eps(&points, 2, 0.15).expect("a pair still produces an eps");+ assert!(eps <= 0.15, "clamped to the ceiling: {eps}");++ let clusters = dbscan(&points, eps, 2);+ assert!(+ clusters.is_empty(),+ "an orthogonal pair is noise, not a merge: {clusters:?}"+ );+ }++ /// Fewer than two points has no distance to measure.+ #[test]+ fn adaptive_eps_needs_two_points() {+ assert_eq!(adaptive_eps(&[vec![1.0, 0.0]], 2, 0.5), None);+ assert_eq!(adaptive_eps(&[], 2, 0.5), None);+ assert_eq!(+ adaptive_eps(&[vec![1.0, 0.0], vec![0.9, 0.1]], 0, 0.5),+ None+ );+ }+#[test]fn deterministic_across_runs() {let points = vec![diff --git a/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs b/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rsindex f0fd9029..fb877469 100644--- a/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs+++ b/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs@@ -195,6 +195,51 @@ async fn flag_off_never_merges() {assert_eq!(report.evicted.len(), 2, "both cold pages evicted as before");}+/// The same pair with NO third page present. This is the shape the fixture+/// above hides: `adaptive_eps` used to return `None` for `points.len() <= k`,+/// so with the default `min_pts = 2` a cold set of exactly two near-duplicates+/// produced an empty plan, while adding any unrelated third page made the same+/// pair collapse.+#[tokio::test]+async fn flag_on_collapses_a_pair_with_no_other_cold_page() {+ let fx = seed_fixture(true).await;+ write(&fx, "sessions/a.md", DUP_A).await;+ write(&fx, "sessions/b.md", DUP_B).await;+ for p in ["sessions/a.md", "sessions/b.md"] {+ backdate_latest(&fx, p, COLD_DAYS);+ }++ let report = run_sweep_with_hygiene(+ &fx.store.reader,+ &fx.store.writer,+ Some(&fx.wiki),+ fx.ws,+ fx.proj,+ &DecayParams::default(),+ 0.0,+ ObservationRetention::default(),+ false,+ dedup_on(),+ false,+ )+ .await+ .unwrap();++ assert_eq!(+ report.merged.len(),+ 1,+ "a pair is the smallest collapsible set under min_pts = 2: {:?}",+ report.merged+ );+ let cluster = &report.merged[0];+ assert!(cluster.applied, "the collapse landed");+ assert_eq!(cluster.merged.len(), 1, "one member merged away");+ assert_ne!(+ cluster.survivor, cluster.merged[0],+ "survivor and loser are distinct"+ );+}+/// Flag ON: the two near-duplicates collapse to one survivor; the survivor keeps/// BOTH facts; the loser is superseded (reachable); the merge is recorded in/// `page_evidence`; the far page is untouched.