Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(consolidate): cluster a cold set of exactly min_pts near-duplicates

Rudy Mizrahi Celeklie085cfc7a786
diff --git a/crates/ai-memory-consolidate/src/cold_cluster.rs b/crates/ai-memory-consolidate/src/cold_cluster.rs
index 27aa7af5..602f7b46 100644
--- a/crates/ai-memory-consolidate/src/cold_cluster.rs
+++ b/crates/ai-memory-consolidate/src/cold_cluster.rs
@@ -49,12 +49,17 @@ pub fn cosine_distance(a: &[f32], b: &[f32]) -> f32 {
/// kneedle construction). The result is then clamped to `max_eps` so an operator
/// keeps a hard conservative ceiling: A3 errs toward NOT merging.
///
-/// Returns `None` when there are too few points to form a `k`-distance
-/// (`points.len() <= k`), which the caller treats as "nothing to cluster".
+/// Returns `None` only when there is nothing a distance can be measured
+/// across (`k == 0` or fewer than two points), which the caller treats as
+/// "nothing to cluster". A set of exactly `min_pts` points is a cluster the
+/// density floor admits by definition — with the default `min_pts = 2`, "a
+/// pair of near-duplicates is the smallest thing worth collapsing" — so it
+/// must still yield an eps. The `max_eps` clamp below is what keeps A3
+/// conservative, not a refusal to measure.
#[must_use]
pub fn adaptive_eps(points: &[Vec<f32>], k: usize, max_eps: f32) -> Option<f32> {
let n = points.len();
- if k == 0 || n <= k {
+ if k == 0 || n < 2 {
return None;
}
// k-th nearest-neighbour distance for every point.
@@ -68,7 +73,10 @@ pub fn adaptive_eps(points: &[Vec<f32>], k: usize, max_eps: f32) -> Option<f32>
.collect();
dists.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
// `dists` excludes self, so the k-th nearest neighbour is index k-1.
- k_distances.push(dists[k - 1]);
+ // With fewer than `k` neighbours available, the farthest one is the
+ // densest honest estimate; clamping here is what lets a set of exactly
+ // `k` points (a pair, by default) produce an eps at all.
+ k_distances.push(dists[(k - 1).min(dists.len() - 1)]);
}
k_distances.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
@@ -212,6 +220,52 @@ mod tests {
);
}
+ /// A cold set of exactly `min_pts` near-duplicates is the smallest thing
+ /// the default density floor is meant to collapse, so it must yield an eps
+ /// and cluster. Before this, `adaptive_eps` returned `None` for `n <= k`,
+ /// so a pair alone was silently never merged — while the same pair with
+ /// any third page present collapsed fine.
+ #[test]
+ fn a_pair_of_near_duplicates_yields_an_eps_and_clusters() {
+ let points = vec![vec![1.0, 0.0, 0.0], vec![0.99, 0.01, 0.0]];
+
+ let eps = adaptive_eps(&points, 2, 0.5).expect("a pair must produce an eps");
+ assert!(eps <= 0.5, "the conservative ceiling still applies: {eps}");
+
+ let clusters = dbscan(&points, eps, 2);
+ assert_eq!(clusters.len(), 1, "the pair collapses: {clusters:?}");
+ let mut members = clusters[0].clone();
+ members.sort_unstable();
+ assert_eq!(members, vec![0, 1]);
+ }
+
+ /// The ceiling, not the point count, is what keeps A3 from over-merging: a
+ /// distant pair still must not cluster under a tight `max_eps`.
+ #[test]
+ fn a_distant_pair_stays_unclustered_under_a_tight_ceiling() {
+ let points = vec![vec![1.0, 0.0], vec![0.0, 1.0]];
+
+ let eps = adaptive_eps(&points, 2, 0.15).expect("a pair still produces an eps");
+ assert!(eps <= 0.15, "clamped to the ceiling: {eps}");
+
+ let clusters = dbscan(&points, eps, 2);
+ assert!(
+ clusters.is_empty(),
+ "an orthogonal pair is noise, not a merge: {clusters:?}"
+ );
+ }
+
+ /// Fewer than two points has no distance to measure.
+ #[test]
+ fn adaptive_eps_needs_two_points() {
+ assert_eq!(adaptive_eps(&[vec![1.0, 0.0]], 2, 0.5), None);
+ assert_eq!(adaptive_eps(&[], 2, 0.5), None);
+ assert_eq!(
+ adaptive_eps(&[vec![1.0, 0.0], vec![0.9, 0.1]], 0, 0.5),
+ None
+ );
+ }
+
#[test]
fn deterministic_across_runs() {
let points = vec![
diff --git a/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs b/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs
index f0fd9029..fb877469 100644
--- a/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs
+++ b/crates/ai-memory-consolidate/tests/suite/cold_cluster_sweep.rs
@@ -195,6 +195,51 @@ async fn flag_off_never_merges() {
assert_eq!(report.evicted.len(), 2, "both cold pages evicted as before");
}
+/// The same pair with NO third page present. This is the shape the fixture
+/// above hides: `adaptive_eps` used to return `None` for `points.len() <= k`,
+/// so with the default `min_pts = 2` a cold set of exactly two near-duplicates
+/// produced an empty plan, while adding any unrelated third page made the same
+/// pair collapse.
+#[tokio::test]
+async fn flag_on_collapses_a_pair_with_no_other_cold_page() {
+ let fx = seed_fixture(true).await;
+ write(&fx, "sessions/a.md", DUP_A).await;
+ write(&fx, "sessions/b.md", DUP_B).await;
+ for p in ["sessions/a.md", "sessions/b.md"] {
+ backdate_latest(&fx, p, COLD_DAYS);
+ }
+
+ let report = run_sweep_with_hygiene(
+ &fx.store.reader,
+ &fx.store.writer,
+ Some(&fx.wiki),
+ fx.ws,
+ fx.proj,
+ &DecayParams::default(),
+ 0.0,
+ ObservationRetention::default(),
+ false,
+ dedup_on(),
+ false,
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ report.merged.len(),
+ 1,
+ "a pair is the smallest collapsible set under min_pts = 2: {:?}",
+ report.merged
+ );
+ let cluster = &report.merged[0];
+ assert!(cluster.applied, "the collapse landed");
+ assert_eq!(cluster.merged.len(), 1, "one member merged away");
+ assert_ne!(
+ cluster.survivor, cluster.merged[0],
+ "survivor and loser are distinct"
+ );
+}
+
/// Flag ON: the two near-duplicates collapse to one survivor; the survivor keeps
/// BOTH facts; the loser is superseded (reachable); the merge is recorded in
/// `page_evidence`; the far page is untouched.