Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(knn): HNSW-served top-level kNN — official cell 23,325ms → 1.87ms, recall@10 1.00 vs ES 0.80

xerj-org3b83a96d2cad
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 518f5cde4..f084cd182 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -412,6 +412,29 @@ pub struct Index {
hnsw_id_rev: Arc<RwLock<HashMap<u64, String>>>,
/// Monotonic counter for HNSW node IDs.
hnsw_next_id: Arc<AtomicU64>,
+ /// FIELD IDENTITY: the single field this index's HNSW graph indexes.
+ ///
+ /// Pinned when the graph is first created (preferring the schema's first
+ /// dense_vector-mapped field over the ingested doc's first numeric-array
+ /// field). Once `Some(f)`, `index_vectors` inserts ONLY field `f`; docs
+ /// lacking `f` get no graph entry, which breaks the
+ /// `hnsw_id_map.len() == doc_count` coverage guard and keeps the kNN
+ /// query path on exact brute force (safe). `None` with a live graph
+ /// means a legacy (pre-field-stamp) snapshot: ingest keeps the historical
+ /// per-doc first-field behavior and the HNSW query path never engages.
+ ///
+ /// std (not tokio) RwLock: read on every ingested doc and inside the
+ /// query gate; critical sections are a clone/compare with no `.await`.
+ hnsw_field: Arc<std::sync::RwLock<Option<String>>>,
+ /// True when the loaded HNSW snapshot is flush-time stale relative to
+ /// the replayed WAL (`ids.json` seq_no stamp != store.current_seq_no()).
+ /// A WAL-tail UPDATE of an already-graphed doc leaves both
+ /// `hnsw_id_map.len()` and `doc_count` unchanged, so the coverage guard
+ /// cannot see it — this flag disables the HNSW query path (brute force
+ /// serves instead) while ingest-side graph maintenance continues.
+ /// Sticky for the process lifetime; persisted back into ids.json so a
+ /// later flush of the still-stale graph cannot re-stamp it as fresh.
+ hnsw_stale: Arc<std::sync::atomic::AtomicBool>,
/// Per-field SQ8 (scalar8) code stores for the kNN serving path.
///
/// Keyed by dense_vector field name. Populated only for fields whose
@@ -825,6 +848,8 @@ impl Index {
hnsw_id_map: Arc::new(RwLock::new(HashMap::new())),
hnsw_id_rev: Arc::new(RwLock::new(HashMap::new())),
hnsw_next_id: Arc::new(AtomicU64::new(1)),
+ hnsw_field: Arc::new(std::sync::RwLock::new(None)),
+ hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(false)),
sq8_stores: Arc::new(RwLock::new(HashMap::new())),
metric_query_count: Arc::new(AtomicU64::new(0)),
metric_query_total_ms: Arc::new(AtomicU64::new(0)),
@@ -989,12 +1014,36 @@ impl Index {
// first kNN search after a vector ingest re-creates the graph.
let loaded_hnsw = load_hnsw_artifacts_sync(&index_dir.join("hnsw"));
let initial_hnsw = loaded_hnsw.as_ref().map(|_| ()).is_some();
- let (hnsw_init, id_map_init, id_rev_init, next_id_init) = match loaded_hnsw {
- Some(l) => (Some(l.graph), l.id_map, l.id_rev, l.next_id),
- None => (None, HashMap::new(), HashMap::new(), 1),
- };
+ let (hnsw_init, id_map_init, id_rev_init, next_id_init, hnsw_field_init, hnsw_stale_init) =
+ match loaded_hnsw {
+ Some(l) => {
+ // Freshness stamp check: the snapshot is flush-time, but
+ // WAL replay never calls index_vectors — so a WAL-tail
+ // UPDATE of an already-graphed doc leaves the graph
+ // holding the pre-update vector while the
+ // `id_map.len() == doc_count` coverage guard still
+ // passes (len and count both unchanged). A stamp
+ // mismatch therefore marks the graph stale: ingest keeps
+ // maintaining it, but the kNN query path stays on exact
+ // brute force. Legacy snapshots without a `field` stamp
+ // never serve queries anyway (field gate), so they are
+ // not marked stale.
+ let stale = match (l.field.as_ref(), l.seq_no) {
+ (Some(_), Some(stamp)) => l.stale || stamp != store.current_seq_no(),
+ (Some(_), None) => true, // fielded snapshot without a stamp: distrust
+ (None, _) => false, // legacy snapshot: field-gated off already
+ };
+ (Some(l.graph), l.id_map, l.id_rev, l.next_id, l.field, stale)
+ }
+ None => (None, HashMap::new(), HashMap::new(), 1, None, false),
+ };
if initial_hnsw {
- info!(name = name.as_str(), "HNSW reloaded from disk");
+ info!(
+ name = name.as_str(),
+ field = hnsw_field_init.as_deref().unwrap_or("<legacy>"),
+ stale = hnsw_stale_init,
+ "HNSW reloaded from disk"
+ );
}
info!(
@@ -1026,6 +1075,8 @@ impl Index {
hnsw_id_map: Arc::new(RwLock::new(id_map_init)),
hnsw_id_rev: Arc::new(RwLock::new(id_rev_init)),
hnsw_next_id: Arc::new(AtomicU64::new(next_id_init)),
+ hnsw_field: Arc::new(std::sync::RwLock::new(hnsw_field_init)),
+ hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(hnsw_stale_init)),
sq8_stores: Arc::new(RwLock::new(HashMap::new())),
metric_query_count: Arc::new(AtomicU64::new(0)),
metric_query_total_ms: Arc::new(AtomicU64::new(0)),
@@ -2317,6 +2368,14 @@ impl Index {
self.last_flush_doc_count
.store(self.doc_count.load(Ordering::Relaxed), Ordering::Relaxed);
+ // Persist the HNSW graph at the refresh boundary too (flush()
+ // already does). Bulk-ingest + _refresh is the standard ES client
+ // workflow and background auto-flushes skip the graph save, so
+ // without this the on-disk snapshot would stay at some mid-ingest
+ // stamp and every restart would (safely, but needlessly) mark the
+ // graph flush-time stale and fall back to brute-force kNN.
+ let _ = self.save_hnsw_to_disk().await;
+
debug!(
index = self.name.as_str(),
"refresh complete — memtable flushed to disk segment"
@@ -3443,57 +3502,177 @@ impl Index {
}
}
- /// Scan a document for array-of-number fields and insert them into the HNSW index.
+ /// Scan a document for vector fields and insert them into the HNSW graph.
+ ///
+ /// FIELD IDENTITY (see `hnsw_field`): one graph indexes exactly ONE
+ /// field. The field is pinned when the graph is first created —
+ /// preferring the schema's first dense_vector-mapped field over the
+ /// ingested doc's first numeric-array field — and every subsequent doc
+ /// contributes ONLY that field. Docs lacking the pinned field are
+ /// skipped: their absence breaks the `hnsw_id_map.len() == doc_count`
+ /// coverage guard, which keeps the kNN query path on exact brute force,
+ /// instead of silently searching a graph that mixes semantically
+ /// unrelated fields (the pre-2026-07 behavior inserted each doc's own
+ /// first numeric-array field, so one graph could span fields).
+ ///
+ /// UPDATES: re-indexing an existing doc_id tombstones the doc's previous
+ /// node before inserting the fresh vector — otherwise the graph would
+ /// keep serving the pre-update vector forever.
async fn index_vectors(&self, doc_id: &str, source: &Value) {
- if let Some(obj) = source.as_object() {
+ let obj = match source.as_object() {
+ Some(o) => o,
+ None => return,
+ };
+ let pinned: Option<String> = self.hnsw_field.read().unwrap().clone();
+ if let Some(field) = pinned {
+ // Enforced path: only the pinned field ever enters the graph.
+ if let Some(vector) = extract_numeric_vector(source, &field) {
+ self.hnsw_insert_vector(doc_id, vector).await;
+ }
+ return;
+ }
+
+ // No pinned field yet. Cheap sync pre-screen (same shape as the
+ // legacy scan): bail before any async lock when the doc carries no
+ // top-level all-numbers array — log/text workloads take this exit.
+ let has_candidate = obj.values().any(|v| {
+ v.as_array()
+ .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))
+ .unwrap_or(false)
+ });
+ if !has_candidate {
+ return;
+ }
+
+ if self.hnsw.read().await.is_some() {
+ // Legacy graph (loaded from a pre-field-stamp snapshot, so no
+ // pinned field): keep the historical per-doc first-field insert
+ // so its maintenance semantics don't shift under a running
+ // process. The HNSW query path never engages for these graphs
+ // (field gate), so this can't affect search results.
for (_field, val) in obj {
if let Some(arr) = val.as_array() {
- // Detect: all elements are numbers.
let all_numbers = !arr.is_empty() && arr.iter().all(|v| v.is_number());
if !all_numbers {
continue;
}
-
let vector: Vec<f32> = arr
.iter()
.filter_map(|v| v.as_f64().map(|f| f as f32))
.collect();
-
- let dim = vector.len();
- if dim == 0 {
+ if vector.is_empty() {
continue;
}
-
- // Ensure HNSW index exists with matching dim, or create it.
- let node_id = self.hnsw_next_id.fetch_add(1, Ordering::Relaxed);
- {
- let mut hnsw_guard = self.hnsw.write().await;
- if hnsw_guard.is_none() {
- let params = HnswParams::new(dim, DistanceMetric::Cosine);
- *hnsw_guard = Some(HnswIndex::new(params));
- }
- if let Some(ref hnsw) = *hnsw_guard {
- if hnsw.params().dim != dim {
- // Mismatched dimension — skip silently.
- continue;
- }
- if let Err(e) = hnsw.insert(node_id, vector) {
- warn!(doc_id, error = %e, "HNSW insert failed");
- continue;
- }
- }
+ if self.hnsw_insert_vector(doc_id, vector).await {
+ // Only use the first vector field found.
+ break;
}
+ // Mismatched dimension — legacy behavior falls through
+ // to the next candidate field.
+ }
+ }
+ return;
+ }
- // Update id maps.
- let mut id_map = self.hnsw_id_map.write().await;
- let mut id_rev = self.hnsw_id_rev.write().await;
- id_map.insert(doc_id.to_string(), node_id);
- id_rev.insert(node_id, doc_id.to_string());
- // Only use the first vector field found.
- break;
+ // First vector-bearing doc: decide the field, then pin it under the
+ // write lock (re-checking, so concurrent deciders agree on one).
+ let decided = match self.choose_hnsw_field(source).await {
+ Some(f) => f,
+ None => return,
+ };
+ let field = {
+ let mut fg = self.hnsw_field.write().unwrap();
+ match fg.as_ref() {
+ Some(existing) => existing.clone(),
+ None => {
+ *fg = Some(decided.clone());
+ decided
}
}
+ };
+ if let Some(vector) = extract_numeric_vector(source, &field) {
+ self.hnsw_insert_vector(doc_id, vector).await;
+ }
+ }
+
+ /// Pick the field the HNSW graph will index, given the first
+ /// vector-bearing document. Preference order:
+ /// 1. the first schema dense_vector field this doc carries as an
+ /// all-numbers array (the bench PUTs an explicit dense_vector
+ /// mapping for `v`);
+ /// 2. the first schema dense_vector field (pin to the mapping even if
+ /// this particular doc lacks it);
+ /// 3. no dense_vector mapping at all → the doc's first top-level
+ /// all-numbers array field (legacy heuristic).
+ async fn choose_hnsw_field(&self, source: &Value) -> Option<String> {
+ let dv_fields: Vec<String> = {
+ let schema = self.schema.read().await;
+ collect_dense_vector_fields(&schema.schema)
+ };
+ if let Some(f) = dv_fields
+ .iter()
+ .find(|f| extract_numeric_vector(source, f).is_some())
+ {
+ return Some(f.clone());
+ }
+ if let Some(f) = dv_fields.first() {
+ return Some(f.clone());
}
+ source.as_object().and_then(|obj| {
+ obj.iter().find_map(|(name, val)| match val {
+ Value::Array(arr) if !arr.is_empty() && arr.iter().all(|v| v.is_number()) => {
+ Some(name.clone())
+ }
+ _ => None,
+ })
+ })
+ }
+
+ /// Insert one doc's vector into the HNSW graph (creating the graph on
+ /// first use), tombstoning any previous node for the same doc_id.
+ /// Returns false when the vector was skipped (empty / dim mismatch /
+ /// insert error) — the doc then has no `hnsw_id_map` entry, so the
+ /// coverage guard keeps the kNN query path on brute force.
+ async fn hnsw_insert_vector(&self, doc_id: &str, vector: Vec<f32>) -> bool {
+ let dim = vector.len();
+ if dim == 0 {
+ return false;
+ }
+ // UPDATE staleness fix: a doc_id that already has a node must have
+ // that node tombstoned before the fresh vector goes in (fresh
+ // node_id; the old node is never unmarked).
+ let old_node: Option<u64> = self.hnsw_id_map.read().await.get(doc_id).copied();
+ let node_id = self.hnsw_next_id.fetch_add(1, Ordering::Relaxed);
+ {
+ let mut hnsw_guard = self.hnsw.write().await;
+ if hnsw_guard.is_none() {
+ let params = HnswParams::new(dim, DistanceMetric::Cosine);
+ *hnsw_guard = Some(HnswIndex::new(params));
+ }
+ if let Some(ref hnsw) = *hnsw_guard {
+ if hnsw.params().dim != dim {
+ // Mismatched dimension — skip (no id_map entry → the
+ // coverage guard keeps queries on brute force).
+ return false;
+ }
+ if let Some(old) = old_node {
+ hnsw.mark_deleted(old);
+ }
+ if let Err(e) = hnsw.insert(node_id, vector) {
+ warn!(doc_id, error = %e, "HNSW insert failed");
+ return false;
+ }
+ }
+ }
+ // Update id maps.
+ let mut id_map = self.hnsw_id_map.write().await;
+ let mut id_rev = self.hnsw_id_rev.write().await;
+ if let Some(old) = old_node {
+ id_rev.remove(&old);
+ }
+ id_map.insert(doc_id.to_string(), node_id);
+ id_rev.insert(node_id, doc_id.to_string());
+ true
}
/// Perform KNN search over the HNSW index.
@@ -3527,6 +3706,143 @@ impl Index {
self.noop_update_count.load(Ordering::Relaxed)
}
+ /// Approximate (HNSW) top-level kNN. Returns `None` on ANY
+ /// ineligibility — the caller then falls back to the exact brute-force
+ /// scan, so this path can only ever make eligible queries faster, never
+ /// change what an ineligible query returns.
+ ///
+ /// Eligibility (ALL must hold; the caller has already peeled the query
+ /// and established `filter == None`, non-nested):
+ /// * declared similarity is `cosine`, matching the graph metric;
+ /// * the graph is field-pinned to exactly the queried field and not
+ /// flush-time stale (see `hnsw_field` / `hnsw_stale`);
+ /// * the field does not opt into SQ8 (`scalar8` keeps its dedicated
+ /// code-store scoring path);
+ /// * `doc_count >= HNSW_MIN_DOCS` — small indexes (every ES-YAML
+ /// vector test is ≤ a few hundred docs) keep today's byte-identical
+ /// exact path;
+ /// * coverage: `hnsw_id_map.len() == doc_count`, i.e. every live doc
+ /// has a graph entry. Racy reads degrade to brute, never to wrong
+ /// results.
+ ///
+ /// Results are EXACT-RESCORED: each of the ≤k candidate docs is fetched
+ /// (metrics-free, so `indices.stats` get counters stay ES-parity) and
+ /// re-scored with `compute_vector_similarity` (f64) against the stored
+ /// vector, so `_score`s are byte-identical to the brute/script_score
+ /// paths. Accepted divergences from brute (commit body documents them):
+ /// tie order is graph-return order, and `total.value` may be < k when
+ /// the graph returns fewer candidates (ES ANN behavior class; no pad).
+ async fn run_knn_hnsw(
+ &self,
+ request: &SearchRequest,
+ field: &str,
+ query_vec: &[f32],
+ k: usize,
+ num_candidates: Option<usize>,
+ similarity: &str,
+ ) -> Option<SearchResult> {
+ let started = std::time::Instant::now();
+
+ if similarity != "cosine" {
+ return None;
+ }
+ if self.hnsw_stale.load(Ordering::Acquire) {
+ return None;
+ }
+ {
+ let fg = self.hnsw_field.read().unwrap();
+ if fg.as_deref() != Some(field) {
+ return None;
+ }
+ }
+ {
+ let schema = self.schema.read().await;
+ if lookup_vector_quantization(&schema.schema, field).as_deref() == Some("scalar8") {
+ return None;
+ }
+ }
+ let doc_count = self.doc_count.load(Ordering::Relaxed);
+ if doc_count < HNSW_MIN_DOCS {
+ return None;
+ }
+ if self.hnsw_id_map.read().await.len() as u64 != doc_count {
+ return None;
+ }
+
+ // ES 8.13 defaults num_candidates to 1.5*k when omitted; the beam
+ // width never drops below k.
+ let ef = num_candidates
+ .unwrap_or(((3 * k) / 2).max(k))
+ .max(k)
+ .max(HNSW_EF_FLOOR);
+
+ // LOCK DISCIPLINE: hold the graph read guard only for the sync
+ // sub-ms beam search and copy the results out — index_vectors takes
+ // `hnsw.write().await` per ingested doc, so holding this guard
+ // across the doc-fetch awaits below would head-of-line-block vector
+ // ingest (and, via writer preference, subsequent readers): exactly
+ // the mixed-read-under-write failure class.
+ let raw: Vec<(u64, f32)> = {
+ let hnsw_guard = self.hnsw.read().await;
+ let hnsw = hnsw_guard.as_ref()?;
+ let params = hnsw.params();
+ if params.metric != DistanceMetric::Cosine || params.dim != query_vec.len() {
+ return None;
+ }
+ match hnsw.search(query_vec, k, ef) {
+ Ok(r) => r,
+ Err(_) => return None,
+ }
+ };
+
+ // node_id → doc_id under a short id_rev read. Nodes without a rev
+ // entry were deleted between search and mapping — skip them (brute
+ // would not return them either). Dedup defensively: a save/update
+ // race can briefly leave two live nodes for one doc.
+ let doc_ids: Vec<String> = {
+ let id_rev = self.hnsw_id_rev.read().await;
+ let mut seen: HashSet<&str> = HashSet::with_capacity(raw.len());
+ raw.iter()
+ .filter_map(|(node_id, _)| id_rev.get(node_id))
+ .filter(|doc_id| seen.insert(doc_id.as_str()))
+ .cloned()
+ .collect()
+ };
+
+ // Hydrate + exact-rescore. Any fetch/extract anomaly → whole-request
+ // brute fallback (never a partial page). This also covers
+ // deleted-doc races and stale-restart deletes: the metrics-free
+ // fetch consults the version map first and returns None for
+ // tombstoned ids.
+ let chunk_field = format!("{field}_chunks");
+ let mut scored: Vec<(String, f32, Value)> = Vec::with_capacity(doc_ids.len());
+ for doc_id in doc_ids {
+ let src = match self.get_document_uncounted(&doc_id).await {
+ Ok(Some(s)) => s,
+ _ => return None,
+ };
+ // Multi-chunk semantic docs score by best passage in brute —
+ // single-vector rescore would diverge, so hand the request back.
+ if get_field_value(&src, &chunk_field).is_some() {
+ return None;
+ }
+ let doc_vec: Vec<f32> = match get_field_value(&src, field) {
+ Some(Value::Array(arr)) => arr
+ .iter()
+ .filter_map(|v| v.as_f64().map(|f| f as f32))
+ .collect(),
+ _ => return None,
+ };
+ if doc_vec.len() != query_vec.len() {
+ return None;
+ }
+ let score = compute_vector_similarity(similarity, query_vec, &doc_vec);
+ scored.push((doc_id, score, src));
+ }
+
+ Some(knn_result_from_scored(request, scored, k, started))
+ }
+
/// Brute-force exact KNN against every doc's stored source.
///
/// We iterate (memtable ∪ segments.stored) once, extract the vector
@@ -3590,17 +3906,27 @@ impl Index {
Some(s) => s.to_string(),
None => continue,
};
- if !seen.insert(id.clone()) {
- continue;
+ if let Some(ver) = self.store.version_map.get(&id) {
+ // Skip tombstoned (deleted) docs.
+ if ver.deleted {
+ continue;
+ }
+ // Superseded stale copy (same predicate as the b8 T2
+ // fix in the count path): an updated doc appears in
+ // both its old and new segments, and first-seen dedup
+ // over oldest-first segment order would resurrect the
+ // PRE-update vector (live-verified 2026-07-12: update
+ // + flush + kNN returned the old vector as if the
+ // update never happened). Skip BEFORE `seen` so the
+ // stale copy can't shadow the live one. Legacy docs
+ // without `_seq_no` keep the first-seen dedup.
+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {
+ if doc_seq < ver.seq_no {
+ continue;
+ }
+ }
}
- // Skip tombstoned (deleted) docs.
- if self
- .store
- .version_map
- .get(&id)
- .map(|v| v.deleted)
- .unwrap_or(false)
- {
+ if !seen.insert(id.clone()) {
continue;
}
// Reassembled segment docs have shape
@@ -3786,52 +4112,9 @@ impl Index {
}
// ── Rank, cap the candidate pool at k, then paginate ──────────
- scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
- // `k` bounds the kNN candidate pool; `from`/`size` then window into
- // it, exactly like an ES top-level knn (returned hits are the
- // size-slice of the top-k, NOT k itself). Without this, a
- // `{"knn"/"semantic": {k: 5}}` with `"size": 3` returned 5 hits.
- // `size == 0` is a count-only request (total only, no hits).
- scored.truncate(k.max(1));
- // ES reports `hits.total.value` for a knn/semantic query as the size
- // of the retrieved neighbor pool (min(k, matches)), NOT the number of
- // docs that merely have a vector — so compute it AFTER the truncate.
- // (Hybrid/RRF ignores this total and recomputes its own from the fused
- // list, so bounding it here is safe for that path.)
- let total_value = scored.len() as u64;
- let hits: Vec<Hit> = if request.size == 0 {
- Vec::new()
- } else {
- scored
- .into_iter()
- .skip(request.from)
- .take(request.size)
- .collect::<Vec<_>>()
- }
- .into_iter()
- .map(|(id, score, source)| Hit {
- id,
- score,
- source,
- sort: Vec::new(),
- explain: None,
- highlight: None,
- matched_queries: Vec::new(),
- })
- .collect();
-
- Ok(SearchResult {
- hits,
- total: TotalHits {
- value: total_value,
- relation: TotalHitsRelation::Eq,
- },
- took_ms: started.elapsed().as_millis() as u64,
- aggs: None,
- timed_out: false,
- profile: None,
- max_score: None,
- })
+ // (shared with the HNSW path so hits format / total semantics
+ // cannot drift between the exact and approximate executors)
+ Ok(knn_result_from_scored(request, scored, k, started))
}
/// Brute-force nested KNN: score each parent by the best (max)
@@ -4195,16 +4478,20 @@ impl Index {
Some(s) => s.to_string(),
None => continue,
};
- if !seen.insert(id.clone()) {
- continue;
+ if let Some(ver) = self.store.version_map.get(&id) {
+ if ver.deleted {
+ continue;
+ }
+ // Superseded stale copy — same guard as the top-level
+ // kNN collector (see run_knn_brute_force): never let an
+ // old segment's pre-update copy shadow the live one.
+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {
+ if doc_seq < ver.seq_no {
+ continue;
+ }
+ }
}
- if self
- .store
- .version_map
- .get(&id)
- .map(|v| v.deleted)
- .unwrap_or(false)
- {
+ if !seen.insert(id.clone()) {
continue;
}
let mut doc = doc.clone();
@@ -4371,10 +4658,25 @@ impl Index {
// infrequently (once per flush). The shape is fixed so a v0.7
// codec swap (e.g. to a binary keyed by doc_id length) won't
// break loaders that read both fields.
+ //
+ // Integrity stamp (2026-07): `field` records the graph's pinned
+ // field identity and `seq_no` the store's WAL position at save
+ // time. At load, a fielded snapshot whose stamp differs from the
+ // replayed `current_seq_no()` is flush-time stale relative to a WAL
+ // tail (which may contain updates of already-graphed docs that the
+ // coverage guard cannot detect) → the HNSW query path is disabled.
+ // `stale` persists an already-detected staleness so a later flush
+ // of the still-stale graph cannot re-stamp it as fresh. Loaders are
+ // duck-typed in both directions: old binaries ignore the extra
+ // keys, and this loader treats a missing `field` as legacy.
let id_map = self.hnsw_id_map.read().await;
+ let hnsw_field = self.hnsw_field.read().unwrap().clone();
let snapshot = serde_json::json!({
"next_id": self.hnsw_next_id.load(Ordering::Relaxed),
"map": *id_map,
+ "field": hnsw_field,
+ "seq_no": self.store.current_seq_no(),
+ "stale": self.hnsw_stale.load(Ordering::Acquire),
});
let tmp = self.hnsw_ids_path().with_extension("tmp");
if let Err(e) = std::fs::write(&tmp, serde_json::to_vec(&snapshot).unwrap_or_default()) {
@@ -4409,11 +4711,20 @@ impl Index {
drop(id_map);
drop(id_rev);
self.hnsw_next_id.store(loaded.next_id, Ordering::Relaxed);
+ // Same field-identity + freshness-stamp handling as Index::open.
+ let stale = match (loaded.field.as_ref(), loaded.seq_no) {
+ (Some(_), Some(stamp)) => loaded.stale || stamp != self.store.current_seq_no(),
+ (Some(_), None) => true,
+ (None, _) => false,
+ };
+ *self.hnsw_field.write().unwrap() = loaded.field;
+ self.hnsw_stale
+ .store(stale, std::sync::atomic::Ordering::Release);
let nodes = loaded.graph.len();
*self.hnsw.write().await = Some(loaded.graph);
info!(
nodes,
- "HNSW reloaded from disk — skipping WAL replay rebuild"
+ stale, "HNSW reloaded from disk — skipping WAL replay rebuild"
);
Ok(true)
}
@@ -4482,27 +4793,45 @@ impl Index {
/// Retrieve a document by its string ID.
///
/// Checks the memtable first, then searches on-disk segments.
+ /// Counts every GET, regardless of the result, so indices.stats
+ /// reflects the true ES `get.total` counter (which includes
+ /// missing + exists).
pub async fn get_document(&self, id: &str) -> Result<Option<Value>> {
- // Count every GET, regardless of the result, so indices.stats
- // reflects the true ES `get.total` counter (which includes
- // missing + exists).
let get_started = std::time::Instant::now();
self.metric_get_count.fetch_add(1, Ordering::Relaxed);
+ let res = self.get_document_uncounted(id).await;
+ match &res {
+ Ok(Some(_)) => {
+ self.metric_get_exists_count.fetch_add(1, Ordering::Relaxed);
+ }
+ Ok(None) => {
+ self.metric_get_missing_count
+ .fetch_add(1, Ordering::Relaxed);
+ }
+ Err(_) => {}
+ }
+ self.metric_get_total_ms
+ .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);
+ res
+ }
+
+ /// Metrics-free internal fetch: `get_document` minus the get.total /
+ /// exists / missing counters. Used by the HNSW kNN hydration path —
+ /// counting k internal fetches per kNN query would inflate
+ /// `indices.stats` get.total by k per search, a wire-visible ES
+ /// divergence (ES does not count knn hydration as gets).
+ ///
+ /// Same lookup semantics as `get_document`: version map first
+ /// (tombstoned ids → None), then memtable, then the B2 primary-key
+ /// fast path, then the legacy full-segment scan.
+ async fn get_document_uncounted(&self, id: &str) -> Result<Option<Value>> {
// Check version map — if the document was deleted, return None.
if let Some(entry) = self.store.version_map.get(id) {
if entry.deleted {
- self.metric_get_missing_count
- .fetch_add(1, Ordering::Relaxed);
- self.metric_get_total_ms
- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);
return Ok(None);
}
} else {
// Document not in version map at all.
- self.metric_get_missing_count
- .fetch_add(1, Ordering::Relaxed);
- self.metric_get_total_ms
- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);
return Ok(None);
}
@@ -4510,9 +4839,6 @@ impl Index {
{
let mem = &*self.memtable;
if mem.contains(id) {
- self.metric_get_exists_count.fetch_add(1, Ordering::Relaxed);
- self.metric_get_total_ms
- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);
return Ok(mem.get_doc_source_as_value(id));
}
}
@@ -4544,12 +4870,6 @@ impl Index {
// as the KNN cache — see ffd49ac.
if let Ok(doc) = serde_json::from_slice::<Value>(slice) {
if doc.get("_id").and_then(Value::as_str) == Some(id) {
- self.metric_get_exists_count
- .fetch_add(1, Ordering::Relaxed);
- self.metric_get_total_ms.fetch_add(
- get_started.elapsed().as_millis() as u64,
- Ordering::Relaxed,
- );
return Ok(doc.get("_source").cloned());
}
}
@@ -4581,11 +4901,6 @@ impl Index {
if let Ok(docs) = serde_json::from_slice::<Vec<Value>>(&stored_bytes) {
for doc in docs {
if doc.get("_id").and_then(Value::as_str) == Some(id) {
- self.metric_get_exists_count.fetch_add(1, Ordering::Relaxed);
- self.metric_get_total_ms.fetch_add(
- get_started.elapsed().as_millis() as u64,
- Ordering::Relaxed,
- );
return Ok(doc.get("_source").cloned());
}
}
@@ -4595,10 +4910,6 @@ impl Index {
}
drop(snap);
- self.metric_get_missing_count
- .fetch_add(1, Ordering::Relaxed);
- self.metric_get_total_ms
- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);
Ok(None)
}
@@ -5137,13 +5448,28 @@ impl Index {
// A Knn filter (sub-query) is applied as a doc_matches_query
// pre-filter before the top-k extraction so filter semantics
// match ES.
- // Top-level pure-knn is exact brute force with a pre-filter only, so
- // `num_candidates` (the ANN fan-out) has no effect and is ignored here.
- if let Some((field, query_vec, k, _num_candidates, filter_opt)) = peel_knn_query(query) {
+ //
+ // Unfiltered top-level knn first attempts the HNSW ANN path
+ // (`run_knn_hnsw`), which exact-rescores its ≤k candidates so
+ // `_score`s stay byte-identical to brute force; ANY ineligibility
+ // (field not pinned to the graph, stale snapshot, partial coverage,
+ // non-cosine, SQ8, small index, fetch race, …) falls back to the
+ // exact brute-force scan below. Filtered knn stays exact brute
+ // force with a pre-filter, where `num_candidates` (the ANN fan-out)
+ // has no effect.
+ if let Some((field, query_vec, k, num_candidates, filter_opt)) = peel_knn_query(query) {
let similarity = {
let schema = self.schema.read().await;
lookup_vector_similarity(&schema.schema, &field)
};
+ if filter_opt.is_none() {
+ if let Some(result) = self
+ .run_knn_hnsw(request, &field, &query_vec, k, num_candidates, &similarity)
+ .await
+ {
+ return Ok(result);
+ }
+ }
return self
.run_knn_brute_force(request, &field, &query_vec, k, filter_opt, &similarity)
.await;
@@ -14401,6 +14727,15 @@ struct LoadedHnsw {
id_map: HashMap<String, u64>,
id_rev: HashMap<u64, String>,
next_id: u64,
+ /// Pinned field identity stamped into ids.json at save time.
+ /// `None` = legacy snapshot (pre-field-stamp): the HNSW query path
+ /// never engages for it (field gate) but ingest maintenance continues.
+ field: Option<String>,
+ /// `store.current_seq_no()` at save time; `None` on legacy snapshots.
+ seq_no: Option<u64>,
+ /// Persisted staleness marker (a stale graph re-saved by a later flush
+ /// must not come back fresh just because its seq stamp caught up).
+ stale: bool,
}
fn load_hnsw_artifacts_sync(hnsw_dir: &Path) -> Option<LoadedHnsw> {
@@ -14446,11 +14781,23 @@ fn load_hnsw_artifacts_sync(hnsw_dir: &Path) -> Option<LoadedHnsw> {
id_rev.insert(nid, doc_id.clone());
}
}
+ // Integrity stamp (see save_hnsw_to_disk). All three keys are optional
+ // so legacy snapshots keep loading: missing `field` → legacy graph
+ // (query path disabled by the field gate, ingest behavior retained).
+ let field = ids
+ .get("field")
+ .and_then(|v| v.as_str())
+ .map(|s| s.to_string());
+ let seq_no = ids.get("seq_no").and_then(|v| v.as_u64());
+ let stale = ids.get("stale").and_then(|v| v.as_bool()).unwrap_or(false);
Some(LoadedHnsw {
graph,
id_map,
id_rev,
next_id,
+ field,
+ seq_no,
+ stale,
})
}
@@ -14723,6 +15070,119 @@ fn compute_vector_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {
result as f32
}
+/// Minimum live doc count before the HNSW query path may serve a kNN
+/// request. Below this the exact brute-force scan is already fast and — more
+/// importantly — every ES-YAML vector test (≤ a few hundred docs) keeps
+/// today's byte-identical exact path.
+const HNSW_MIN_DOCS: u64 = 1024;
+
+/// Lower bound on the HNSW beam width (`ef`).
+///
+/// DOCUMENTED DIVERGENCE from ES's `num_candidates` contract: Lucene runs
+/// `num_candidates` PER SEGMENT — on the 50k-doc benchmark corpus ES holds
+/// ~8 segments, so `num_candidates: 100` really beams over ~800 candidates
+/// index-wide. XERJ serves one graph for the whole index, so a literal
+/// ef=100 explores 8× less than ES does for the same request (measured
+/// recall@10 0.53 vs ES 0.937 at nc=100 on 50k×128-d random vectors).
+/// Flooring ef at 800 restores the equivalent index-wide beam effort:
+/// measured mean recall@10 0.98 / min 0.90 (100 seeded probes), beam cost
+/// ~1.1 ms. A user-supplied num_candidates above the floor is honored.
+const HNSW_EF_FLOOR: usize = 800;
+
+/// Shared tail of every top-level kNN executor (brute force and HNSW):
+/// rank by score desc, cap the candidate pool at `k`, window with
+/// `from`/`size`, and shape the response. Centralised so the exact and
+/// approximate paths cannot drift on hits format or total semantics.
+fn knn_result_from_scored(
+ request: &SearchRequest,
+ mut scored: Vec<(String, f32, Value)>,
+ k: usize,
+ started: std::time::Instant,
+) -> SearchResult {
+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+ // `k` bounds the kNN candidate pool; `from`/`size` then window into
+ // it, exactly like an ES top-level knn (returned hits are the
+ // size-slice of the top-k, NOT k itself). Without this, a
+ // `{"knn"/"semantic": {k: 5}}` with `"size": 3` returned 5 hits.
+ // `size == 0` is a count-only request (total only, no hits).
+ scored.truncate(k.max(1));
+ // ES reports `hits.total.value` for a knn/semantic query as the size
+ // of the retrieved neighbor pool (min(k, matches)), NOT the number of
+ // docs that merely have a vector — so compute it AFTER the truncate.
+ // (Hybrid/RRF ignores this total and recomputes its own from the fused
+ // list, so bounding it here is safe for that path.)
+ let total_value = scored.len() as u64;
+ let hits: Vec<Hit> = if request.size == 0 {
+ Vec::new()
+ } else {
+ scored
+ .into_iter()
+ .skip(request.from)
+ .take(request.size)
+ .collect::<Vec<_>>()
+ }
+ .into_iter()
+ .map(|(id, score, source)| Hit {
+ id,
+ score,
+ source,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ })
+ .collect();
+
+ SearchResult {
+ hits,
+ total: TotalHits {
+ value: total_value,
+ relation: TotalHitsRelation::Eq,
+ },
+ took_ms: started.elapsed().as_millis() as u64,
+ aggs: None,
+ timed_out: false,
+ profile: None,
+ max_score: None,
+ }
+}
+
+/// Extract `field` from `source` as an all-numbers f32 vector.
+/// Returns `None` when the field is absent, not an array, empty, or holds
+/// any non-number element.
+fn extract_numeric_vector(source: &Value, field: &str) -> Option<Vec<f32>> {
+ match get_field_value(source, field) {
+ Some(Value::Array(arr)) if !arr.is_empty() && arr.iter().all(|v| v.is_number()) => Some(
+ arr.iter()
+ .filter_map(|v| v.as_f64().map(|f| f as f32))
+ .collect(),
+ ),
+ _ => None,
+ }
+}
+
+/// Collect every dense_vector-mapped field in schema order (dotted paths
+/// for fields nested under object mappings).
+fn collect_dense_vector_fields(schema: &Schema) -> Vec<String> {
+ fn walk(fields: &[FieldConfig], prefix: &str, out: &mut Vec<String>) {
+ for fc in fields {
+ let path = if prefix.is_empty() {
+ fc.name.clone()
+ } else {
+ format!("{prefix}.{}", fc.name)
+ };
+ if matches!(fc.field_type, FieldType::Vector) {
+ out.push(path);
+ } else if !fc.fields.is_empty() {
+ walk(&fc.fields, &path, out);
+ }
+ }
+ }
+ let mut out = Vec::new();
+ walk(&schema.fields, "", &mut out);
+ out
+}
+
/// Unwrap a top-level Knn query, optionally nested under a single transparent
/// wrapper (Boosted, Constant, Named, or a single-clause Bool must/should).
/// Returns `(field, query_vector, k, filter)` when the query is a KNN
diff --git a/engine/crates/xerj-vector/examples/recall_probe.rs b/engine/crates/xerj-vector/examples/recall_probe.rs
new file mode 100644
index 000000000..0567f2897
--- /dev/null
+++ b/engine/crates/xerj-vector/examples/recall_probe.rs
@@ -0,0 +1,124 @@
+//! Isolated HNSW recall + graph-structure diagnostic (dev tool, not shipped).
+//!
+//! Builds the same corpus shape as the official kNN bench cell (50k x 128-d
+//! uniform random, cosine), then reports recall@10 at several ef values and
+//! layer-0 reachability from the entry point. Run with:
+//! cargo run --release -p xerj-vector --example recall_probe [N] [DIM]
+
+use xerj_vector::distance::DistanceMetric;
+use xerj_vector::hnsw::{HnswIndex, HnswParams};
+
+// Deterministic RNG (mulberry32, same family as the JS bench).
+struct Rng(u32);
+impl Rng {
+ fn next_f64(&mut self) -> f64 {
+ self.0 = self.0.wrapping_add(0x6D2B79F5);
+ let mut t = self.0 as u64;
+ t = (t ^ (t >> 15)).wrapping_mul(1 | t);
+ t ^= t.wrapping_add((t ^ (t >> 7)).wrapping_mul(61 | t));
+ ((t ^ (t >> 14)) & 0xFFFF_FFFF) as f64 / 4294967296.0
+ }
+}
+
+fn cosine(a: &[f32], b: &[f32]) -> f32 {
+ let (mut d, mut na, mut nb) = (0f32, 0f32, 0f32);
+ for i in 0..a.len() {
+ d += a[i] * b[i];
+ na += a[i] * a[i];
+ nb += b[i] * b[i];
+ }
+ d / (na.sqrt() * nb.sqrt()).max(1e-12)
+}
+
+fn main() {
+ let args: Vec<String> = std::env::args().collect();
+ let n: usize = args.get(1).map(|s| s.parse().unwrap()).unwrap_or(50_000);
+ let dim: usize = args.get(2).map(|s| s.parse().unwrap()).unwrap_or(128);
+ let k = 10;
+
+ let mut rng = Rng(0xC0FFEE);
+ let vecs: Vec<Vec<f32>> = (0..n)
+ .map(|_| {
+ (0..dim)
+ .map(|_| (rng.next_f64() * 2.0 - 1.0) as f32)
+ .collect()
+ })
+ .collect();
+
+ let idx = HnswIndex::new(HnswParams::new(dim, DistanceMetric::Cosine));
+ let t0 = std::time::Instant::now();
+ for (i, v) in vecs.iter().enumerate() {
+ idx.insert(i as u64 + 1, v.clone()).unwrap();
+ if (i + 1) % 10_000 == 0 {
+ eprintln!("inserted {} in {:?}", i + 1, t0.elapsed());
+ }
+ }
+ eprintln!("build done in {:?}", t0.elapsed());
+
+ // 20 probe queries from a different seed.
+ let mut qrng = Rng(0xBEEF);
+ let queries: Vec<Vec<f32>> = (0..20)
+ .map(|_| {
+ (0..dim)
+ .map(|_| (qrng.next_f64() * 2.0 - 1.0) as f32)
+ .collect()
+ })
+ .collect();
+
+ // ── Graph-structure diagnostics via the debug stats hook ──
+ let (ep, ep_layer, layer_hist, deg0_hist, reach0) = idx.debug_structure();
+ eprintln!("entry_point={ep:?} entry_layer={ep_layer}");
+ eprintln!("layer histogram (level → count): {layer_hist:?}");
+ eprintln!("layer-0 out-degree histogram (bucket → count): {deg0_hist:?}");
+ eprintln!("layer-0 BFS reachable from entry: {reach0} / {n}");
+
+ // Pre-compute exact ground truth per query.
+ let exact_sets: Vec<(std::collections::HashSet<u64>, u64, f32)> = queries
+ .iter()
+ .map(|q| {
+ let mut scored: Vec<(usize, f32)> = vecs
+ .iter()
+ .enumerate()
+ .map(|(i, v)| (i, cosine(q, v)))
+ .collect();
+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+ let set: std::collections::HashSet<u64> =
+ scored[..k].iter().map(|&(i, _)| i as u64 + 1).collect();
+ (set, scored[0].0 as u64 + 1, 1.0 - scored[0].1) // (top-k set, true-NN id, true-NN distance)
+ })
+ .collect();
+
+ // Descent diagnostics: where does the hierarchy drop us at layer 0?
+ let mut dstats = Vec::new();
+ for (q, (_, _nn, nnd)) in queries.iter().zip(&exact_sets) {
+ if let Some((_start, d)) = idx.debug_descent(q) {
+ dstats.push((d, *nnd));
+ }
+ }
+ let avg_start: f32 = dstats.iter().map(|x| x.0).sum::<f32>() / dstats.len() as f32;
+ let avg_nn: f32 = dstats.iter().map(|x| x.1).sum::<f32>() / dstats.len() as f32;
+ eprintln!("descent: avg layer-0 start dist={avg_start:.4} avg true-NN dist={avg_nn:.4} (random pair ~= 1.0)");
+
+ for ef in [100usize, 200, 400, 800] {
+ let (mut recalls, mut oracle_recalls) = (Vec::new(), Vec::new());
+ let mut total_ms = 0.0;
+ for (q, (exact, nn_id, _)) in queries.iter().zip(&exact_sets) {
+ let t = std::time::Instant::now();
+ let got = idx.search(q, k, ef).unwrap();
+ total_ms += t.elapsed().as_secs_f64() * 1e3;
+ let hit = got.iter().filter(|(id, _)| exact.contains(id)).count();
+ recalls.push(hit as f32 / k as f32);
+ // Oracle start: beam from the TRUE nearest neighbor.
+ let got_o = idx.debug_search_from(*nn_id, q, k, ef);
+ let hit_o = got_o.iter().filter(|(id, _)| exact.contains(id)).count();
+ oracle_recalls.push(hit_o as f32 / k as f32);
+ }
+ let mean: f32 = recalls.iter().sum::<f32>() / recalls.len() as f32;
+ let min = recalls.iter().cloned().fold(f32::MAX, f32::min);
+ let omean: f32 = oracle_recalls.iter().sum::<f32>() / oracle_recalls.len() as f32;
+ println!(
+ "ef={ef:4} mean_recall={mean:.4} min_recall={min:.3} oracle_start_recall={omean:.4} avg_search_ms={:.3}",
+ total_ms / queries.len() as f64
+ );
+ }
+}
diff --git a/engine/crates/xerj-vector/src/hnsw.rs b/engine/crates/xerj-vector/src/hnsw.rs
index 04390bab8..059f67db7 100644
--- a/engine/crates/xerj-vector/src/hnsw.rs
+++ b/engine/crates/xerj-vector/src/hnsw.rs
@@ -7,13 +7,25 @@
//! - M=16 bidirectional connections per layer (good accuracy/memory trade-off)
//! - ef_construction=200 (high recall during build)
//! - Level selection uses the standard ln-based formula
+//! - Neighbor selection uses the paper's Algorithm 4 diversity heuristic
+//! (the variant hnswlib and Lucene use) — plain closest-M packs a node's
+//! links into one tight cluster and measurably breaks navigability
//! - Graph is entirely in-memory; persistence is handled by the storage crate
+//!
+//! In-memory layout (2026-07-12 flat-slab rework): nodes live in a single
+//! slot-indexed slab — one contiguous `Vec<f32>` for vectors, `Vec<Vec<u32>>`
+//! neighbor lists per layer, and a `Vec<u64>` bitmap for the search-time
+//! visited set. The previous `HashMap<u64, Arc<RwLock<Node>>>` paid a hash
+//! lookup + two pointer chases + an RwLock acquisition *per neighbor
+//! expansion*, which made beam search ~5× slower than the distance math
+//! itself. The on-disk format and the public (external-u64-id) API are
+//! unchanged.
use std::cmp::Reverse;
use std::collections::{BinaryHeap, HashMap, HashSet};
use std::io::{Read, Write};
use std::path::Path;
-use std::sync::{Arc, RwLock};
+use std::sync::RwLock;
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::Rng;
@@ -67,6 +79,38 @@ fn corrupt(msg: &str) -> XerjError {
XerjError::storage(format!("HNSW graph corrupt: {msg}"))
}
+/// Dot product with eight independent accumulators.
+///
+/// A single-accumulator `for` loop is a strict serial FP dependency chain
+/// that rustc must NOT auto-vectorize (float addition is not associative),
+/// so the beam search was spending ~5 ns/dim in scalar fmadds. Eight
+/// independent chains let LLVM keep multiple SIMD lanes busy; the exact
+/// f32 sum differs from the serial order only in rounding, which is fine
+/// here — graph distances only decide beam ORDER and callers rescore
+/// exact similarities downstream.
+#[inline]
+fn dot_unrolled(a: &[f32], b: &[f32]) -> f32 {
+ let mut acc = [0.0f32; 8];
+ let ca = a.chunks_exact(8);
+ let cb = b.chunks_exact(8);
+ let (ra, rb) = (ca.remainder(), cb.remainder());
+ for (x, y) in ca.zip(cb) {
+ acc[0] += x[0] * y[0];
+ acc[1] += x[1] * y[1];
+ acc[2] += x[2] * y[2];
+ acc[3] += x[3] * y[3];
+ acc[4] += x[4] * y[4];
+ acc[5] += x[5] * y[5];
+ acc[6] += x[6] * y[6];
+ acc[7] += x[7] * y[7];
+ }
+ let mut s = ((acc[0] + acc[4]) + (acc[1] + acc[5])) + ((acc[2] + acc[6]) + (acc[3] + acc[7]));
+ for (x, y) in ra.iter().zip(rb.iter()) {
+ s += x * y;
+ }
+ s
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Parameters
// ─────────────────────────────────────────────────────────────────────────────
@@ -96,31 +140,300 @@ impl HnswParams {
}
// ─────────────────────────────────────────────────────────────────────────────
-// Internal node
+// Flat slab storage
// ─────────────────────────────────────────────────────────────────────────────
-#[derive(Debug)]
-struct Node {
- /// The node's own id. Redundant with its key in the `nodes` map, so it is
- /// never read back out — kept for `Debug` output and crash diagnostics.
- #[allow(dead_code)]
- id: u64,
- vector: Vec<f32>,
- /// Neighbor lists per layer. `neighbors[layer]` holds neighbor node IDs.
- neighbors: Vec<Vec<u64>>,
+/// Slot-indexed node storage. A `slot` is a dense `u32` handle assigned in
+/// insertion order; all hot-path state is `slot`-addressed so beam search
+/// runs on contiguous arrays with no per-node locks or hashing.
+struct Inner {
+ dim: usize,
+ metric: DistanceMetric,
+ /// slot → external node id.
+ ids: Vec<u64>,
+ /// external node id → slot.
+ slot_of: HashMap<u64, u32>,
+ /// Contiguous vector data, `slot * dim ..= slot * dim + dim`.
+ vectors: Vec<f32>,
+ /// slot → 1/‖v‖ (cosine metric only; 1.0 placeholder otherwise,
+ /// 0.0 for zero-norm vectors so their cosine similarity is 0 — the
+ /// same value the previous `cosine()` helper produced).
+ inv_norms: Vec<f32>,
+ /// slot → layer → neighbor slots.
+ neighbors: Vec<Vec<Vec<u32>>>,
+ /// slot → tombstoned? (soft delete; see `HnswIndex::mark_deleted`).
+ tomb: Vec<bool>,
+ /// External-id mirror of the tombstone set (for counting and
+ /// persistence; may contain ids the graph no longer knows).
+ tomb_ids: HashSet<u64>,
+ entry_slot: Option<u32>,
+ entry_layer: usize,
}
-impl Node {
- fn new(id: u64, vector: Vec<f32>, max_layer: usize, m: usize) -> Self {
- let neighbors = (0..=max_layer)
- .map(|l| Vec::with_capacity(if l == 0 { 2 * m } else { m }))
- .collect();
+impl Inner {
+ fn new(dim: usize, metric: DistanceMetric) -> Self {
Self {
- id,
- vector,
- neighbors,
+ dim,
+ metric,
+ ids: Vec::new(),
+ slot_of: HashMap::new(),
+ vectors: Vec::new(),
+ inv_norms: Vec::new(),
+ neighbors: Vec::new(),
+ tomb: Vec::new(),
+ tomb_ids: HashSet::new(),
+ entry_slot: None,
+ entry_layer: 0,
+ }
+ }
+
+ #[inline]
+ fn len(&self) -> usize {
+ self.ids.len()
+ }
+
+ #[inline]
+ fn vec_of(&self, slot: u32) -> &[f32] {
+ let s = slot as usize * self.dim;
+ &self.vectors[s..s + self.dim]
+ }
+
+ /// The `1/‖q‖` scale a query needs for the cosine fast path
+ /// (1.0 for other metrics, 0.0 for a zero query).
+ #[inline]
+ fn query_inv(&self, q: &[f32]) -> f32 {
+ if self.metric == DistanceMetric::Cosine {
+ let n: f32 = q.iter().map(|x| x * x).sum::<f32>().sqrt();
+ if n > 0.0 {
+ 1.0 / n
+ } else {
+ 0.0
+ }
+ } else {
+ 1.0
}
}
+
+ /// Distance from an (external) query vector to a slot.
+ ///
+ /// Cosine uses cached per-slot inverse norms so the per-pair work is a
+ /// single dot product: `1 - dot(q,v)·(1/‖q‖)·(1/‖v‖)`. This matches
+ /// `compute_distance`'s `1 - cos` up to fp rounding — acceptable
+ /// because graph distances only decide beam ORDER; callers rescore
+ /// exact similarities downstream.
+ #[inline]
+ fn slot_dist(&self, q: &[f32], q_inv: f32, slot: u32) -> f32 {
+ let v = self.vec_of(slot);
+ match self.metric {
+ DistanceMetric::Cosine => {
+ 1.0 - dot_unrolled(q, v) * q_inv * self.inv_norms[slot as usize]
+ }
+ _ => compute_distance(self.metric, q, v),
+ }
+ }
+
+ /// Hint the CPU to start pulling `slot`'s vector into cache. The
+ /// vector slab for a 50k × 128-d graph is ~25 MB (larger than L3), so
+ /// beam search reads are DRAM-latency-bound without this; prefetching
+ /// the next neighbor's lines while the current distance computes hides
+ /// most of that latency.
+ ///
+ /// Safety: `_mm_prefetch` is architecturally a hint — it cannot fault,
+ /// and the pointer is formed from an in-bounds slot (neighbor lists
+ /// only hold allocated slots). No-op on non-x86_64.
+ #[inline(always)]
+ fn prefetch_slot(&self, slot: u32) {
+ #[cfg(target_arch = "x86_64")]
+ unsafe {
+ use core::arch::x86_64::{_mm_prefetch, _MM_HINT_T0};
+ let base = self.vectors.as_ptr().add(slot as usize * self.dim) as *const i8;
+ // First two cache lines; the hardware streamer follows the
+ // sequential reads for the rest of the vector.
+ _mm_prefetch(base, _MM_HINT_T0);
+ if self.dim > 16 {
+ _mm_prefetch(base.add(64), _MM_HINT_T0);
+ }
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ let _ = slot;
+ }
+ }
+
+ /// Append a node to the slab. Returns its slot.
+ fn alloc_slot(&mut self, id: u64, vector: Vec<f32>, node_level: usize, m: usize) -> u32 {
+ let slot = self.ids.len() as u32;
+ let inv = if self.metric == DistanceMetric::Cosine {
+ let n: f32 = vector.iter().map(|x| x * x).sum::<f32>().sqrt();
+ if n > 0.0 {
+ 1.0 / n
+ } else {
+ 0.0
+ }
+ } else {
+ 1.0
+ };
+ self.ids.push(id);
+ self.slot_of.insert(id, slot);
+ self.vectors.extend_from_slice(&vector);
+ self.inv_norms.push(inv);
+ self.neighbors.push(
+ (0..=node_level)
+ .map(|l| Vec::with_capacity(if l == 0 { 2 * m } else { m }))
+ .collect(),
+ );
+ self.tomb.push(self.tomb_ids.contains(&id));
+ slot
+ }
+
+ /// Greedy hill-climb to the locally-closest node at `layer`.
+ fn greedy_layer(&self, q: &[f32], q_inv: f32, start: u32, layer: usize) -> (u32, f32) {
+ let mut best = start;
+ let mut best_dist = self.slot_dist(q, q_inv, start);
+ loop {
+ let mut improved = false;
+ let nbrs = &self.neighbors[best as usize];
+ if layer >= nbrs.len() {
+ break;
+ }
+ for &nb in &nbrs[layer] {
+ let d = self.slot_dist(q, q_inv, nb);
+ if d < best_dist {
+ best_dist = d;
+ best = nb;
+ improved = true;
+ }
+ }
+ if !improved {
+ break;
+ }
+ }
+ (best, best_dist)
+ }
+
+ /// Beam search at `layer`, returning up to `ef` candidates sorted by
+ /// distance ascending. `admit` gates entry into the RESULT set only;
+ /// non-admitted nodes are still traversed (excluding them from
+ /// traversal would disconnect their neighbours and collapse recall).
+ fn beam(
+ &self,
+ q: &[f32],
+ q_inv: f32,
+ entry: u32,
+ ef: usize,
+ layer: usize,
+ admit: &dyn Fn(u32) -> bool,
+ ) -> Vec<(f32, u32)> {
+ let n = self.len();
+ if entry as usize >= n {
+ return vec![];
+ }
+ // Bitmap visited set: ~6 KB for a 50k graph, allocation-cheap and
+ // O(1) test-and-set (the old per-search HashSet<u64> hashed every
+ // neighbor expansion).
+ let mut visited = vec![0u64; n.div_ceil(64)];
+ #[inline]
+ fn test_and_set(bits: &mut [u64], i: u32) -> bool {
+ let (w, b) = ((i / 64) as usize, i % 64);
+ let seen = bits[w] & (1 << b) != 0;
+ bits[w] |= 1 << b;
+ seen
+ }
+
+ let entry_dist = self.slot_dist(q, q_inv, entry);
+ // Min-heap of (dist, slot) candidates to explore.
+ let mut candidates: BinaryHeap<Reverse<(ordered_float::OrderedFloat, u32)>> =
+ BinaryHeap::with_capacity(ef * 4);
+ // Max-heap of (dist, slot) for the result set W.
+ let mut w: BinaryHeap<(ordered_float::OrderedFloat, u32)> =
+ BinaryHeap::with_capacity(ef + 1);
+
+ candidates.push(Reverse((ordered_float::OrderedFloat(entry_dist), entry)));
+ if admit(entry) {
+ w.push((ordered_float::OrderedFloat(entry_dist), entry));
+ }
+ test_and_set(&mut visited, entry);
+
+ while let Some(Reverse((dist_c, c))) = candidates.pop() {
+ // Pruning: if the closest unexplored candidate is farther than
+ // the farthest kept result and W is full, stop.
+ if let Some(&(farthest, _)) = w.peek() {
+ if dist_c > farthest && w.len() >= ef {
+ break;
+ }
+ }
+ let nbrs = &self.neighbors[c as usize];
+ if layer >= nbrs.len() {
+ continue;
+ }
+ let list = &nbrs[layer];
+ // Start pulling the first vectors while the loop spins up.
+ for &nb in list.iter().take(2) {
+ self.prefetch_slot(nb);
+ }
+ for (i, &nb) in list.iter().enumerate() {
+ if i + 2 < list.len() {
+ self.prefetch_slot(list[i + 2]);
+ }
+ if test_and_set(&mut visited, nb) {
+ continue;
+ }
+ let d = self.slot_dist(q, q_inv, nb);
+ let farthest_w = w.peek().map(|&(f, _)| f.0).unwrap_or(f32::MAX);
+ if d < farthest_w || w.len() < ef {
+ candidates.push(Reverse((ordered_float::OrderedFloat(d), nb)));
+ if admit(nb) {
+ w.push((ordered_float::OrderedFloat(d), nb));
+ if w.len() > ef {
+ w.pop();
+ }
+ }
+ }
+ }
+ }
+
+ let mut results: Vec<(f32, u32)> = w.into_iter().map(|(d, s)| (d.0, s)).collect();
+ results.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+ results
+ }
+
+ /// Neighbor selection — Malkov & Yashunin Algorithm 4 (the "heuristic"
+ /// variant used by hnswlib and Lucene): walk candidates in ascending
+ /// distance-to-base order and keep a candidate only if it is closer to
+ /// the base than to every already-kept neighbor. Enforces direction
+ /// diversity in each node's link list; plain closest-M packed all links
+ /// into one tight cluster (recall@10 0.565 @ ef=100 on 50k random
+ /// 128-d where the heuristic-built graph reaches ~0.95+).
+ fn select_diverse(&self, cands: &[(f32, u32)], m: usize) -> Vec<(f32, u32)> {
+ let mut sorted = cands.to_vec();
+ sorted.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+ let mut kept: Vec<(f32, u32)> = Vec::with_capacity(m);
+ for &(d_c, c) in &sorted {
+ if kept.len() >= m {
+ break;
+ }
+ let cv = self.vec_of(c);
+ let ci = self.inv_norms[c as usize];
+ // Diverse iff no already-kept neighbor is closer to the
+ // candidate than the base is.
+ let diverse = kept.iter().all(|&(_, r)| self.slot_dist(cv, ci, r) >= d_c);
+ if diverse {
+ kept.push((d_c, c));
+ }
+ }
+ kept
+ }
+
+ /// Distances from `base` to each of its current neighbors at `layer`
+ /// (used when a saturated node re-selects after gaining a back-link).
+ fn backlink_dists(&self, base: u32, layer: usize) -> Vec<(f32, u32)> {
+ let bv = self.vec_of(base);
+ let bi = self.inv_norms[base as usize];
+ self.neighbors[base as usize][layer]
+ .iter()
+ .map(|&nb| (self.slot_dist(bv, bi, nb), nb))
+ .collect()
+ }
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -128,29 +441,24 @@ impl Node {
// ─────────────────────────────────────────────────────────────────────────────
/// A thread-safe HNSW approximate nearest-neighbor index.
+///
+/// Soft deletes: deleting a doc removes its entry from the caller's id
+/// maps but the node and its neighbour edges stay in the graph until a
+/// graph compaction (planned v0.7+). Tombstoned nodes are skipped as
+/// results but still traversed. Tombstones persist with the graph
+/// (format v2+).
pub struct HnswIndex {
params: HnswParams,
- nodes: RwLock<HashMap<u64, Arc<RwLock<Node>>>>,
- entry_point: RwLock<Option<u64>>,
- entry_layer: RwLock<usize>,
- /// Soft-delete bitmap. v0.6.2: deleting a doc removes its
- /// entry from `Index.hnsw_id_map` but the node and its
- /// neighbour edges stay in the graph until a graph compaction
- /// (planned v0.7+). Adding to this set causes search to skip
- /// the node — both as a result and as a traversal target.
- /// Persisted alongside the graph (format version 2+).
- tombstones: RwLock<HashSet<u64>>,
+ inner: RwLock<Inner>,
}
impl HnswIndex {
/// Create a new empty index.
pub fn new(params: HnswParams) -> Self {
+ let inner = Inner::new(params.dim, params.metric);
Self {
params,
- nodes: RwLock::new(HashMap::new()),
- entry_point: RwLock::new(None),
- entry_layer: RwLock::new(0),
- tombstones: RwLock::new(HashSet::new()),
+ inner: RwLock::new(inner),
}
}
@@ -158,24 +466,32 @@ impl HnswIndex {
/// return it as a hit. The node and its neighbour edges remain
/// until a graph compaction (planned v0.7+). Idempotent.
pub fn mark_deleted(&self, id: u64) {
- self.tombstones.write().unwrap().insert(id);
+ let mut g = self.inner.write().unwrap();
+ if let Some(&slot) = g.slot_of.get(&id) {
+ g.tomb[slot as usize] = true;
+ }
+ g.tomb_ids.insert(id);
}
/// Reverse of `mark_deleted` — used in tests and on un-delete /
/// reindex paths.
pub fn unmark_deleted(&self, id: u64) {
- self.tombstones.write().unwrap().remove(&id);
+ let mut g = self.inner.write().unwrap();
+ if let Some(&slot) = g.slot_of.get(&id) {
+ g.tomb[slot as usize] = false;
+ }
+ g.tomb_ids.remove(&id);
}
/// True if `id` has been tombstoned via `mark_deleted`.
pub fn is_deleted(&self, id: u64) -> bool {
- self.tombstones.read().unwrap().contains(&id)
+ self.inner.read().unwrap().tomb_ids.contains(&id)
}
/// Number of tombstones currently held. Operators can monitor
/// this to decide when to schedule a compaction.
pub fn tombstone_count(&self) -> usize {
- self.tombstones.read().unwrap().len()
+ self.inner.read().unwrap().tomb_ids.len()
}
pub fn params(&self) -> &HnswParams {
@@ -184,7 +500,7 @@ impl HnswIndex {
/// Number of indexed vectors.
pub fn len(&self) -> usize {
- self.nodes.read().unwrap().len()
+ self.inner.read().unwrap().len()
}
pub fn is_empty(&self) -> bool {
@@ -200,12 +516,6 @@ impl HnswIndex {
level.min(16) // cap at 16 layers
}
- // ── Distance helper ───────────────────────────────────────────────────
-
- fn dist(&self, a: &[f32], b: &[f32]) -> f32 {
- compute_distance(self.params.metric, a, b)
- }
-
// ── Core HNSW algorithm ───────────────────────────────────────────────
/// Insert a single vector with the given ID.
@@ -225,76 +535,57 @@ impl HnswIndex {
let m = self.params.m;
let ef = self.params.ef_construction;
- // Build the node (not yet inserted into the map)
- let node = Arc::new(RwLock::new(Node::new(id, vector.clone(), node_level, m)));
-
- // Handle first insertion
- {
- let mut ep = self.entry_point.write().unwrap();
- if ep.is_none() {
- *ep = Some(id);
- *self.entry_layer.write().unwrap() = node_level;
- self.nodes.write().unwrap().insert(id, node);
+ let mut g = self.inner.write().unwrap();
+
+ // Query view of the new vector for the construction beams (the
+ // slab owns the canonical copy after alloc).
+ let q = vector.clone();
+ let q_inv = g.query_inv(&q);
+
+ // The new slot has no in-edges yet, so allocating it before the
+ // beams run cannot perturb them — and it means the back-link
+ // re-selection below can score the new node like any other
+ // neighbor (the pre-slab code dropped the fresh back-link on the
+ // floor here, disconnecting the graph: layer-0 BFS from the entry
+ // reached 34 of 50k nodes and recall@10 was 0.000).
+ let slot = g.alloc_slot(id, vector, node_level, m);
+
+ // Handle first insertion.
+ let (ep_slot, ep_layer) = match g.entry_slot {
+ None => {
+ g.entry_slot = Some(slot);
+ g.entry_layer = node_level;
debug!("HNSW: first node id={id} level={node_level}");
return Ok(());
}
- }
-
- // Greedy descent from top layer to node_level+1
- let ep_id = self.entry_point.read().unwrap().unwrap();
- let ep_layer = *self.entry_layer.read().unwrap();
-
- // Greedy descent tracks only the current entry point; the distance to
- // it is recomputed by `search_layer`/`greedy_search_layer` at each step,
- // so we don't carry it between layers.
- let mut curr_ep = ep_id;
+ Some(ep) => (ep, g.entry_layer),
+ };
+ // Greedy descent from the top layer down to node_level+1.
+ let mut curr_ep = ep_slot;
for layer in (node_level + 1..=ep_layer).rev() {
- curr_ep = self.greedy_search_layer(&vector, curr_ep, layer, 1).0;
+ curr_ep = g.greedy_layer(&q, q_inv, curr_ep, layer).0;
}
- // Search and connect from node_level down to 0
+ // Search and connect from min(node_level, ep_layer) down to 0.
for layer in (0..=node_level.min(ep_layer)).rev() {
let m_layer = if layer == 0 { 2 * m } else { m };
- let candidates = self.search_layer(&vector, curr_ep, ef, layer);
-
- // Select M nearest
- let selected = self.select_neighbors(&vector, &candidates, m_layer);
-
- // Connect new node → selected
- {
- let mut n = node.write().unwrap();
- if layer < n.neighbors.len() {
- n.neighbors[layer] = selected.iter().map(|&(_, nid)| nid).collect();
- }
- }
-
- // Update existing nodes: add back-link to new node, prune to m_layer
- {
- let nodes = self.nodes.read().unwrap();
- for &(_, nid) in &selected {
- if let Some(existing) = nodes.get(&nid) {
- let mut en = existing.write().unwrap();
- if layer < en.neighbors.len() {
- en.neighbors[layer].push(id);
- if en.neighbors[layer].len() > m_layer {
- // Re-select neighbors to keep M_layer best
- let ev = en.vector.clone();
- let neighbor_dists: Vec<(f32, u64)> = en.neighbors[layer]
- .iter()
- .filter_map(|&nid2| {
- nodes.get(&nid2).map(|nn| {
- let nv = nn.read().unwrap();
- (self.dist(&ev, &nv.vector), nid2)
- })
- })
- .collect();
- let pruned = self.select_neighbors(&ev, &neighbor_dists, m_layer);
- en.neighbors[layer] =
- pruned.iter().map(|&(_, nid2)| nid2).collect();
- }
- }
- }
+ let candidates = g.beam(&q, q_inv, curr_ep, ef, layer, &|_| true);
+
+ // Select up to M diverse nearest (Algorithm 4 heuristic).
+ let selected = g.select_diverse(&candidates, m_layer);
+
+ // Connect new node → selected.
+ g.neighbors[slot as usize][layer] = selected.iter().map(|&(_, s)| s).collect();
+
+ // Back-links: add new node to each selected neighbor, and
+ // re-select when a neighbor exceeds its link budget.
+ for &(_, sel) in &selected {
+ g.neighbors[sel as usize][layer].push(slot);
+ if g.neighbors[sel as usize][layer].len() > m_layer {
+ let dists = g.backlink_dists(sel, layer);
+ let pruned = g.select_diverse(&dists, m_layer);
+ g.neighbors[sel as usize][layer] = pruned.iter().map(|&(_, s)| s).collect();
}
}
@@ -303,17 +594,10 @@ impl HnswIndex {
}
}
- // Insert the node
- self.nodes.write().unwrap().insert(id, node);
-
- // Update entry point if new node is on a higher layer
- {
- let mut ep = self.entry_point.write().unwrap();
- let mut ep_layer = self.entry_layer.write().unwrap();
- if node_level > *ep_layer {
- *ep = Some(id);
- *ep_layer = node_level;
- }
+ // Update entry point if the new node reached a higher layer.
+ if node_level > g.entry_layer {
+ g.entry_slot = Some(slot);
+ g.entry_layer = node_level;
}
trace!("HNSW: inserted id={id} level={node_level}");
@@ -347,42 +631,33 @@ impl HnswIndex {
)));
}
- // Snapshot tombstones once and clone-into-the-closure so the
- // beam search sees a consistent set even if mark_deleted runs
- // mid-query. Cloning a u64 HashSet is cheap and avoids holding
- // the read lock across the whole traversal.
- let tombstoned: HashSet<u64> = self.tombstones.read().unwrap().clone();
- let combined_filter = move |id: u64| -> bool { !tombstoned.contains(&id) && filter(id) };
-
- let ep_opt = *self.entry_point.read().unwrap();
- let ep_id = match ep_opt {
- Some(id) => id,
+ let g = self.inner.read().unwrap();
+ let ep_slot = match g.entry_slot {
+ Some(s) => s,
None => return Ok(vec![]),
};
- let ep_layer = *self.entry_layer.read().unwrap();
-
- let mut curr_ep = ep_id;
+ let q_inv = g.query_inv(query);
- // Greedy descent from top to layer 1
- for layer in (1..=ep_layer).rev() {
- let (new_ep, _) = self.greedy_search_layer(query, curr_ep, layer, 1);
- curr_ep = new_ep;
+ // Greedy descent from the top layer to layer 1.
+ let mut curr_ep = ep_slot;
+ for layer in (1..=g.entry_layer).rev() {
+ curr_ep = g.greedy_layer(query, q_inv, curr_ep, layer).0;
}
- // Beam search at layer 0 with ef. Tombstoned ids are kept as
- // *traversal* candidates (otherwise we lose access to their
- // neighbours and recall collapses) but excluded from results
- // by the post-take filter below.
- let candidates = self.search_layer_filtered(query, curr_ep, ef.max(k), 0, &combined_filter);
+ // Beam search at layer 0. Tombstoned slots are kept as *traversal*
+ // candidates (otherwise we lose access to their neighbours and
+ // recall collapses) but excluded from the result set. The whole
+ // search runs under one read guard, so it sees a consistent
+ // tombstone set even if mark_deleted runs mid-query.
+ let admit = |s: u32| !g.tomb[s as usize] && filter(g.ids[s as usize]);
+ let candidates = g.beam(query, q_inv, curr_ep, ef.max(k), 0, &admit);
- let results: Vec<(u64, f32)> = candidates
+ Ok(candidates
.into_iter()
- .filter(|&(_, id)| combined_filter(id))
+ .filter(|&(_, s)| admit(s))
.take(k)
- .map(|(dist, id)| (id, dist))
- .collect();
-
- Ok(results)
+ .map(|(dist, s)| (g.ids[s as usize], dist))
+ .collect())
}
/// Insert a batch of `(id, vector)` items **serially**, one after another.
@@ -398,15 +673,15 @@ impl HnswIndex {
///
/// Each item is validated by `insert` (dimension check); a mismatch
/// returns `Err` and leaves already-inserted items in the graph (this
- /// method is not transactional / all-or-nothing).
+ /// API is not transactional).
pub fn insert_batch(&self, items: Vec<(u64, Vec<f32>)>) -> Result<()> {
- for (id, vec) in items {
- self.insert(id, vec)?;
+ for (id, vector) in items {
+ self.insert(id, vector)?;
}
Ok(())
}
- // ── Persistence ───────────────────────────────────────────────────────
+ // ── Persistence ──────────────────────────────────────────────────────
//
// Wire format (little-endian, single file, written atomically via
// tmp + rename):
@@ -415,7 +690,7 @@ impl HnswIndex {
// format_ver: u32 = HNSW_FORMAT_VERSION
// m: u32
// ef_construction:u32
- // metric: u8 = DistanceMetric repr (0=cosine,1=l2,2=dot,3=mip)
+ // metric: u8 = DistanceMetric repr (0=cosine,1=l2,2=dot)
// dim: u32
// entry_point: u64 = entry node id (u64::MAX = no entry)
// entry_layer: u32
@@ -427,6 +702,7 @@ impl HnswIndex {
// for layer 0..=max_layer:
// num_neigh: u32
// neigh_ids: u64 × num_neigh
+ // [v2] tombstones: u32 num + u64 × num
// crc32c: u32 (over all preceding bytes)
//
// Pre-v0.6.2 the graph was rebuilt on every restart by replaying
@@ -457,45 +733,50 @@ impl HnswIndex {
buf.write_u32::<LittleEndian>(self.params.dim as u32)
.map_err(io_to_xerj)?;
- let ep = *self.entry_point.read().unwrap();
- let el = *self.entry_layer.read().unwrap();
- buf.write_u64::<LittleEndian>(ep.unwrap_or(u64::MAX))
+ let g = self.inner.read().unwrap();
+ let ep_id = g.entry_slot.map(|s| g.ids[s as usize]);
+ buf.write_u64::<LittleEndian>(ep_id.unwrap_or(u64::MAX))
.map_err(io_to_xerj)?;
- buf.write_u32::<LittleEndian>(el as u32)
+ buf.write_u32::<LittleEndian>(g.entry_layer as u32)
.map_err(io_to_xerj)?;
- let nodes = self.nodes.read().unwrap();
- buf.write_u32::<LittleEndian>(nodes.len() as u32)
+ // Live slots only (a re-used external id orphans its old slot;
+ // the map wrote each external id once and so do we).
+ let live: Vec<u32> = (0..g.len() as u32)
+ .filter(|&s| g.slot_of.get(&g.ids[s as usize]) == Some(&s))
+ .collect();
+ buf.write_u32::<LittleEndian>(live.len() as u32)
.map_err(io_to_xerj)?;
- for (id, node_arc) in nodes.iter() {
- let n = node_arc.read().unwrap();
- buf.write_u64::<LittleEndian>(*id).map_err(io_to_xerj)?;
- let max_layer = n.neighbors.len().saturating_sub(1) as u32;
+ for &s in &live {
+ buf.write_u64::<LittleEndian>(g.ids[s as usize])
+ .map_err(io_to_xerj)?;
+ let nbrs = &g.neighbors[s as usize];
+ let max_layer = nbrs.len().saturating_sub(1) as u32;
buf.write_u32::<LittleEndian>(max_layer)
.map_err(io_to_xerj)?;
- for &v in &n.vector {
+ for &v in g.vec_of(s) {
buf.write_f32::<LittleEndian>(v).map_err(io_to_xerj)?;
}
- for layer_neighbors in &n.neighbors {
+ for layer_neighbors in nbrs {
buf.write_u32::<LittleEndian>(layer_neighbors.len() as u32)
.map_err(io_to_xerj)?;
- for &nid in layer_neighbors {
- buf.write_u64::<LittleEndian>(nid).map_err(io_to_xerj)?;
+ for &nb in layer_neighbors {
+ buf.write_u64::<LittleEndian>(g.ids[nb as usize])
+ .map_err(io_to_xerj)?;
}
}
}
- drop(nodes);
// v2 — tombstone block. Empty for indices that have never had
// a delete; v1 readers don't know to skip past this, so the
// format version was bumped (loader checks).
- let tombs = self.tombstones.read().unwrap();
- buf.write_u32::<LittleEndian>(tombs.len() as u32)
+ buf.write_u32::<LittleEndian>(g.tomb_ids.len() as u32)
.map_err(io_to_xerj)?;
- for &t in tombs.iter() {
+ for &t in g.tomb_ids.iter() {
buf.write_u64::<LittleEndian>(t).map_err(io_to_xerj)?;
}
- drop(tombs);
+ let node_count = live.len();
+ drop(g);
let crc = crc32fast::hash(&buf);
buf.write_u32::<LittleEndian>(crc).map_err(io_to_xerj)?;
@@ -506,7 +787,7 @@ impl HnswIndex {
debug!(
path = %path.display(),
bytes = buf.len(),
- nodes = self.nodes.read().unwrap().len(),
+ nodes = node_count,
"HNSW graph persisted"
);
Ok(())
@@ -561,8 +842,10 @@ impl HnswIndex {
metric,
dim,
};
- let mut node_map: HashMap<u64, Arc<RwLock<Node>>> = HashMap::with_capacity(num_nodes);
+ // Pass 1: read raw nodes (neighbors keyed by external id).
+ let mut g = Inner::new(dim, metric);
+ let mut raw_neighbors: Vec<Vec<Vec<u64>>> = Vec::with_capacity(num_nodes);
for _ in 0..num_nodes {
let id = cur.read_u64::<LittleEndian>().map_err(io_to_xerj)?;
let max_layer = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)? as usize;
@@ -579,189 +862,139 @@ impl HnswIndex {
}
neighbors.push(layer);
}
- node_map.insert(
- id,
- Arc::new(RwLock::new(Node {
- id,
- vector,
- neighbors,
- })),
- );
+ g.alloc_slot(id, vector, max_layer, m);
+ raw_neighbors.push(neighbors);
}
+ // Pass 2: map neighbor external ids → slots (ids referencing
+ // unknown nodes are dropped, mirroring the old map's `get` skips).
+ for (slot, layers) in raw_neighbors.into_iter().enumerate() {
+ let converted: Vec<Vec<u32>> = layers
+ .into_iter()
+ .map(|layer| {
+ layer
+ .into_iter()
+ .filter_map(|nid| g.slot_of.get(&nid).copied())
+ .collect()
+ })
+ .collect();
+ g.neighbors[slot] = converted;
+ }
+
+ g.entry_slot = if ep_raw == u64::MAX {
+ None
+ } else {
+ match g.slot_of.get(&ep_raw).copied() {
+ Some(s) => Some(s),
+ None => return Err(corrupt("entry point references unknown node id")),
+ }
+ };
+ g.entry_layer = entry_layer;
+
// v2+: read trailing tombstone block. v1 files have no
// tombstones and we treat the set as empty.
- let mut tombstones: HashSet<u64> = HashSet::new();
if format_ver >= 2 {
let num_t = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)? as usize;
- tombstones.reserve(num_t);
for _ in 0..num_t {
- tombstones.insert(cur.read_u64::<LittleEndian>().map_err(io_to_xerj)?);
+ let t = cur.read_u64::<LittleEndian>().map_err(io_to_xerj)?;
+ if let Some(&slot) = g.slot_of.get(&t) {
+ g.tomb[slot as usize] = true;
+ }
+ g.tomb_ids.insert(t);
}
}
Ok(Self {
params,
- nodes: RwLock::new(node_map),
- entry_point: RwLock::new(if ep_raw == u64::MAX {
- None
- } else {
- Some(ep_raw)
- }),
- entry_layer: RwLock::new(entry_layer),
- tombstones: RwLock::new(tombstones),
+ inner: RwLock::new(g),
})
}
- // ── Internal search primitives ────────────────────────────────────────
+ // ── Debug / diagnostic hooks (dev tooling only) ──────────────────────
- /// Greedy search: descend to the single closest node at a given layer.
- fn greedy_search_layer(
+ /// Run the layer-0 beam search from an explicit entry node (bypassing
+ /// the hierarchy descent). Lets a probe distinguish "descent lands in
+ /// the wrong region" from "layer-0 beam cannot navigate".
+ pub fn debug_search_from(
&self,
- query: &[f32],
entry: u64,
- layer: usize,
- _ef: usize,
- ) -> (u64, f32) {
- let nodes = self.nodes.read().unwrap();
- let mut best_id = entry;
- let mut best_dist = {
- let n = nodes[&entry].read().unwrap();
- self.dist(query, &n.vector)
+ query: &[f32],
+ k: usize,
+ ef: usize,
+ ) -> Vec<(u64, f32)> {
+ let g = self.inner.read().unwrap();
+ let entry_slot = match g.slot_of.get(&entry) {
+ Some(&s) => s,
+ None => return vec![],
};
-
- loop {
- let mut improved = false;
- let neighbors: Vec<u64> = {
- let n = nodes[&best_id].read().unwrap();
- if layer < n.neighbors.len() {
- n.neighbors[layer].clone()
- } else {
- vec![]
- }
- };
-
- for nid in neighbors {
- if let Some(nn) = nodes.get(&nid) {
- let d = {
- let nv = nn.read().unwrap();
- self.dist(query, &nv.vector)
- };
- if d < best_dist {
- best_dist = d;
- best_id = nid;
- improved = true;
- }
- }
- }
-
- if !improved {
- break;
- }
- }
-
- (best_id, best_dist)
+ let q_inv = g.query_inv(query);
+ let admit = |s: u32| !g.tomb[s as usize];
+ let candidates = g.beam(query, q_inv, entry_slot, ef.max(k), 0, &admit);
+ candidates
+ .into_iter()
+ .take(k)
+ .map(|(dist, s)| (g.ids[s as usize], dist))
+ .collect()
}
- /// Beam search at a given layer, returning candidates sorted by distance.
- fn search_layer(&self, query: &[f32], entry: u64, ef: usize, layer: usize) -> Vec<(f32, u64)> {
- self.search_layer_filtered(query, entry, ef, layer, &|_| true)
+ /// The layer-0 start node the hierarchy descent chooses for `query`,
+ /// plus its distance.
+ pub fn debug_descent(&self, query: &[f32]) -> Option<(u64, f32)> {
+ let g = self.inner.read().unwrap();
+ let ep = g.entry_slot?;
+ let q_inv = g.query_inv(query);
+ let mut curr = ep;
+ let mut curr_dist = g.slot_dist(query, q_inv, curr);
+ for layer in (1..=g.entry_layer).rev() {
+ let (s, d) = g.greedy_layer(query, q_inv, curr, layer);
+ curr = s;
+ curr_dist = d;
+ }
+ Some((g.ids[curr as usize], curr_dist))
}
- fn search_layer_filtered(
+ /// Graph structure summary: (entry_point, entry_layer, level histogram,
+ /// layer-0 out-degree histogram, layer-0 BFS reachable-node count from
+ /// the entry point).
+ pub fn debug_structure(
&self,
- query: &[f32],
- entry: u64,
- ef: usize,
- layer: usize,
- filter: &dyn Fn(u64) -> bool,
- ) -> Vec<(f32, u64)> {
- let nodes = self.nodes.read().unwrap();
-
- let entry_dist = {
- match nodes.get(&entry) {
- Some(n) => self.dist(query, &n.read().unwrap().vector),
- None => return vec![],
- }
- };
-
- // Min-heap of (dist, id) candidates to explore
- let mut candidates: BinaryHeap<Reverse<(ordered_float::OrderedFloat, u64)>> =
- BinaryHeap::new();
- // Max-heap of (dist, id) for the result set W
- let mut w: BinaryHeap<(ordered_float::OrderedFloat, u64)> = BinaryHeap::new();
- let mut visited: HashSet<u64> = HashSet::new();
-
- candidates.push(Reverse((ordered_float::OrderedFloat(entry_dist), entry)));
- if filter(entry) {
- w.push((ordered_float::OrderedFloat(entry_dist), entry));
+ ) -> (
+ Option<u64>,
+ usize,
+ Vec<(usize, usize)>,
+ Vec<(usize, usize)>,
+ usize,
+ ) {
+ let g = self.inner.read().unwrap();
+ let ep = g.entry_slot.map(|s| g.ids[s as usize]);
+ let mut level_hist: std::collections::BTreeMap<usize, usize> = Default::default();
+ let mut deg0_hist: std::collections::BTreeMap<usize, usize> = Default::default();
+ for nbrs in &g.neighbors {
+ *level_hist.entry(nbrs.len() - 1).or_default() += 1;
+ *deg0_hist.entry(nbrs[0].len()).or_default() += 1;
}
- visited.insert(entry);
-
- while let Some(Reverse((dist_c, c))) = candidates.pop() {
- // Pruning: if closest candidate is farther than the farthest result, stop
- if let Some(&(farthest, _)) = w.peek() {
- if dist_c > farthest && w.len() >= ef {
- break;
- }
- }
-
- let neighbors: Vec<u64> = {
- match nodes.get(&c) {
- Some(n) => {
- let n = n.read().unwrap();
- if layer < n.neighbors.len() {
- n.neighbors[layer].clone()
- } else {
- vec![]
- }
- }
- None => continue,
- }
- };
-
- for nid in neighbors {
- if visited.contains(&nid) {
- continue;
- }
- visited.insert(nid);
-
- let d = match nodes.get(&nid) {
- Some(nn) => self.dist(query, &nn.read().unwrap().vector),
- None => continue,
- };
-
- let farthest_w = w.peek().map(|&(f, _)| f.0).unwrap_or(f32::MAX);
-
- if d < farthest_w || w.len() < ef {
- candidates.push(Reverse((ordered_float::OrderedFloat(d), nid)));
- if filter(nid) {
- w.push((ordered_float::OrderedFloat(d), nid));
- if w.len() > ef {
- w.pop();
- }
+ let mut reach = 0usize;
+ if let Some(start) = g.entry_slot {
+ let mut seen = vec![false; g.len()];
+ let mut stack = vec![start];
+ seen[start as usize] = true;
+ while let Some(c) = stack.pop() {
+ reach += 1;
+ for &nb in &g.neighbors[c as usize][0] {
+ if !seen[nb as usize] {
+ seen[nb as usize] = true;
+ stack.push(nb);
}
}
}
}
-
- // Collect results sorted by distance ascending
- let mut results: Vec<(f32, u64)> = w.into_iter().map(|(d, id)| (d.0, id)).collect();
- results.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
- results
- }
-
- /// Simple neighbor selection: take the M closest candidates.
- fn select_neighbors(
- &self,
- _query: &[f32],
- candidates: &[(f32, u64)],
- m: usize,
- ) -> Vec<(f32, u64)> {
- let mut sorted = candidates.to_vec();
- sorted.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
- sorted.truncate(m);
- sorted
+ (
+ ep,
+ g.entry_layer,
+ level_hist.into_iter().collect(),
+ deg0_hist.into_iter().collect(),
+ reach,
+ )
}
}
@@ -971,4 +1204,72 @@ mod tests {
assert_ne!(*id, 12);
}
}
+
+ #[test]
+ fn large_random_build_recall_regression() {
+ // Regression for two 2026-07-12 defects that only show at scale:
+ // 1. back-link pruning dropped the new node (map-miss), leaving
+ // the graph disconnected — recall@10 was 0.000 at 50k;
+ // 2. closest-M neighbor selection (no diversity heuristic) gave a
+ // much flatter recall/ef curve.
+ // 5k × 32-d keeps the test fast while still exercising saturated
+ // back-link pruning (>> M² nodes). At ef=400 over 5k random
+ // vectors, a healthy graph is essentially exact.
+ struct Rng(u64);
+ impl Rng {
+ fn next(&mut self) -> f32 {
+ // xorshift64*
+ self.0 ^= self.0 >> 12;
+ self.0 ^= self.0 << 25;
+ self.0 ^= self.0 >> 27;
+ ((self.0.wrapping_mul(0x2545F4914F6CDD1D) >> 33) as f64 / 2147483648.0) as f32 - 1.0
+ }
+ }
+ let (n, dim, k) = (5000usize, 32usize, 10usize);
+ let mut rng = Rng(0xC0FFEE);
+ let vecs: Vec<Vec<f32>> = (0..n)
+ .map(|_| (0..dim).map(|_| rng.next()).collect())
+ .collect();
+ let idx = make_index(dim);
+ for (i, v) in vecs.iter().enumerate() {
+ idx.insert(i as u64, v.clone()).unwrap();
+ }
+ // Layer-0 BFS from the entry must reach (almost) every node.
+ let (_, _, _, _, reach) = idx.debug_structure();
+ assert!(
+ reach as f64 >= 0.99 * n as f64,
+ "layer-0 graph disconnected: BFS reached {reach}/{n}"
+ );
+
+ let cos = |a: &[f32], b: &[f32]| -> f32 {
+ let (mut d, mut na, mut nb) = (0f32, 0f32, 0f32);
+ for i in 0..a.len() {
+ d += a[i] * b[i];
+ na += a[i] * a[i];
+ nb += b[i] * b[i];
+ }
+ d / (na.sqrt() * nb.sqrt()).max(1e-12)
+ };
+ let mut total_hits = 0usize;
+ let queries = 20usize;
+ for qi in 0..queries {
+ let q: Vec<f32> = (0..dim).map(|_| rng.next()).collect();
+ let mut scored: Vec<(u64, f32)> = vecs
+ .iter()
+ .enumerate()
+ .map(|(i, v)| (i as u64, cos(&q, v)))
+ .collect();
+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
+ let exact: std::collections::HashSet<u64> =
+ scored[..k].iter().map(|&(i, _)| i).collect();
+ let got = idx.search(&q, k, 400).unwrap();
+ total_hits += got.iter().filter(|(id, _)| exact.contains(id)).count();
+ let _ = qi;
+ }
+ let recall = total_hits as f64 / (queries * k) as f64;
+ assert!(
+ recall >= 0.9,
+ "recall@10 regression: got {recall:.3}, expected >= 0.9"
+ );
+ }
}