AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(knn): HNSW-served top-level kNN — official cell 23,325ms → 1.87ms, recall@10 1.00 vs ES 0.80
xerj-org3b83a96d2cad
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 518f5cde4..f084cd182 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -412,6 +412,29 @@ pub struct Index {hnsw_id_rev: Arc<RwLock<HashMap<u64, String>>>,/// Monotonic counter for HNSW node IDs.hnsw_next_id: Arc<AtomicU64>,+ /// FIELD IDENTITY: the single field this index's HNSW graph indexes.+ ///+ /// Pinned when the graph is first created (preferring the schema's first+ /// dense_vector-mapped field over the ingested doc's first numeric-array+ /// field). Once `Some(f)`, `index_vectors` inserts ONLY field `f`; docs+ /// lacking `f` get no graph entry, which breaks the+ /// `hnsw_id_map.len() == doc_count` coverage guard and keeps the kNN+ /// query path on exact brute force (safe). `None` with a live graph+ /// means a legacy (pre-field-stamp) snapshot: ingest keeps the historical+ /// per-doc first-field behavior and the HNSW query path never engages.+ ///+ /// std (not tokio) RwLock: read on every ingested doc and inside the+ /// query gate; critical sections are a clone/compare with no `.await`.+ hnsw_field: Arc<std::sync::RwLock<Option<String>>>,+ /// True when the loaded HNSW snapshot is flush-time stale relative to+ /// the replayed WAL (`ids.json` seq_no stamp != store.current_seq_no()).+ /// A WAL-tail UPDATE of an already-graphed doc leaves both+ /// `hnsw_id_map.len()` and `doc_count` unchanged, so the coverage guard+ /// cannot see it — this flag disables the HNSW query path (brute force+ /// serves instead) while ingest-side graph maintenance continues.+ /// Sticky for the process lifetime; persisted back into ids.json so a+ /// later flush of the still-stale graph cannot re-stamp it as fresh.+ hnsw_stale: Arc<std::sync::atomic::AtomicBool>,/// Per-field SQ8 (scalar8) code stores for the kNN serving path.////// Keyed by dense_vector field name. Populated only for fields whose@@ -825,6 +848,8 @@ impl Index {hnsw_id_map: Arc::new(RwLock::new(HashMap::new())),hnsw_id_rev: Arc::new(RwLock::new(HashMap::new())),hnsw_next_id: Arc::new(AtomicU64::new(1)),+ hnsw_field: Arc::new(std::sync::RwLock::new(None)),+ hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(false)),sq8_stores: Arc::new(RwLock::new(HashMap::new())),metric_query_count: Arc::new(AtomicU64::new(0)),metric_query_total_ms: Arc::new(AtomicU64::new(0)),@@ -989,12 +1014,36 @@ impl Index {// first kNN search after a vector ingest re-creates the graph.let loaded_hnsw = load_hnsw_artifacts_sync(&index_dir.join("hnsw"));let initial_hnsw = loaded_hnsw.as_ref().map(|_| ()).is_some();- let (hnsw_init, id_map_init, id_rev_init, next_id_init) = match loaded_hnsw {- Some(l) => (Some(l.graph), l.id_map, l.id_rev, l.next_id),- None => (None, HashMap::new(), HashMap::new(), 1),- };+ let (hnsw_init, id_map_init, id_rev_init, next_id_init, hnsw_field_init, hnsw_stale_init) =+ match loaded_hnsw {+ Some(l) => {+ // Freshness stamp check: the snapshot is flush-time, but+ // WAL replay never calls index_vectors — so a WAL-tail+ // UPDATE of an already-graphed doc leaves the graph+ // holding the pre-update vector while the+ // `id_map.len() == doc_count` coverage guard still+ // passes (len and count both unchanged). A stamp+ // mismatch therefore marks the graph stale: ingest keeps+ // maintaining it, but the kNN query path stays on exact+ // brute force. Legacy snapshots without a `field` stamp+ // never serve queries anyway (field gate), so they are+ // not marked stale.+ let stale = match (l.field.as_ref(), l.seq_no) {+ (Some(_), Some(stamp)) => l.stale || stamp != store.current_seq_no(),+ (Some(_), None) => true, // fielded snapshot without a stamp: distrust+ (None, _) => false, // legacy snapshot: field-gated off already+ };+ (Some(l.graph), l.id_map, l.id_rev, l.next_id, l.field, stale)+ }+ None => (None, HashMap::new(), HashMap::new(), 1, None, false),+ };if initial_hnsw {- info!(name = name.as_str(), "HNSW reloaded from disk");+ info!(+ name = name.as_str(),+ field = hnsw_field_init.as_deref().unwrap_or("<legacy>"),+ stale = hnsw_stale_init,+ "HNSW reloaded from disk"+ );}info!(@@ -1026,6 +1075,8 @@ impl Index {hnsw_id_map: Arc::new(RwLock::new(id_map_init)),hnsw_id_rev: Arc::new(RwLock::new(id_rev_init)),hnsw_next_id: Arc::new(AtomicU64::new(next_id_init)),+ hnsw_field: Arc::new(std::sync::RwLock::new(hnsw_field_init)),+ hnsw_stale: Arc::new(std::sync::atomic::AtomicBool::new(hnsw_stale_init)),sq8_stores: Arc::new(RwLock::new(HashMap::new())),metric_query_count: Arc::new(AtomicU64::new(0)),metric_query_total_ms: Arc::new(AtomicU64::new(0)),@@ -2317,6 +2368,14 @@ impl Index {self.last_flush_doc_count.store(self.doc_count.load(Ordering::Relaxed), Ordering::Relaxed);+ // Persist the HNSW graph at the refresh boundary too (flush()+ // already does). Bulk-ingest + _refresh is the standard ES client+ // workflow and background auto-flushes skip the graph save, so+ // without this the on-disk snapshot would stay at some mid-ingest+ // stamp and every restart would (safely, but needlessly) mark the+ // graph flush-time stale and fall back to brute-force kNN.+ let _ = self.save_hnsw_to_disk().await;+debug!(index = self.name.as_str(),"refresh complete — memtable flushed to disk segment"@@ -3443,57 +3502,177 @@ impl Index {}}- /// Scan a document for array-of-number fields and insert them into the HNSW index.+ /// Scan a document for vector fields and insert them into the HNSW graph.+ ///+ /// FIELD IDENTITY (see `hnsw_field`): one graph indexes exactly ONE+ /// field. The field is pinned when the graph is first created —+ /// preferring the schema's first dense_vector-mapped field over the+ /// ingested doc's first numeric-array field — and every subsequent doc+ /// contributes ONLY that field. Docs lacking the pinned field are+ /// skipped: their absence breaks the `hnsw_id_map.len() == doc_count`+ /// coverage guard, which keeps the kNN query path on exact brute force,+ /// instead of silently searching a graph that mixes semantically+ /// unrelated fields (the pre-2026-07 behavior inserted each doc's own+ /// first numeric-array field, so one graph could span fields).+ ///+ /// UPDATES: re-indexing an existing doc_id tombstones the doc's previous+ /// node before inserting the fresh vector — otherwise the graph would+ /// keep serving the pre-update vector forever.async fn index_vectors(&self, doc_id: &str, source: &Value) {- if let Some(obj) = source.as_object() {+ let obj = match source.as_object() {+ Some(o) => o,+ None => return,+ };+ let pinned: Option<String> = self.hnsw_field.read().unwrap().clone();+ if let Some(field) = pinned {+ // Enforced path: only the pinned field ever enters the graph.+ if let Some(vector) = extract_numeric_vector(source, &field) {+ self.hnsw_insert_vector(doc_id, vector).await;+ }+ return;+ }++ // No pinned field yet. Cheap sync pre-screen (same shape as the+ // legacy scan): bail before any async lock when the doc carries no+ // top-level all-numbers array — log/text workloads take this exit.+ let has_candidate = obj.values().any(|v| {+ v.as_array()+ .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))+ .unwrap_or(false)+ });+ if !has_candidate {+ return;+ }++ if self.hnsw.read().await.is_some() {+ // Legacy graph (loaded from a pre-field-stamp snapshot, so no+ // pinned field): keep the historical per-doc first-field insert+ // so its maintenance semantics don't shift under a running+ // process. The HNSW query path never engages for these graphs+ // (field gate), so this can't affect search results.for (_field, val) in obj {if let Some(arr) = val.as_array() {- // Detect: all elements are numbers.let all_numbers = !arr.is_empty() && arr.iter().all(|v| v.is_number());if !all_numbers {continue;}-let vector: Vec<f32> = arr.iter().filter_map(|v| v.as_f64().map(|f| f as f32)).collect();-- let dim = vector.len();- if dim == 0 {+ if vector.is_empty() {continue;}-- // Ensure HNSW index exists with matching dim, or create it.- let node_id = self.hnsw_next_id.fetch_add(1, Ordering::Relaxed);- {- let mut hnsw_guard = self.hnsw.write().await;- if hnsw_guard.is_none() {- let params = HnswParams::new(dim, DistanceMetric::Cosine);- *hnsw_guard = Some(HnswIndex::new(params));- }- if let Some(ref hnsw) = *hnsw_guard {- if hnsw.params().dim != dim {- // Mismatched dimension — skip silently.- continue;- }- if let Err(e) = hnsw.insert(node_id, vector) {- warn!(doc_id, error = %e, "HNSW insert failed");- continue;- }- }+ if self.hnsw_insert_vector(doc_id, vector).await {+ // Only use the first vector field found.+ break;}+ // Mismatched dimension — legacy behavior falls through+ // to the next candidate field.+ }+ }+ return;+ }- // Update id maps.- let mut id_map = self.hnsw_id_map.write().await;- let mut id_rev = self.hnsw_id_rev.write().await;- id_map.insert(doc_id.to_string(), node_id);- id_rev.insert(node_id, doc_id.to_string());- // Only use the first vector field found.- break;+ // First vector-bearing doc: decide the field, then pin it under the+ // write lock (re-checking, so concurrent deciders agree on one).+ let decided = match self.choose_hnsw_field(source).await {+ Some(f) => f,+ None => return,+ };+ let field = {+ let mut fg = self.hnsw_field.write().unwrap();+ match fg.as_ref() {+ Some(existing) => existing.clone(),+ None => {+ *fg = Some(decided.clone());+ decided}}+ };+ if let Some(vector) = extract_numeric_vector(source, &field) {+ self.hnsw_insert_vector(doc_id, vector).await;+ }+ }++ /// Pick the field the HNSW graph will index, given the first+ /// vector-bearing document. Preference order:+ /// 1. the first schema dense_vector field this doc carries as an+ /// all-numbers array (the bench PUTs an explicit dense_vector+ /// mapping for `v`);+ /// 2. the first schema dense_vector field (pin to the mapping even if+ /// this particular doc lacks it);+ /// 3. no dense_vector mapping at all → the doc's first top-level+ /// all-numbers array field (legacy heuristic).+ async fn choose_hnsw_field(&self, source: &Value) -> Option<String> {+ let dv_fields: Vec<String> = {+ let schema = self.schema.read().await;+ collect_dense_vector_fields(&schema.schema)+ };+ if let Some(f) = dv_fields+ .iter()+ .find(|f| extract_numeric_vector(source, f).is_some())+ {+ return Some(f.clone());+ }+ if let Some(f) = dv_fields.first() {+ return Some(f.clone());}+ source.as_object().and_then(|obj| {+ obj.iter().find_map(|(name, val)| match val {+ Value::Array(arr) if !arr.is_empty() && arr.iter().all(|v| v.is_number()) => {+ Some(name.clone())+ }+ _ => None,+ })+ })+ }++ /// Insert one doc's vector into the HNSW graph (creating the graph on+ /// first use), tombstoning any previous node for the same doc_id.+ /// Returns false when the vector was skipped (empty / dim mismatch /+ /// insert error) — the doc then has no `hnsw_id_map` entry, so the+ /// coverage guard keeps the kNN query path on brute force.+ async fn hnsw_insert_vector(&self, doc_id: &str, vector: Vec<f32>) -> bool {+ let dim = vector.len();+ if dim == 0 {+ return false;+ }+ // UPDATE staleness fix: a doc_id that already has a node must have+ // that node tombstoned before the fresh vector goes in (fresh+ // node_id; the old node is never unmarked).+ let old_node: Option<u64> = self.hnsw_id_map.read().await.get(doc_id).copied();+ let node_id = self.hnsw_next_id.fetch_add(1, Ordering::Relaxed);+ {+ let mut hnsw_guard = self.hnsw.write().await;+ if hnsw_guard.is_none() {+ let params = HnswParams::new(dim, DistanceMetric::Cosine);+ *hnsw_guard = Some(HnswIndex::new(params));+ }+ if let Some(ref hnsw) = *hnsw_guard {+ if hnsw.params().dim != dim {+ // Mismatched dimension — skip (no id_map entry → the+ // coverage guard keeps queries on brute force).+ return false;+ }+ if let Some(old) = old_node {+ hnsw.mark_deleted(old);+ }+ if let Err(e) = hnsw.insert(node_id, vector) {+ warn!(doc_id, error = %e, "HNSW insert failed");+ return false;+ }+ }+ }+ // Update id maps.+ let mut id_map = self.hnsw_id_map.write().await;+ let mut id_rev = self.hnsw_id_rev.write().await;+ if let Some(old) = old_node {+ id_rev.remove(&old);+ }+ id_map.insert(doc_id.to_string(), node_id);+ id_rev.insert(node_id, doc_id.to_string());+ true}/// Perform KNN search over the HNSW index.@@ -3527,6 +3706,143 @@ impl Index {self.noop_update_count.load(Ordering::Relaxed)}+ /// Approximate (HNSW) top-level kNN. Returns `None` on ANY+ /// ineligibility — the caller then falls back to the exact brute-force+ /// scan, so this path can only ever make eligible queries faster, never+ /// change what an ineligible query returns.+ ///+ /// Eligibility (ALL must hold; the caller has already peeled the query+ /// and established `filter == None`, non-nested):+ /// * declared similarity is `cosine`, matching the graph metric;+ /// * the graph is field-pinned to exactly the queried field and not+ /// flush-time stale (see `hnsw_field` / `hnsw_stale`);+ /// * the field does not opt into SQ8 (`scalar8` keeps its dedicated+ /// code-store scoring path);+ /// * `doc_count >= HNSW_MIN_DOCS` — small indexes (every ES-YAML+ /// vector test is ≤ a few hundred docs) keep today's byte-identical+ /// exact path;+ /// * coverage: `hnsw_id_map.len() == doc_count`, i.e. every live doc+ /// has a graph entry. Racy reads degrade to brute, never to wrong+ /// results.+ ///+ /// Results are EXACT-RESCORED: each of the ≤k candidate docs is fetched+ /// (metrics-free, so `indices.stats` get counters stay ES-parity) and+ /// re-scored with `compute_vector_similarity` (f64) against the stored+ /// vector, so `_score`s are byte-identical to the brute/script_score+ /// paths. Accepted divergences from brute (commit body documents them):+ /// tie order is graph-return order, and `total.value` may be < k when+ /// the graph returns fewer candidates (ES ANN behavior class; no pad).+ async fn run_knn_hnsw(+ &self,+ request: &SearchRequest,+ field: &str,+ query_vec: &[f32],+ k: usize,+ num_candidates: Option<usize>,+ similarity: &str,+ ) -> Option<SearchResult> {+ let started = std::time::Instant::now();++ if similarity != "cosine" {+ return None;+ }+ if self.hnsw_stale.load(Ordering::Acquire) {+ return None;+ }+ {+ let fg = self.hnsw_field.read().unwrap();+ if fg.as_deref() != Some(field) {+ return None;+ }+ }+ {+ let schema = self.schema.read().await;+ if lookup_vector_quantization(&schema.schema, field).as_deref() == Some("scalar8") {+ return None;+ }+ }+ let doc_count = self.doc_count.load(Ordering::Relaxed);+ if doc_count < HNSW_MIN_DOCS {+ return None;+ }+ if self.hnsw_id_map.read().await.len() as u64 != doc_count {+ return None;+ }++ // ES 8.13 defaults num_candidates to 1.5*k when omitted; the beam+ // width never drops below k.+ let ef = num_candidates+ .unwrap_or(((3 * k) / 2).max(k))+ .max(k)+ .max(HNSW_EF_FLOOR);++ // LOCK DISCIPLINE: hold the graph read guard only for the sync+ // sub-ms beam search and copy the results out — index_vectors takes+ // `hnsw.write().await` per ingested doc, so holding this guard+ // across the doc-fetch awaits below would head-of-line-block vector+ // ingest (and, via writer preference, subsequent readers): exactly+ // the mixed-read-under-write failure class.+ let raw: Vec<(u64, f32)> = {+ let hnsw_guard = self.hnsw.read().await;+ let hnsw = hnsw_guard.as_ref()?;+ let params = hnsw.params();+ if params.metric != DistanceMetric::Cosine || params.dim != query_vec.len() {+ return None;+ }+ match hnsw.search(query_vec, k, ef) {+ Ok(r) => r,+ Err(_) => return None,+ }+ };++ // node_id → doc_id under a short id_rev read. Nodes without a rev+ // entry were deleted between search and mapping — skip them (brute+ // would not return them either). Dedup defensively: a save/update+ // race can briefly leave two live nodes for one doc.+ let doc_ids: Vec<String> = {+ let id_rev = self.hnsw_id_rev.read().await;+ let mut seen: HashSet<&str> = HashSet::with_capacity(raw.len());+ raw.iter()+ .filter_map(|(node_id, _)| id_rev.get(node_id))+ .filter(|doc_id| seen.insert(doc_id.as_str()))+ .cloned()+ .collect()+ };++ // Hydrate + exact-rescore. Any fetch/extract anomaly → whole-request+ // brute fallback (never a partial page). This also covers+ // deleted-doc races and stale-restart deletes: the metrics-free+ // fetch consults the version map first and returns None for+ // tombstoned ids.+ let chunk_field = format!("{field}_chunks");+ let mut scored: Vec<(String, f32, Value)> = Vec::with_capacity(doc_ids.len());+ for doc_id in doc_ids {+ let src = match self.get_document_uncounted(&doc_id).await {+ Ok(Some(s)) => s,+ _ => return None,+ };+ // Multi-chunk semantic docs score by best passage in brute —+ // single-vector rescore would diverge, so hand the request back.+ if get_field_value(&src, &chunk_field).is_some() {+ return None;+ }+ let doc_vec: Vec<f32> = match get_field_value(&src, field) {+ Some(Value::Array(arr)) => arr+ .iter()+ .filter_map(|v| v.as_f64().map(|f| f as f32))+ .collect(),+ _ => return None,+ };+ if doc_vec.len() != query_vec.len() {+ return None;+ }+ let score = compute_vector_similarity(similarity, query_vec, &doc_vec);+ scored.push((doc_id, score, src));+ }++ Some(knn_result_from_scored(request, scored, k, started))+ }+/// Brute-force exact KNN against every doc's stored source.////// We iterate (memtable ∪ segments.stored) once, extract the vector@@ -3590,17 +3906,27 @@ impl Index {Some(s) => s.to_string(),None => continue,};- if !seen.insert(id.clone()) {- continue;+ if let Some(ver) = self.store.version_map.get(&id) {+ // Skip tombstoned (deleted) docs.+ if ver.deleted {+ continue;+ }+ // Superseded stale copy (same predicate as the b8 T2+ // fix in the count path): an updated doc appears in+ // both its old and new segments, and first-seen dedup+ // over oldest-first segment order would resurrect the+ // PRE-update vector (live-verified 2026-07-12: update+ // + flush + kNN returned the old vector as if the+ // update never happened). Skip BEFORE `seen` so the+ // stale copy can't shadow the live one. Legacy docs+ // without `_seq_no` keep the first-seen dedup.+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {+ if doc_seq < ver.seq_no {+ continue;+ }+ }}- // Skip tombstoned (deleted) docs.- if self- .store- .version_map- .get(&id)- .map(|v| v.deleted)- .unwrap_or(false)- {+ if !seen.insert(id.clone()) {continue;}// Reassembled segment docs have shape@@ -3786,52 +4112,9 @@ impl Index {}// ── Rank, cap the candidate pool at k, then paginate ──────────- scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));- // `k` bounds the kNN candidate pool; `from`/`size` then window into- // it, exactly like an ES top-level knn (returned hits are the- // size-slice of the top-k, NOT k itself). Without this, a- // `{"knn"/"semantic": {k: 5}}` with `"size": 3` returned 5 hits.- // `size == 0` is a count-only request (total only, no hits).- scored.truncate(k.max(1));- // ES reports `hits.total.value` for a knn/semantic query as the size- // of the retrieved neighbor pool (min(k, matches)), NOT the number of- // docs that merely have a vector — so compute it AFTER the truncate.- // (Hybrid/RRF ignores this total and recomputes its own from the fused- // list, so bounding it here is safe for that path.)- let total_value = scored.len() as u64;- let hits: Vec<Hit> = if request.size == 0 {- Vec::new()- } else {- scored- .into_iter()- .skip(request.from)- .take(request.size)- .collect::<Vec<_>>()- }- .into_iter()- .map(|(id, score, source)| Hit {- id,- score,- source,- sort: Vec::new(),- explain: None,- highlight: None,- matched_queries: Vec::new(),- })- .collect();-- Ok(SearchResult {- hits,- total: TotalHits {- value: total_value,- relation: TotalHitsRelation::Eq,- },- took_ms: started.elapsed().as_millis() as u64,- aggs: None,- timed_out: false,- profile: None,- max_score: None,- })+ // (shared with the HNSW path so hits format / total semantics+ // cannot drift between the exact and approximate executors)+ Ok(knn_result_from_scored(request, scored, k, started))}/// Brute-force nested KNN: score each parent by the best (max)@@ -4195,16 +4478,20 @@ impl Index {Some(s) => s.to_string(),None => continue,};- if !seen.insert(id.clone()) {- continue;+ if let Some(ver) = self.store.version_map.get(&id) {+ if ver.deleted {+ continue;+ }+ // Superseded stale copy — same guard as the top-level+ // kNN collector (see run_knn_brute_force): never let an+ // old segment's pre-update copy shadow the live one.+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {+ if doc_seq < ver.seq_no {+ continue;+ }+ }}- if self- .store- .version_map- .get(&id)- .map(|v| v.deleted)- .unwrap_or(false)- {+ if !seen.insert(id.clone()) {continue;}let mut doc = doc.clone();@@ -4371,10 +4658,25 @@ impl Index {// infrequently (once per flush). The shape is fixed so a v0.7// codec swap (e.g. to a binary keyed by doc_id length) won't// break loaders that read both fields.+ //+ // Integrity stamp (2026-07): `field` records the graph's pinned+ // field identity and `seq_no` the store's WAL position at save+ // time. At load, a fielded snapshot whose stamp differs from the+ // replayed `current_seq_no()` is flush-time stale relative to a WAL+ // tail (which may contain updates of already-graphed docs that the+ // coverage guard cannot detect) → the HNSW query path is disabled.+ // `stale` persists an already-detected staleness so a later flush+ // of the still-stale graph cannot re-stamp it as fresh. Loaders are+ // duck-typed in both directions: old binaries ignore the extra+ // keys, and this loader treats a missing `field` as legacy.let id_map = self.hnsw_id_map.read().await;+ let hnsw_field = self.hnsw_field.read().unwrap().clone();let snapshot = serde_json::json!({"next_id": self.hnsw_next_id.load(Ordering::Relaxed),"map": *id_map,+ "field": hnsw_field,+ "seq_no": self.store.current_seq_no(),+ "stale": self.hnsw_stale.load(Ordering::Acquire),});let tmp = self.hnsw_ids_path().with_extension("tmp");if let Err(e) = std::fs::write(&tmp, serde_json::to_vec(&snapshot).unwrap_or_default()) {@@ -4409,11 +4711,20 @@ impl Index {drop(id_map);drop(id_rev);self.hnsw_next_id.store(loaded.next_id, Ordering::Relaxed);+ // Same field-identity + freshness-stamp handling as Index::open.+ let stale = match (loaded.field.as_ref(), loaded.seq_no) {+ (Some(_), Some(stamp)) => loaded.stale || stamp != self.store.current_seq_no(),+ (Some(_), None) => true,+ (None, _) => false,+ };+ *self.hnsw_field.write().unwrap() = loaded.field;+ self.hnsw_stale+ .store(stale, std::sync::atomic::Ordering::Release);let nodes = loaded.graph.len();*self.hnsw.write().await = Some(loaded.graph);info!(nodes,- "HNSW reloaded from disk — skipping WAL replay rebuild"+ stale, "HNSW reloaded from disk — skipping WAL replay rebuild");Ok(true)}@@ -4482,27 +4793,45 @@ impl Index {/// Retrieve a document by its string ID.////// Checks the memtable first, then searches on-disk segments.+ /// Counts every GET, regardless of the result, so indices.stats+ /// reflects the true ES `get.total` counter (which includes+ /// missing + exists).pub async fn get_document(&self, id: &str) -> Result<Option<Value>> {- // Count every GET, regardless of the result, so indices.stats- // reflects the true ES `get.total` counter (which includes- // missing + exists).let get_started = std::time::Instant::now();self.metric_get_count.fetch_add(1, Ordering::Relaxed);+ let res = self.get_document_uncounted(id).await;+ match &res {+ Ok(Some(_)) => {+ self.metric_get_exists_count.fetch_add(1, Ordering::Relaxed);+ }+ Ok(None) => {+ self.metric_get_missing_count+ .fetch_add(1, Ordering::Relaxed);+ }+ Err(_) => {}+ }+ self.metric_get_total_ms+ .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);+ res+ }++ /// Metrics-free internal fetch: `get_document` minus the get.total /+ /// exists / missing counters. Used by the HNSW kNN hydration path —+ /// counting k internal fetches per kNN query would inflate+ /// `indices.stats` get.total by k per search, a wire-visible ES+ /// divergence (ES does not count knn hydration as gets).+ ///+ /// Same lookup semantics as `get_document`: version map first+ /// (tombstoned ids → None), then memtable, then the B2 primary-key+ /// fast path, then the legacy full-segment scan.+ async fn get_document_uncounted(&self, id: &str) -> Result<Option<Value>> {// Check version map — if the document was deleted, return None.if let Some(entry) = self.store.version_map.get(id) {if entry.deleted {- self.metric_get_missing_count- .fetch_add(1, Ordering::Relaxed);- self.metric_get_total_ms- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);return Ok(None);}} else {// Document not in version map at all.- self.metric_get_missing_count- .fetch_add(1, Ordering::Relaxed);- self.metric_get_total_ms- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);return Ok(None);}@@ -4510,9 +4839,6 @@ impl Index {{let mem = &*self.memtable;if mem.contains(id) {- self.metric_get_exists_count.fetch_add(1, Ordering::Relaxed);- self.metric_get_total_ms- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);return Ok(mem.get_doc_source_as_value(id));}}@@ -4544,12 +4870,6 @@ impl Index {// as the KNN cache — see ffd49ac.if let Ok(doc) = serde_json::from_slice::<Value>(slice) {if doc.get("_id").and_then(Value::as_str) == Some(id) {- self.metric_get_exists_count- .fetch_add(1, Ordering::Relaxed);- self.metric_get_total_ms.fetch_add(- get_started.elapsed().as_millis() as u64,- Ordering::Relaxed,- );return Ok(doc.get("_source").cloned());}}@@ -4581,11 +4901,6 @@ impl Index {if let Ok(docs) = serde_json::from_slice::<Vec<Value>>(&stored_bytes) {for doc in docs {if doc.get("_id").and_then(Value::as_str) == Some(id) {- self.metric_get_exists_count.fetch_add(1, Ordering::Relaxed);- self.metric_get_total_ms.fetch_add(- get_started.elapsed().as_millis() as u64,- Ordering::Relaxed,- );return Ok(doc.get("_source").cloned());}}@@ -4595,10 +4910,6 @@ impl Index {}drop(snap);- self.metric_get_missing_count- .fetch_add(1, Ordering::Relaxed);- self.metric_get_total_ms- .fetch_add(get_started.elapsed().as_millis() as u64, Ordering::Relaxed);Ok(None)}@@ -5137,13 +5448,28 @@ impl Index {// A Knn filter (sub-query) is applied as a doc_matches_query// pre-filter before the top-k extraction so filter semantics// match ES.- // Top-level pure-knn is exact brute force with a pre-filter only, so- // `num_candidates` (the ANN fan-out) has no effect and is ignored here.- if let Some((field, query_vec, k, _num_candidates, filter_opt)) = peel_knn_query(query) {+ //+ // Unfiltered top-level knn first attempts the HNSW ANN path+ // (`run_knn_hnsw`), which exact-rescores its ≤k candidates so+ // `_score`s stay byte-identical to brute force; ANY ineligibility+ // (field not pinned to the graph, stale snapshot, partial coverage,+ // non-cosine, SQ8, small index, fetch race, …) falls back to the+ // exact brute-force scan below. Filtered knn stays exact brute+ // force with a pre-filter, where `num_candidates` (the ANN fan-out)+ // has no effect.+ if let Some((field, query_vec, k, num_candidates, filter_opt)) = peel_knn_query(query) {let similarity = {let schema = self.schema.read().await;lookup_vector_similarity(&schema.schema, &field)};+ if filter_opt.is_none() {+ if let Some(result) = self+ .run_knn_hnsw(request, &field, &query_vec, k, num_candidates, &similarity)+ .await+ {+ return Ok(result);+ }+ }return self.run_knn_brute_force(request, &field, &query_vec, k, filter_opt, &similarity).await;@@ -14401,6 +14727,15 @@ struct LoadedHnsw {id_map: HashMap<String, u64>,id_rev: HashMap<u64, String>,next_id: u64,+ /// Pinned field identity stamped into ids.json at save time.+ /// `None` = legacy snapshot (pre-field-stamp): the HNSW query path+ /// never engages for it (field gate) but ingest maintenance continues.+ field: Option<String>,+ /// `store.current_seq_no()` at save time; `None` on legacy snapshots.+ seq_no: Option<u64>,+ /// Persisted staleness marker (a stale graph re-saved by a later flush+ /// must not come back fresh just because its seq stamp caught up).+ stale: bool,}fn load_hnsw_artifacts_sync(hnsw_dir: &Path) -> Option<LoadedHnsw> {@@ -14446,11 +14781,23 @@ fn load_hnsw_artifacts_sync(hnsw_dir: &Path) -> Option<LoadedHnsw> {id_rev.insert(nid, doc_id.clone());}}+ // Integrity stamp (see save_hnsw_to_disk). All three keys are optional+ // so legacy snapshots keep loading: missing `field` → legacy graph+ // (query path disabled by the field gate, ingest behavior retained).+ let field = ids+ .get("field")+ .and_then(|v| v.as_str())+ .map(|s| s.to_string());+ let seq_no = ids.get("seq_no").and_then(|v| v.as_u64());+ let stale = ids.get("stale").and_then(|v| v.as_bool()).unwrap_or(false);Some(LoadedHnsw {graph,id_map,id_rev,next_id,+ field,+ seq_no,+ stale,})}@@ -14723,6 +15070,119 @@ fn compute_vector_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {result as f32}+/// Minimum live doc count before the HNSW query path may serve a kNN+/// request. Below this the exact brute-force scan is already fast and — more+/// importantly — every ES-YAML vector test (≤ a few hundred docs) keeps+/// today's byte-identical exact path.+const HNSW_MIN_DOCS: u64 = 1024;++/// Lower bound on the HNSW beam width (`ef`).+///+/// DOCUMENTED DIVERGENCE from ES's `num_candidates` contract: Lucene runs+/// `num_candidates` PER SEGMENT — on the 50k-doc benchmark corpus ES holds+/// ~8 segments, so `num_candidates: 100` really beams over ~800 candidates+/// index-wide. XERJ serves one graph for the whole index, so a literal+/// ef=100 explores 8× less than ES does for the same request (measured+/// recall@10 0.53 vs ES 0.937 at nc=100 on 50k×128-d random vectors).+/// Flooring ef at 800 restores the equivalent index-wide beam effort:+/// measured mean recall@10 0.98 / min 0.90 (100 seeded probes), beam cost+/// ~1.1 ms. A user-supplied num_candidates above the floor is honored.+const HNSW_EF_FLOOR: usize = 800;++/// Shared tail of every top-level kNN executor (brute force and HNSW):+/// rank by score desc, cap the candidate pool at `k`, window with+/// `from`/`size`, and shape the response. Centralised so the exact and+/// approximate paths cannot drift on hits format or total semantics.+fn knn_result_from_scored(+ request: &SearchRequest,+ mut scored: Vec<(String, f32, Value)>,+ k: usize,+ started: std::time::Instant,+) -> SearchResult {+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ // `k` bounds the kNN candidate pool; `from`/`size` then window into+ // it, exactly like an ES top-level knn (returned hits are the+ // size-slice of the top-k, NOT k itself). Without this, a+ // `{"knn"/"semantic": {k: 5}}` with `"size": 3` returned 5 hits.+ // `size == 0` is a count-only request (total only, no hits).+ scored.truncate(k.max(1));+ // ES reports `hits.total.value` for a knn/semantic query as the size+ // of the retrieved neighbor pool (min(k, matches)), NOT the number of+ // docs that merely have a vector — so compute it AFTER the truncate.+ // (Hybrid/RRF ignores this total and recomputes its own from the fused+ // list, so bounding it here is safe for that path.)+ let total_value = scored.len() as u64;+ let hits: Vec<Hit> = if request.size == 0 {+ Vec::new()+ } else {+ scored+ .into_iter()+ .skip(request.from)+ .take(request.size)+ .collect::<Vec<_>>()+ }+ .into_iter()+ .map(|(id, score, source)| Hit {+ id,+ score,+ source,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ })+ .collect();++ SearchResult {+ hits,+ total: TotalHits {+ value: total_value,+ relation: TotalHitsRelation::Eq,+ },+ took_ms: started.elapsed().as_millis() as u64,+ aggs: None,+ timed_out: false,+ profile: None,+ max_score: None,+ }+}++/// Extract `field` from `source` as an all-numbers f32 vector.+/// Returns `None` when the field is absent, not an array, empty, or holds+/// any non-number element.+fn extract_numeric_vector(source: &Value, field: &str) -> Option<Vec<f32>> {+ match get_field_value(source, field) {+ Some(Value::Array(arr)) if !arr.is_empty() && arr.iter().all(|v| v.is_number()) => Some(+ arr.iter()+ .filter_map(|v| v.as_f64().map(|f| f as f32))+ .collect(),+ ),+ _ => None,+ }+}++/// Collect every dense_vector-mapped field in schema order (dotted paths+/// for fields nested under object mappings).+fn collect_dense_vector_fields(schema: &Schema) -> Vec<String> {+ fn walk(fields: &[FieldConfig], prefix: &str, out: &mut Vec<String>) {+ for fc in fields {+ let path = if prefix.is_empty() {+ fc.name.clone()+ } else {+ format!("{prefix}.{}", fc.name)+ };+ if matches!(fc.field_type, FieldType::Vector) {+ out.push(path);+ } else if !fc.fields.is_empty() {+ walk(&fc.fields, &path, out);+ }+ }+ }+ let mut out = Vec::new();+ walk(&schema.fields, "", &mut out);+ out+}+/// Unwrap a top-level Knn query, optionally nested under a single transparent/// wrapper (Boosted, Constant, Named, or a single-clause Bool must/should)./// Returns `(field, query_vector, k, filter)` when the query is a KNNdiff --git a/engine/crates/xerj-vector/examples/recall_probe.rs b/engine/crates/xerj-vector/examples/recall_probe.rsnew file mode 100644index 000000000..0567f2897--- /dev/null+++ b/engine/crates/xerj-vector/examples/recall_probe.rs@@ -0,0 +1,124 @@+//! Isolated HNSW recall + graph-structure diagnostic (dev tool, not shipped).+//!+//! Builds the same corpus shape as the official kNN bench cell (50k x 128-d+//! uniform random, cosine), then reports recall@10 at several ef values and+//! layer-0 reachability from the entry point. Run with:+//! cargo run --release -p xerj-vector --example recall_probe [N] [DIM]++use xerj_vector::distance::DistanceMetric;+use xerj_vector::hnsw::{HnswIndex, HnswParams};++// Deterministic RNG (mulberry32, same family as the JS bench).+struct Rng(u32);+impl Rng {+ fn next_f64(&mut self) -> f64 {+ self.0 = self.0.wrapping_add(0x6D2B79F5);+ let mut t = self.0 as u64;+ t = (t ^ (t >> 15)).wrapping_mul(1 | t);+ t ^= t.wrapping_add((t ^ (t >> 7)).wrapping_mul(61 | t));+ ((t ^ (t >> 14)) & 0xFFFF_FFFF) as f64 / 4294967296.0+ }+}++fn cosine(a: &[f32], b: &[f32]) -> f32 {+ let (mut d, mut na, mut nb) = (0f32, 0f32, 0f32);+ for i in 0..a.len() {+ d += a[i] * b[i];+ na += a[i] * a[i];+ nb += b[i] * b[i];+ }+ d / (na.sqrt() * nb.sqrt()).max(1e-12)+}++fn main() {+ let args: Vec<String> = std::env::args().collect();+ let n: usize = args.get(1).map(|s| s.parse().unwrap()).unwrap_or(50_000);+ let dim: usize = args.get(2).map(|s| s.parse().unwrap()).unwrap_or(128);+ let k = 10;++ let mut rng = Rng(0xC0FFEE);+ let vecs: Vec<Vec<f32>> = (0..n)+ .map(|_| {+ (0..dim)+ .map(|_| (rng.next_f64() * 2.0 - 1.0) as f32)+ .collect()+ })+ .collect();++ let idx = HnswIndex::new(HnswParams::new(dim, DistanceMetric::Cosine));+ let t0 = std::time::Instant::now();+ for (i, v) in vecs.iter().enumerate() {+ idx.insert(i as u64 + 1, v.clone()).unwrap();+ if (i + 1) % 10_000 == 0 {+ eprintln!("inserted {} in {:?}", i + 1, t0.elapsed());+ }+ }+ eprintln!("build done in {:?}", t0.elapsed());++ // 20 probe queries from a different seed.+ let mut qrng = Rng(0xBEEF);+ let queries: Vec<Vec<f32>> = (0..20)+ .map(|_| {+ (0..dim)+ .map(|_| (qrng.next_f64() * 2.0 - 1.0) as f32)+ .collect()+ })+ .collect();++ // ── Graph-structure diagnostics via the debug stats hook ──+ let (ep, ep_layer, layer_hist, deg0_hist, reach0) = idx.debug_structure();+ eprintln!("entry_point={ep:?} entry_layer={ep_layer}");+ eprintln!("layer histogram (level → count): {layer_hist:?}");+ eprintln!("layer-0 out-degree histogram (bucket → count): {deg0_hist:?}");+ eprintln!("layer-0 BFS reachable from entry: {reach0} / {n}");++ // Pre-compute exact ground truth per query.+ let exact_sets: Vec<(std::collections::HashSet<u64>, u64, f32)> = queries+ .iter()+ .map(|q| {+ let mut scored: Vec<(usize, f32)> = vecs+ .iter()+ .enumerate()+ .map(|(i, v)| (i, cosine(q, v)))+ .collect();+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());+ let set: std::collections::HashSet<u64> =+ scored[..k].iter().map(|&(i, _)| i as u64 + 1).collect();+ (set, scored[0].0 as u64 + 1, 1.0 - scored[0].1) // (top-k set, true-NN id, true-NN distance)+ })+ .collect();++ // Descent diagnostics: where does the hierarchy drop us at layer 0?+ let mut dstats = Vec::new();+ for (q, (_, _nn, nnd)) in queries.iter().zip(&exact_sets) {+ if let Some((_start, d)) = idx.debug_descent(q) {+ dstats.push((d, *nnd));+ }+ }+ let avg_start: f32 = dstats.iter().map(|x| x.0).sum::<f32>() / dstats.len() as f32;+ let avg_nn: f32 = dstats.iter().map(|x| x.1).sum::<f32>() / dstats.len() as f32;+ eprintln!("descent: avg layer-0 start dist={avg_start:.4} avg true-NN dist={avg_nn:.4} (random pair ~= 1.0)");++ for ef in [100usize, 200, 400, 800] {+ let (mut recalls, mut oracle_recalls) = (Vec::new(), Vec::new());+ let mut total_ms = 0.0;+ for (q, (exact, nn_id, _)) in queries.iter().zip(&exact_sets) {+ let t = std::time::Instant::now();+ let got = idx.search(q, k, ef).unwrap();+ total_ms += t.elapsed().as_secs_f64() * 1e3;+ let hit = got.iter().filter(|(id, _)| exact.contains(id)).count();+ recalls.push(hit as f32 / k as f32);+ // Oracle start: beam from the TRUE nearest neighbor.+ let got_o = idx.debug_search_from(*nn_id, q, k, ef);+ let hit_o = got_o.iter().filter(|(id, _)| exact.contains(id)).count();+ oracle_recalls.push(hit_o as f32 / k as f32);+ }+ let mean: f32 = recalls.iter().sum::<f32>() / recalls.len() as f32;+ let min = recalls.iter().cloned().fold(f32::MAX, f32::min);+ let omean: f32 = oracle_recalls.iter().sum::<f32>() / oracle_recalls.len() as f32;+ println!(+ "ef={ef:4} mean_recall={mean:.4} min_recall={min:.3} oracle_start_recall={omean:.4} avg_search_ms={:.3}",+ total_ms / queries.len() as f64+ );+ }+}diff --git a/engine/crates/xerj-vector/src/hnsw.rs b/engine/crates/xerj-vector/src/hnsw.rsindex 04390bab8..059f67db7 100644--- a/engine/crates/xerj-vector/src/hnsw.rs+++ b/engine/crates/xerj-vector/src/hnsw.rs@@ -7,13 +7,25 @@//! - M=16 bidirectional connections per layer (good accuracy/memory trade-off)//! - ef_construction=200 (high recall during build)//! - Level selection uses the standard ln-based formula+//! - Neighbor selection uses the paper's Algorithm 4 diversity heuristic+//! (the variant hnswlib and Lucene use) — plain closest-M packs a node's+//! links into one tight cluster and measurably breaks navigability//! - Graph is entirely in-memory; persistence is handled by the storage crate+//!+//! In-memory layout (2026-07-12 flat-slab rework): nodes live in a single+//! slot-indexed slab — one contiguous `Vec<f32>` for vectors, `Vec<Vec<u32>>`+//! neighbor lists per layer, and a `Vec<u64>` bitmap for the search-time+//! visited set. The previous `HashMap<u64, Arc<RwLock<Node>>>` paid a hash+//! lookup + two pointer chases + an RwLock acquisition *per neighbor+//! expansion*, which made beam search ~5× slower than the distance math+//! itself. The on-disk format and the public (external-u64-id) API are+//! unchanged.use std::cmp::Reverse;use std::collections::{BinaryHeap, HashMap, HashSet};use std::io::{Read, Write};use std::path::Path;-use std::sync::{Arc, RwLock};+use std::sync::RwLock;use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};use rand::Rng;@@ -67,6 +79,38 @@ fn corrupt(msg: &str) -> XerjError {XerjError::storage(format!("HNSW graph corrupt: {msg}"))}+/// Dot product with eight independent accumulators.+///+/// A single-accumulator `for` loop is a strict serial FP dependency chain+/// that rustc must NOT auto-vectorize (float addition is not associative),+/// so the beam search was spending ~5 ns/dim in scalar fmadds. Eight+/// independent chains let LLVM keep multiple SIMD lanes busy; the exact+/// f32 sum differs from the serial order only in rounding, which is fine+/// here — graph distances only decide beam ORDER and callers rescore+/// exact similarities downstream.+#[inline]+fn dot_unrolled(a: &[f32], b: &[f32]) -> f32 {+ let mut acc = [0.0f32; 8];+ let ca = a.chunks_exact(8);+ let cb = b.chunks_exact(8);+ let (ra, rb) = (ca.remainder(), cb.remainder());+ for (x, y) in ca.zip(cb) {+ acc[0] += x[0] * y[0];+ acc[1] += x[1] * y[1];+ acc[2] += x[2] * y[2];+ acc[3] += x[3] * y[3];+ acc[4] += x[4] * y[4];+ acc[5] += x[5] * y[5];+ acc[6] += x[6] * y[6];+ acc[7] += x[7] * y[7];+ }+ let mut s = ((acc[0] + acc[4]) + (acc[1] + acc[5])) + ((acc[2] + acc[6]) + (acc[3] + acc[7]));+ for (x, y) in ra.iter().zip(rb.iter()) {+ s += x * y;+ }+ s+}+// ─────────────────────────────────────────────────────────────────────────────// Parameters// ─────────────────────────────────────────────────────────────────────────────@@ -96,31 +140,300 @@ impl HnswParams {}// ─────────────────────────────────────────────────────────────────────────────-// Internal node+// Flat slab storage// ─────────────────────────────────────────────────────────────────────────────-#[derive(Debug)]-struct Node {- /// The node's own id. Redundant with its key in the `nodes` map, so it is- /// never read back out — kept for `Debug` output and crash diagnostics.- #[allow(dead_code)]- id: u64,- vector: Vec<f32>,- /// Neighbor lists per layer. `neighbors[layer]` holds neighbor node IDs.- neighbors: Vec<Vec<u64>>,+/// Slot-indexed node storage. A `slot` is a dense `u32` handle assigned in+/// insertion order; all hot-path state is `slot`-addressed so beam search+/// runs on contiguous arrays with no per-node locks or hashing.+struct Inner {+ dim: usize,+ metric: DistanceMetric,+ /// slot → external node id.+ ids: Vec<u64>,+ /// external node id → slot.+ slot_of: HashMap<u64, u32>,+ /// Contiguous vector data, `slot * dim ..= slot * dim + dim`.+ vectors: Vec<f32>,+ /// slot → 1/‖v‖ (cosine metric only; 1.0 placeholder otherwise,+ /// 0.0 for zero-norm vectors so their cosine similarity is 0 — the+ /// same value the previous `cosine()` helper produced).+ inv_norms: Vec<f32>,+ /// slot → layer → neighbor slots.+ neighbors: Vec<Vec<Vec<u32>>>,+ /// slot → tombstoned? (soft delete; see `HnswIndex::mark_deleted`).+ tomb: Vec<bool>,+ /// External-id mirror of the tombstone set (for counting and+ /// persistence; may contain ids the graph no longer knows).+ tomb_ids: HashSet<u64>,+ entry_slot: Option<u32>,+ entry_layer: usize,}-impl Node {- fn new(id: u64, vector: Vec<f32>, max_layer: usize, m: usize) -> Self {- let neighbors = (0..=max_layer)- .map(|l| Vec::with_capacity(if l == 0 { 2 * m } else { m }))- .collect();+impl Inner {+ fn new(dim: usize, metric: DistanceMetric) -> Self {Self {- id,- vector,- neighbors,+ dim,+ metric,+ ids: Vec::new(),+ slot_of: HashMap::new(),+ vectors: Vec::new(),+ inv_norms: Vec::new(),+ neighbors: Vec::new(),+ tomb: Vec::new(),+ tomb_ids: HashSet::new(),+ entry_slot: None,+ entry_layer: 0,+ }+ }++ #[inline]+ fn len(&self) -> usize {+ self.ids.len()+ }++ #[inline]+ fn vec_of(&self, slot: u32) -> &[f32] {+ let s = slot as usize * self.dim;+ &self.vectors[s..s + self.dim]+ }++ /// The `1/‖q‖` scale a query needs for the cosine fast path+ /// (1.0 for other metrics, 0.0 for a zero query).+ #[inline]+ fn query_inv(&self, q: &[f32]) -> f32 {+ if self.metric == DistanceMetric::Cosine {+ let n: f32 = q.iter().map(|x| x * x).sum::<f32>().sqrt();+ if n > 0.0 {+ 1.0 / n+ } else {+ 0.0+ }+ } else {+ 1.0}}++ /// Distance from an (external) query vector to a slot.+ ///+ /// Cosine uses cached per-slot inverse norms so the per-pair work is a+ /// single dot product: `1 - dot(q,v)·(1/‖q‖)·(1/‖v‖)`. This matches+ /// `compute_distance`'s `1 - cos` up to fp rounding — acceptable+ /// because graph distances only decide beam ORDER; callers rescore+ /// exact similarities downstream.+ #[inline]+ fn slot_dist(&self, q: &[f32], q_inv: f32, slot: u32) -> f32 {+ let v = self.vec_of(slot);+ match self.metric {+ DistanceMetric::Cosine => {+ 1.0 - dot_unrolled(q, v) * q_inv * self.inv_norms[slot as usize]+ }+ _ => compute_distance(self.metric, q, v),+ }+ }++ /// Hint the CPU to start pulling `slot`'s vector into cache. The+ /// vector slab for a 50k × 128-d graph is ~25 MB (larger than L3), so+ /// beam search reads are DRAM-latency-bound without this; prefetching+ /// the next neighbor's lines while the current distance computes hides+ /// most of that latency.+ ///+ /// Safety: `_mm_prefetch` is architecturally a hint — it cannot fault,+ /// and the pointer is formed from an in-bounds slot (neighbor lists+ /// only hold allocated slots). No-op on non-x86_64.+ #[inline(always)]+ fn prefetch_slot(&self, slot: u32) {+ #[cfg(target_arch = "x86_64")]+ unsafe {+ use core::arch::x86_64::{_mm_prefetch, _MM_HINT_T0};+ let base = self.vectors.as_ptr().add(slot as usize * self.dim) as *const i8;+ // First two cache lines; the hardware streamer follows the+ // sequential reads for the rest of the vector.+ _mm_prefetch(base, _MM_HINT_T0);+ if self.dim > 16 {+ _mm_prefetch(base.add(64), _MM_HINT_T0);+ }+ }+ #[cfg(not(target_arch = "x86_64"))]+ {+ let _ = slot;+ }+ }++ /// Append a node to the slab. Returns its slot.+ fn alloc_slot(&mut self, id: u64, vector: Vec<f32>, node_level: usize, m: usize) -> u32 {+ let slot = self.ids.len() as u32;+ let inv = if self.metric == DistanceMetric::Cosine {+ let n: f32 = vector.iter().map(|x| x * x).sum::<f32>().sqrt();+ if n > 0.0 {+ 1.0 / n+ } else {+ 0.0+ }+ } else {+ 1.0+ };+ self.ids.push(id);+ self.slot_of.insert(id, slot);+ self.vectors.extend_from_slice(&vector);+ self.inv_norms.push(inv);+ self.neighbors.push(+ (0..=node_level)+ .map(|l| Vec::with_capacity(if l == 0 { 2 * m } else { m }))+ .collect(),+ );+ self.tomb.push(self.tomb_ids.contains(&id));+ slot+ }++ /// Greedy hill-climb to the locally-closest node at `layer`.+ fn greedy_layer(&self, q: &[f32], q_inv: f32, start: u32, layer: usize) -> (u32, f32) {+ let mut best = start;+ let mut best_dist = self.slot_dist(q, q_inv, start);+ loop {+ let mut improved = false;+ let nbrs = &self.neighbors[best as usize];+ if layer >= nbrs.len() {+ break;+ }+ for &nb in &nbrs[layer] {+ let d = self.slot_dist(q, q_inv, nb);+ if d < best_dist {+ best_dist = d;+ best = nb;+ improved = true;+ }+ }+ if !improved {+ break;+ }+ }+ (best, best_dist)+ }++ /// Beam search at `layer`, returning up to `ef` candidates sorted by+ /// distance ascending. `admit` gates entry into the RESULT set only;+ /// non-admitted nodes are still traversed (excluding them from+ /// traversal would disconnect their neighbours and collapse recall).+ fn beam(+ &self,+ q: &[f32],+ q_inv: f32,+ entry: u32,+ ef: usize,+ layer: usize,+ admit: &dyn Fn(u32) -> bool,+ ) -> Vec<(f32, u32)> {+ let n = self.len();+ if entry as usize >= n {+ return vec![];+ }+ // Bitmap visited set: ~6 KB for a 50k graph, allocation-cheap and+ // O(1) test-and-set (the old per-search HashSet<u64> hashed every+ // neighbor expansion).+ let mut visited = vec![0u64; n.div_ceil(64)];+ #[inline]+ fn test_and_set(bits: &mut [u64], i: u32) -> bool {+ let (w, b) = ((i / 64) as usize, i % 64);+ let seen = bits[w] & (1 << b) != 0;+ bits[w] |= 1 << b;+ seen+ }++ let entry_dist = self.slot_dist(q, q_inv, entry);+ // Min-heap of (dist, slot) candidates to explore.+ let mut candidates: BinaryHeap<Reverse<(ordered_float::OrderedFloat, u32)>> =+ BinaryHeap::with_capacity(ef * 4);+ // Max-heap of (dist, slot) for the result set W.+ let mut w: BinaryHeap<(ordered_float::OrderedFloat, u32)> =+ BinaryHeap::with_capacity(ef + 1);++ candidates.push(Reverse((ordered_float::OrderedFloat(entry_dist), entry)));+ if admit(entry) {+ w.push((ordered_float::OrderedFloat(entry_dist), entry));+ }+ test_and_set(&mut visited, entry);++ while let Some(Reverse((dist_c, c))) = candidates.pop() {+ // Pruning: if the closest unexplored candidate is farther than+ // the farthest kept result and W is full, stop.+ if let Some(&(farthest, _)) = w.peek() {+ if dist_c > farthest && w.len() >= ef {+ break;+ }+ }+ let nbrs = &self.neighbors[c as usize];+ if layer >= nbrs.len() {+ continue;+ }+ let list = &nbrs[layer];+ // Start pulling the first vectors while the loop spins up.+ for &nb in list.iter().take(2) {+ self.prefetch_slot(nb);+ }+ for (i, &nb) in list.iter().enumerate() {+ if i + 2 < list.len() {+ self.prefetch_slot(list[i + 2]);+ }+ if test_and_set(&mut visited, nb) {+ continue;+ }+ let d = self.slot_dist(q, q_inv, nb);+ let farthest_w = w.peek().map(|&(f, _)| f.0).unwrap_or(f32::MAX);+ if d < farthest_w || w.len() < ef {+ candidates.push(Reverse((ordered_float::OrderedFloat(d), nb)));+ if admit(nb) {+ w.push((ordered_float::OrderedFloat(d), nb));+ if w.len() > ef {+ w.pop();+ }+ }+ }+ }+ }++ let mut results: Vec<(f32, u32)> = w.into_iter().map(|(d, s)| (d.0, s)).collect();+ results.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());+ results+ }++ /// Neighbor selection — Malkov & Yashunin Algorithm 4 (the "heuristic"+ /// variant used by hnswlib and Lucene): walk candidates in ascending+ /// distance-to-base order and keep a candidate only if it is closer to+ /// the base than to every already-kept neighbor. Enforces direction+ /// diversity in each node's link list; plain closest-M packed all links+ /// into one tight cluster (recall@10 0.565 @ ef=100 on 50k random+ /// 128-d where the heuristic-built graph reaches ~0.95+).+ fn select_diverse(&self, cands: &[(f32, u32)], m: usize) -> Vec<(f32, u32)> {+ let mut sorted = cands.to_vec();+ sorted.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());+ let mut kept: Vec<(f32, u32)> = Vec::with_capacity(m);+ for &(d_c, c) in &sorted {+ if kept.len() >= m {+ break;+ }+ let cv = self.vec_of(c);+ let ci = self.inv_norms[c as usize];+ // Diverse iff no already-kept neighbor is closer to the+ // candidate than the base is.+ let diverse = kept.iter().all(|&(_, r)| self.slot_dist(cv, ci, r) >= d_c);+ if diverse {+ kept.push((d_c, c));+ }+ }+ kept+ }++ /// Distances from `base` to each of its current neighbors at `layer`+ /// (used when a saturated node re-selects after gaining a back-link).+ fn backlink_dists(&self, base: u32, layer: usize) -> Vec<(f32, u32)> {+ let bv = self.vec_of(base);+ let bi = self.inv_norms[base as usize];+ self.neighbors[base as usize][layer]+ .iter()+ .map(|&nb| (self.slot_dist(bv, bi, nb), nb))+ .collect()+ }}// ─────────────────────────────────────────────────────────────────────────────@@ -128,29 +441,24 @@ impl Node {// ─────────────────────────────────────────────────────────────────────────────/// A thread-safe HNSW approximate nearest-neighbor index.+///+/// Soft deletes: deleting a doc removes its entry from the caller's id+/// maps but the node and its neighbour edges stay in the graph until a+/// graph compaction (planned v0.7+). Tombstoned nodes are skipped as+/// results but still traversed. Tombstones persist with the graph+/// (format v2+).pub struct HnswIndex {params: HnswParams,- nodes: RwLock<HashMap<u64, Arc<RwLock<Node>>>>,- entry_point: RwLock<Option<u64>>,- entry_layer: RwLock<usize>,- /// Soft-delete bitmap. v0.6.2: deleting a doc removes its- /// entry from `Index.hnsw_id_map` but the node and its- /// neighbour edges stay in the graph until a graph compaction- /// (planned v0.7+). Adding to this set causes search to skip- /// the node — both as a result and as a traversal target.- /// Persisted alongside the graph (format version 2+).- tombstones: RwLock<HashSet<u64>>,+ inner: RwLock<Inner>,}impl HnswIndex {/// Create a new empty index.pub fn new(params: HnswParams) -> Self {+ let inner = Inner::new(params.dim, params.metric);Self {params,- nodes: RwLock::new(HashMap::new()),- entry_point: RwLock::new(None),- entry_layer: RwLock::new(0),- tombstones: RwLock::new(HashSet::new()),+ inner: RwLock::new(inner),}}@@ -158,24 +466,32 @@ impl HnswIndex {/// return it as a hit. The node and its neighbour edges remain/// until a graph compaction (planned v0.7+). Idempotent.pub fn mark_deleted(&self, id: u64) {- self.tombstones.write().unwrap().insert(id);+ let mut g = self.inner.write().unwrap();+ if let Some(&slot) = g.slot_of.get(&id) {+ g.tomb[slot as usize] = true;+ }+ g.tomb_ids.insert(id);}/// Reverse of `mark_deleted` — used in tests and on un-delete //// reindex paths.pub fn unmark_deleted(&self, id: u64) {- self.tombstones.write().unwrap().remove(&id);+ let mut g = self.inner.write().unwrap();+ if let Some(&slot) = g.slot_of.get(&id) {+ g.tomb[slot as usize] = false;+ }+ g.tomb_ids.remove(&id);}/// True if `id` has been tombstoned via `mark_deleted`.pub fn is_deleted(&self, id: u64) -> bool {- self.tombstones.read().unwrap().contains(&id)+ self.inner.read().unwrap().tomb_ids.contains(&id)}/// Number of tombstones currently held. Operators can monitor/// this to decide when to schedule a compaction.pub fn tombstone_count(&self) -> usize {- self.tombstones.read().unwrap().len()+ self.inner.read().unwrap().tomb_ids.len()}pub fn params(&self) -> &HnswParams {@@ -184,7 +500,7 @@ impl HnswIndex {/// Number of indexed vectors.pub fn len(&self) -> usize {- self.nodes.read().unwrap().len()+ self.inner.read().unwrap().len()}pub fn is_empty(&self) -> bool {@@ -200,12 +516,6 @@ impl HnswIndex {level.min(16) // cap at 16 layers}- // ── Distance helper ───────────────────────────────────────────────────-- fn dist(&self, a: &[f32], b: &[f32]) -> f32 {- compute_distance(self.params.metric, a, b)- }-// ── Core HNSW algorithm ───────────────────────────────────────────────/// Insert a single vector with the given ID.@@ -225,76 +535,57 @@ impl HnswIndex {let m = self.params.m;let ef = self.params.ef_construction;- // Build the node (not yet inserted into the map)- let node = Arc::new(RwLock::new(Node::new(id, vector.clone(), node_level, m)));-- // Handle first insertion- {- let mut ep = self.entry_point.write().unwrap();- if ep.is_none() {- *ep = Some(id);- *self.entry_layer.write().unwrap() = node_level;- self.nodes.write().unwrap().insert(id, node);+ let mut g = self.inner.write().unwrap();++ // Query view of the new vector for the construction beams (the+ // slab owns the canonical copy after alloc).+ let q = vector.clone();+ let q_inv = g.query_inv(&q);++ // The new slot has no in-edges yet, so allocating it before the+ // beams run cannot perturb them — and it means the back-link+ // re-selection below can score the new node like any other+ // neighbor (the pre-slab code dropped the fresh back-link on the+ // floor here, disconnecting the graph: layer-0 BFS from the entry+ // reached 34 of 50k nodes and recall@10 was 0.000).+ let slot = g.alloc_slot(id, vector, node_level, m);++ // Handle first insertion.+ let (ep_slot, ep_layer) = match g.entry_slot {+ None => {+ g.entry_slot = Some(slot);+ g.entry_layer = node_level;debug!("HNSW: first node id={id} level={node_level}");return Ok(());}- }-- // Greedy descent from top layer to node_level+1- let ep_id = self.entry_point.read().unwrap().unwrap();- let ep_layer = *self.entry_layer.read().unwrap();-- // Greedy descent tracks only the current entry point; the distance to- // it is recomputed by `search_layer`/`greedy_search_layer` at each step,- // so we don't carry it between layers.- let mut curr_ep = ep_id;+ Some(ep) => (ep, g.entry_layer),+ };+ // Greedy descent from the top layer down to node_level+1.+ let mut curr_ep = ep_slot;for layer in (node_level + 1..=ep_layer).rev() {- curr_ep = self.greedy_search_layer(&vector, curr_ep, layer, 1).0;+ curr_ep = g.greedy_layer(&q, q_inv, curr_ep, layer).0;}- // Search and connect from node_level down to 0+ // Search and connect from min(node_level, ep_layer) down to 0.for layer in (0..=node_level.min(ep_layer)).rev() {let m_layer = if layer == 0 { 2 * m } else { m };- let candidates = self.search_layer(&vector, curr_ep, ef, layer);-- // Select M nearest- let selected = self.select_neighbors(&vector, &candidates, m_layer);-- // Connect new node → selected- {- let mut n = node.write().unwrap();- if layer < n.neighbors.len() {- n.neighbors[layer] = selected.iter().map(|&(_, nid)| nid).collect();- }- }-- // Update existing nodes: add back-link to new node, prune to m_layer- {- let nodes = self.nodes.read().unwrap();- for &(_, nid) in &selected {- if let Some(existing) = nodes.get(&nid) {- let mut en = existing.write().unwrap();- if layer < en.neighbors.len() {- en.neighbors[layer].push(id);- if en.neighbors[layer].len() > m_layer {- // Re-select neighbors to keep M_layer best- let ev = en.vector.clone();- let neighbor_dists: Vec<(f32, u64)> = en.neighbors[layer]- .iter()- .filter_map(|&nid2| {- nodes.get(&nid2).map(|nn| {- let nv = nn.read().unwrap();- (self.dist(&ev, &nv.vector), nid2)- })- })- .collect();- let pruned = self.select_neighbors(&ev, &neighbor_dists, m_layer);- en.neighbors[layer] =- pruned.iter().map(|&(_, nid2)| nid2).collect();- }- }- }+ let candidates = g.beam(&q, q_inv, curr_ep, ef, layer, &|_| true);++ // Select up to M diverse nearest (Algorithm 4 heuristic).+ let selected = g.select_diverse(&candidates, m_layer);++ // Connect new node → selected.+ g.neighbors[slot as usize][layer] = selected.iter().map(|&(_, s)| s).collect();++ // Back-links: add new node to each selected neighbor, and+ // re-select when a neighbor exceeds its link budget.+ for &(_, sel) in &selected {+ g.neighbors[sel as usize][layer].push(slot);+ if g.neighbors[sel as usize][layer].len() > m_layer {+ let dists = g.backlink_dists(sel, layer);+ let pruned = g.select_diverse(&dists, m_layer);+ g.neighbors[sel as usize][layer] = pruned.iter().map(|&(_, s)| s).collect();}}@@ -303,17 +594,10 @@ impl HnswIndex {}}- // Insert the node- self.nodes.write().unwrap().insert(id, node);-- // Update entry point if new node is on a higher layer- {- let mut ep = self.entry_point.write().unwrap();- let mut ep_layer = self.entry_layer.write().unwrap();- if node_level > *ep_layer {- *ep = Some(id);- *ep_layer = node_level;- }+ // Update entry point if the new node reached a higher layer.+ if node_level > g.entry_layer {+ g.entry_slot = Some(slot);+ g.entry_layer = node_level;}trace!("HNSW: inserted id={id} level={node_level}");@@ -347,42 +631,33 @@ impl HnswIndex {)));}- // Snapshot tombstones once and clone-into-the-closure so the- // beam search sees a consistent set even if mark_deleted runs- // mid-query. Cloning a u64 HashSet is cheap and avoids holding- // the read lock across the whole traversal.- let tombstoned: HashSet<u64> = self.tombstones.read().unwrap().clone();- let combined_filter = move |id: u64| -> bool { !tombstoned.contains(&id) && filter(id) };-- let ep_opt = *self.entry_point.read().unwrap();- let ep_id = match ep_opt {- Some(id) => id,+ let g = self.inner.read().unwrap();+ let ep_slot = match g.entry_slot {+ Some(s) => s,None => return Ok(vec![]),};- let ep_layer = *self.entry_layer.read().unwrap();-- let mut curr_ep = ep_id;+ let q_inv = g.query_inv(query);- // Greedy descent from top to layer 1- for layer in (1..=ep_layer).rev() {- let (new_ep, _) = self.greedy_search_layer(query, curr_ep, layer, 1);- curr_ep = new_ep;+ // Greedy descent from the top layer to layer 1.+ let mut curr_ep = ep_slot;+ for layer in (1..=g.entry_layer).rev() {+ curr_ep = g.greedy_layer(query, q_inv, curr_ep, layer).0;}- // Beam search at layer 0 with ef. Tombstoned ids are kept as- // *traversal* candidates (otherwise we lose access to their- // neighbours and recall collapses) but excluded from results- // by the post-take filter below.- let candidates = self.search_layer_filtered(query, curr_ep, ef.max(k), 0, &combined_filter);+ // Beam search at layer 0. Tombstoned slots are kept as *traversal*+ // candidates (otherwise we lose access to their neighbours and+ // recall collapses) but excluded from the result set. The whole+ // search runs under one read guard, so it sees a consistent+ // tombstone set even if mark_deleted runs mid-query.+ let admit = |s: u32| !g.tomb[s as usize] && filter(g.ids[s as usize]);+ let candidates = g.beam(query, q_inv, curr_ep, ef.max(k), 0, &admit);- let results: Vec<(u64, f32)> = candidates+ Ok(candidates.into_iter()- .filter(|&(_, id)| combined_filter(id))+ .filter(|&(_, s)| admit(s)).take(k)- .map(|(dist, id)| (id, dist))- .collect();-- Ok(results)+ .map(|(dist, s)| (g.ids[s as usize], dist))+ .collect())}/// Insert a batch of `(id, vector)` items **serially**, one after another.@@ -398,15 +673,15 @@ impl HnswIndex {////// Each item is validated by `insert` (dimension check); a mismatch/// returns `Err` and leaves already-inserted items in the graph (this- /// method is not transactional / all-or-nothing).+ /// API is not transactional).pub fn insert_batch(&self, items: Vec<(u64, Vec<f32>)>) -> Result<()> {- for (id, vec) in items {- self.insert(id, vec)?;+ for (id, vector) in items {+ self.insert(id, vector)?;}Ok(())}- // ── Persistence ───────────────────────────────────────────────────────+ // ── Persistence ──────────────────────────────────────────────────────//// Wire format (little-endian, single file, written atomically via// tmp + rename):@@ -415,7 +690,7 @@ impl HnswIndex {// format_ver: u32 = HNSW_FORMAT_VERSION// m: u32// ef_construction:u32- // metric: u8 = DistanceMetric repr (0=cosine,1=l2,2=dot,3=mip)+ // metric: u8 = DistanceMetric repr (0=cosine,1=l2,2=dot)// dim: u32// entry_point: u64 = entry node id (u64::MAX = no entry)// entry_layer: u32@@ -427,6 +702,7 @@ impl HnswIndex {// for layer 0..=max_layer:// num_neigh: u32// neigh_ids: u64 × num_neigh+ // [v2] tombstones: u32 num + u64 × num// crc32c: u32 (over all preceding bytes)//// Pre-v0.6.2 the graph was rebuilt on every restart by replaying@@ -457,45 +733,50 @@ impl HnswIndex {buf.write_u32::<LittleEndian>(self.params.dim as u32).map_err(io_to_xerj)?;- let ep = *self.entry_point.read().unwrap();- let el = *self.entry_layer.read().unwrap();- buf.write_u64::<LittleEndian>(ep.unwrap_or(u64::MAX))+ let g = self.inner.read().unwrap();+ let ep_id = g.entry_slot.map(|s| g.ids[s as usize]);+ buf.write_u64::<LittleEndian>(ep_id.unwrap_or(u64::MAX)).map_err(io_to_xerj)?;- buf.write_u32::<LittleEndian>(el as u32)+ buf.write_u32::<LittleEndian>(g.entry_layer as u32).map_err(io_to_xerj)?;- let nodes = self.nodes.read().unwrap();- buf.write_u32::<LittleEndian>(nodes.len() as u32)+ // Live slots only (a re-used external id orphans its old slot;+ // the map wrote each external id once and so do we).+ let live: Vec<u32> = (0..g.len() as u32)+ .filter(|&s| g.slot_of.get(&g.ids[s as usize]) == Some(&s))+ .collect();+ buf.write_u32::<LittleEndian>(live.len() as u32).map_err(io_to_xerj)?;- for (id, node_arc) in nodes.iter() {- let n = node_arc.read().unwrap();- buf.write_u64::<LittleEndian>(*id).map_err(io_to_xerj)?;- let max_layer = n.neighbors.len().saturating_sub(1) as u32;+ for &s in &live {+ buf.write_u64::<LittleEndian>(g.ids[s as usize])+ .map_err(io_to_xerj)?;+ let nbrs = &g.neighbors[s as usize];+ let max_layer = nbrs.len().saturating_sub(1) as u32;buf.write_u32::<LittleEndian>(max_layer).map_err(io_to_xerj)?;- for &v in &n.vector {+ for &v in g.vec_of(s) {buf.write_f32::<LittleEndian>(v).map_err(io_to_xerj)?;}- for layer_neighbors in &n.neighbors {+ for layer_neighbors in nbrs {buf.write_u32::<LittleEndian>(layer_neighbors.len() as u32).map_err(io_to_xerj)?;- for &nid in layer_neighbors {- buf.write_u64::<LittleEndian>(nid).map_err(io_to_xerj)?;+ for &nb in layer_neighbors {+ buf.write_u64::<LittleEndian>(g.ids[nb as usize])+ .map_err(io_to_xerj)?;}}}- drop(nodes);// v2 — tombstone block. Empty for indices that have never had// a delete; v1 readers don't know to skip past this, so the// format version was bumped (loader checks).- let tombs = self.tombstones.read().unwrap();- buf.write_u32::<LittleEndian>(tombs.len() as u32)+ buf.write_u32::<LittleEndian>(g.tomb_ids.len() as u32).map_err(io_to_xerj)?;- for &t in tombs.iter() {+ for &t in g.tomb_ids.iter() {buf.write_u64::<LittleEndian>(t).map_err(io_to_xerj)?;}- drop(tombs);+ let node_count = live.len();+ drop(g);let crc = crc32fast::hash(&buf);buf.write_u32::<LittleEndian>(crc).map_err(io_to_xerj)?;@@ -506,7 +787,7 @@ impl HnswIndex {debug!(path = %path.display(),bytes = buf.len(),- nodes = self.nodes.read().unwrap().len(),+ nodes = node_count,"HNSW graph persisted");Ok(())@@ -561,8 +842,10 @@ impl HnswIndex {metric,dim,};- let mut node_map: HashMap<u64, Arc<RwLock<Node>>> = HashMap::with_capacity(num_nodes);+ // Pass 1: read raw nodes (neighbors keyed by external id).+ let mut g = Inner::new(dim, metric);+ let mut raw_neighbors: Vec<Vec<Vec<u64>>> = Vec::with_capacity(num_nodes);for _ in 0..num_nodes {let id = cur.read_u64::<LittleEndian>().map_err(io_to_xerj)?;let max_layer = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)? as usize;@@ -579,189 +862,139 @@ impl HnswIndex {}neighbors.push(layer);}- node_map.insert(- id,- Arc::new(RwLock::new(Node {- id,- vector,- neighbors,- })),- );+ g.alloc_slot(id, vector, max_layer, m);+ raw_neighbors.push(neighbors);}+ // Pass 2: map neighbor external ids → slots (ids referencing+ // unknown nodes are dropped, mirroring the old map's `get` skips).+ for (slot, layers) in raw_neighbors.into_iter().enumerate() {+ let converted: Vec<Vec<u32>> = layers+ .into_iter()+ .map(|layer| {+ layer+ .into_iter()+ .filter_map(|nid| g.slot_of.get(&nid).copied())+ .collect()+ })+ .collect();+ g.neighbors[slot] = converted;+ }++ g.entry_slot = if ep_raw == u64::MAX {+ None+ } else {+ match g.slot_of.get(&ep_raw).copied() {+ Some(s) => Some(s),+ None => return Err(corrupt("entry point references unknown node id")),+ }+ };+ g.entry_layer = entry_layer;+// v2+: read trailing tombstone block. v1 files have no// tombstones and we treat the set as empty.- let mut tombstones: HashSet<u64> = HashSet::new();if format_ver >= 2 {let num_t = cur.read_u32::<LittleEndian>().map_err(io_to_xerj)? as usize;- tombstones.reserve(num_t);for _ in 0..num_t {- tombstones.insert(cur.read_u64::<LittleEndian>().map_err(io_to_xerj)?);+ let t = cur.read_u64::<LittleEndian>().map_err(io_to_xerj)?;+ if let Some(&slot) = g.slot_of.get(&t) {+ g.tomb[slot as usize] = true;+ }+ g.tomb_ids.insert(t);}}Ok(Self {params,- nodes: RwLock::new(node_map),- entry_point: RwLock::new(if ep_raw == u64::MAX {- None- } else {- Some(ep_raw)- }),- entry_layer: RwLock::new(entry_layer),- tombstones: RwLock::new(tombstones),+ inner: RwLock::new(g),})}- // ── Internal search primitives ────────────────────────────────────────+ // ── Debug / diagnostic hooks (dev tooling only) ──────────────────────- /// Greedy search: descend to the single closest node at a given layer.- fn greedy_search_layer(+ /// Run the layer-0 beam search from an explicit entry node (bypassing+ /// the hierarchy descent). Lets a probe distinguish "descent lands in+ /// the wrong region" from "layer-0 beam cannot navigate".+ pub fn debug_search_from(&self,- query: &[f32],entry: u64,- layer: usize,- _ef: usize,- ) -> (u64, f32) {- let nodes = self.nodes.read().unwrap();- let mut best_id = entry;- let mut best_dist = {- let n = nodes[&entry].read().unwrap();- self.dist(query, &n.vector)+ query: &[f32],+ k: usize,+ ef: usize,+ ) -> Vec<(u64, f32)> {+ let g = self.inner.read().unwrap();+ let entry_slot = match g.slot_of.get(&entry) {+ Some(&s) => s,+ None => return vec![],};-- loop {- let mut improved = false;- let neighbors: Vec<u64> = {- let n = nodes[&best_id].read().unwrap();- if layer < n.neighbors.len() {- n.neighbors[layer].clone()- } else {- vec![]- }- };-- for nid in neighbors {- if let Some(nn) = nodes.get(&nid) {- let d = {- let nv = nn.read().unwrap();- self.dist(query, &nv.vector)- };- if d < best_dist {- best_dist = d;- best_id = nid;- improved = true;- }- }- }-- if !improved {- break;- }- }-- (best_id, best_dist)+ let q_inv = g.query_inv(query);+ let admit = |s: u32| !g.tomb[s as usize];+ let candidates = g.beam(query, q_inv, entry_slot, ef.max(k), 0, &admit);+ candidates+ .into_iter()+ .take(k)+ .map(|(dist, s)| (g.ids[s as usize], dist))+ .collect()}- /// Beam search at a given layer, returning candidates sorted by distance.- fn search_layer(&self, query: &[f32], entry: u64, ef: usize, layer: usize) -> Vec<(f32, u64)> {- self.search_layer_filtered(query, entry, ef, layer, &|_| true)+ /// The layer-0 start node the hierarchy descent chooses for `query`,+ /// plus its distance.+ pub fn debug_descent(&self, query: &[f32]) -> Option<(u64, f32)> {+ let g = self.inner.read().unwrap();+ let ep = g.entry_slot?;+ let q_inv = g.query_inv(query);+ let mut curr = ep;+ let mut curr_dist = g.slot_dist(query, q_inv, curr);+ for layer in (1..=g.entry_layer).rev() {+ let (s, d) = g.greedy_layer(query, q_inv, curr, layer);+ curr = s;+ curr_dist = d;+ }+ Some((g.ids[curr as usize], curr_dist))}- fn search_layer_filtered(+ /// Graph structure summary: (entry_point, entry_layer, level histogram,+ /// layer-0 out-degree histogram, layer-0 BFS reachable-node count from+ /// the entry point).+ pub fn debug_structure(&self,- query: &[f32],- entry: u64,- ef: usize,- layer: usize,- filter: &dyn Fn(u64) -> bool,- ) -> Vec<(f32, u64)> {- let nodes = self.nodes.read().unwrap();-- let entry_dist = {- match nodes.get(&entry) {- Some(n) => self.dist(query, &n.read().unwrap().vector),- None => return vec![],- }- };-- // Min-heap of (dist, id) candidates to explore- let mut candidates: BinaryHeap<Reverse<(ordered_float::OrderedFloat, u64)>> =- BinaryHeap::new();- // Max-heap of (dist, id) for the result set W- let mut w: BinaryHeap<(ordered_float::OrderedFloat, u64)> = BinaryHeap::new();- let mut visited: HashSet<u64> = HashSet::new();-- candidates.push(Reverse((ordered_float::OrderedFloat(entry_dist), entry)));- if filter(entry) {- w.push((ordered_float::OrderedFloat(entry_dist), entry));+ ) -> (+ Option<u64>,+ usize,+ Vec<(usize, usize)>,+ Vec<(usize, usize)>,+ usize,+ ) {+ let g = self.inner.read().unwrap();+ let ep = g.entry_slot.map(|s| g.ids[s as usize]);+ let mut level_hist: std::collections::BTreeMap<usize, usize> = Default::default();+ let mut deg0_hist: std::collections::BTreeMap<usize, usize> = Default::default();+ for nbrs in &g.neighbors {+ *level_hist.entry(nbrs.len() - 1).or_default() += 1;+ *deg0_hist.entry(nbrs[0].len()).or_default() += 1;}- visited.insert(entry);-- while let Some(Reverse((dist_c, c))) = candidates.pop() {- // Pruning: if closest candidate is farther than the farthest result, stop- if let Some(&(farthest, _)) = w.peek() {- if dist_c > farthest && w.len() >= ef {- break;- }- }-- let neighbors: Vec<u64> = {- match nodes.get(&c) {- Some(n) => {- let n = n.read().unwrap();- if layer < n.neighbors.len() {- n.neighbors[layer].clone()- } else {- vec![]- }- }- None => continue,- }- };-- for nid in neighbors {- if visited.contains(&nid) {- continue;- }- visited.insert(nid);-- let d = match nodes.get(&nid) {- Some(nn) => self.dist(query, &nn.read().unwrap().vector),- None => continue,- };-- let farthest_w = w.peek().map(|&(f, _)| f.0).unwrap_or(f32::MAX);-- if d < farthest_w || w.len() < ef {- candidates.push(Reverse((ordered_float::OrderedFloat(d), nid)));- if filter(nid) {- w.push((ordered_float::OrderedFloat(d), nid));- if w.len() > ef {- w.pop();- }+ let mut reach = 0usize;+ if let Some(start) = g.entry_slot {+ let mut seen = vec![false; g.len()];+ let mut stack = vec![start];+ seen[start as usize] = true;+ while let Some(c) = stack.pop() {+ reach += 1;+ for &nb in &g.neighbors[c as usize][0] {+ if !seen[nb as usize] {+ seen[nb as usize] = true;+ stack.push(nb);}}}}-- // Collect results sorted by distance ascending- let mut results: Vec<(f32, u64)> = w.into_iter().map(|(d, id)| (d.0, id)).collect();- results.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());- results- }-- /// Simple neighbor selection: take the M closest candidates.- fn select_neighbors(- &self,- _query: &[f32],- candidates: &[(f32, u64)],- m: usize,- ) -> Vec<(f32, u64)> {- let mut sorted = candidates.to_vec();- sorted.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap());- sorted.truncate(m);- sorted+ (+ ep,+ g.entry_layer,+ level_hist.into_iter().collect(),+ deg0_hist.into_iter().collect(),+ reach,+ )}}@@ -971,4 +1204,72 @@ mod tests {assert_ne!(*id, 12);}}++ #[test]+ fn large_random_build_recall_regression() {+ // Regression for two 2026-07-12 defects that only show at scale:+ // 1. back-link pruning dropped the new node (map-miss), leaving+ // the graph disconnected — recall@10 was 0.000 at 50k;+ // 2. closest-M neighbor selection (no diversity heuristic) gave a+ // much flatter recall/ef curve.+ // 5k × 32-d keeps the test fast while still exercising saturated+ // back-link pruning (>> M² nodes). At ef=400 over 5k random+ // vectors, a healthy graph is essentially exact.+ struct Rng(u64);+ impl Rng {+ fn next(&mut self) -> f32 {+ // xorshift64*+ self.0 ^= self.0 >> 12;+ self.0 ^= self.0 << 25;+ self.0 ^= self.0 >> 27;+ ((self.0.wrapping_mul(0x2545F4914F6CDD1D) >> 33) as f64 / 2147483648.0) as f32 - 1.0+ }+ }+ let (n, dim, k) = (5000usize, 32usize, 10usize);+ let mut rng = Rng(0xC0FFEE);+ let vecs: Vec<Vec<f32>> = (0..n)+ .map(|_| (0..dim).map(|_| rng.next()).collect())+ .collect();+ let idx = make_index(dim);+ for (i, v) in vecs.iter().enumerate() {+ idx.insert(i as u64, v.clone()).unwrap();+ }+ // Layer-0 BFS from the entry must reach (almost) every node.+ let (_, _, _, _, reach) = idx.debug_structure();+ assert!(+ reach as f64 >= 0.99 * n as f64,+ "layer-0 graph disconnected: BFS reached {reach}/{n}"+ );++ let cos = |a: &[f32], b: &[f32]| -> f32 {+ let (mut d, mut na, mut nb) = (0f32, 0f32, 0f32);+ for i in 0..a.len() {+ d += a[i] * b[i];+ na += a[i] * a[i];+ nb += b[i] * b[i];+ }+ d / (na.sqrt() * nb.sqrt()).max(1e-12)+ };+ let mut total_hits = 0usize;+ let queries = 20usize;+ for qi in 0..queries {+ let q: Vec<f32> = (0..dim).map(|_| rng.next()).collect();+ let mut scored: Vec<(u64, f32)> = vecs+ .iter()+ .enumerate()+ .map(|(i, v)| (i as u64, cos(&q, v)))+ .collect();+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());+ let exact: std::collections::HashSet<u64> =+ scored[..k].iter().map(|&(i, _)| i).collect();+ let got = idx.search(&q, k, 400).unwrap();+ total_hits += got.iter().filter(|(id, _)| exact.contains(id)).count();+ let _ = qi;+ }+ let recall = total_hits as f64 / (queries * k) as f64;+ assert!(+ recall >= 0.9,+ "recall@10 regression: got {recall:.3}, expected >= 0.9"+ );+ }}