AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
merge main (979 hydrate-winners) into fix/issue-940-rrf-stable-ties
xerj-org23f467507a41
diff --cc engine/crates/xerj-engine/src/index.rsindex 7cc013eb9,ef4e9b9fc..9dc57aed0--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@@ -53439,175 -53821,828 +53889,1003 @@@ mod pinned_knn_fts_892_tests}}+++/// #940 — the fused order is total, and a function of the leg lists and the+/// documents' arrival order alone.+///+/// These drive `fuse_rrf`/`fuse_linear` directly, so they exercise the+/// accumulator on every call — the per-index result cache that made repeated+/// in-process requests look stable in the issue's own runs is not in the way.+#[cfg(test)]+mod hybrid_fused_order_tests {+ use super::*;++ fn hit(id: &str, score: f32) -> Hit {+ Hit {+ id: id.to_string(),+ score,+ source: serde_json::json!({ "id": id }),+ seq_no: None,+ version: None,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ passage: None,+ }+ }++ fn ids(hits: &[Hit]) -> Vec<&str> {+ hits.iter().map(|h| h.id.as_str()).collect()+ }++ /// Arrival order used by every test here: the trailing number of the id,+ /// scrambled against `_id` order on purpose (`doc-10` arrives before+ /// `doc-9` but sorts after it as a string) so a test cannot pass by+ /// tying on `_id` alone.+ fn arrival(id: &str) -> u64 {+ id.rsplit('-').next().unwrap().parse().unwrap()+ }++ /// Two disjoint legs of equal weight: the document at rank r of leg A and+ /// the one at rank r of leg B both score exactly `1/(60+r)`. Twelve+ /// structural ties — under the old `HashMap` drain the chance that one+ /// call returns them all in arrival order is 2^-12, and the chance that+ /// 64 calls agree with each other is nil.+ fn disjoint_legs() -> Vec<(Vec<Hit>, f32)> {+ // Leg A holds the ODD arrivals, leg B the EVEN ones, each already in+ // rank order. Scores inside a leg are distinct, as a real leg's are.+ let a: Vec<Hit> = (0..12)+ .map(|r| hit(&format!("doc-{}", 2 * r + 1), 9.0 - r as f32 * 0.5))+ .collect();+ let b: Vec<Hit> = (0..12)+ .map(|r| hit(&format!("doc-{}", 2 * r), 0.99 - r as f32 * 0.01))+ .collect();+ vec![(a, 1.0), (b, 1.0)]+ }++ #[test]+ fn rrf_structural_ties_resolve_by_arrival_order_on_every_call() {+ let legs = disjoint_legs();+ // Rank r → the even arrival (leg B) precedes the odd one (leg A):+ // doc-0, doc-1, doc-2, … — NOT the `_id` order, which would put+ // doc-10 ahead of doc-2.+ let expected: Vec<String> = (0..24).map(|n| format!("doc-{n}")).collect();+ for call in 0..64 {+ let fused = fuse_rrf(&legs, 60, &arrival);+ assert_eq!(+ ids(&fused),+ expected.iter().map(String::as_str).collect::<Vec<_>>(),+ "call {call}: tied fused scores must come back in arrival order"+ );+ // The ties are exact — otherwise this test proves nothing.+ for pair in fused.chunks(2) {+ assert_eq!(+ pair[0].score.to_bits(),+ pair[1].score.to_bits(),+ "fixture must produce EXACT ties, got {} vs {}",+ pair[0].score,+ pair[1].score+ );+ }+ }+ }++ /// The symmetric pair the issue names: two documents at swapped ranks+ /// (1,2)/(2,1) both score `1/61 + 1/62`.+ #[test]+ fn rrf_swapped_rank_pair_resolves_by_arrival_order() {+ let legs = vec![+ (vec![hit("doc-7", 3.0), hit("doc-4", 2.0)], 1.0),+ (vec![hit("doc-4", 0.9), hit("doc-7", 0.8)], 1.0),+ ];+ for _ in 0..64 {+ let fused = fuse_rrf(&legs, 60, &arrival);+ assert_eq!(fused[0].score.to_bits(), fused[1].score.to_bits());+ assert_eq!(ids(&fused), ["doc-4", "doc-7"]);+ }+ }++ /// RRF with equal weights is symmetric in its legs, so listing the legs+ /// in the other order must not change the page. This is the property a+ /// rank-derived tie-break ("the earlier-listed leg wins") cannot keep.+ #[test]+ fn rrf_order_does_not_depend_on_how_the_request_lists_its_legs() {+ let legs = disjoint_legs();+ let mut swapped = legs.clone();+ swapped.reverse();+ assert_eq!(+ ids(&fuse_rrf(&legs, 60, &arrival)),+ ids(&fuse_rrf(&swapped, 60, &arrival))+ );+ }++ /// Two documents sharing an arrival key (both unknown to the version map+ /// → `u64::MAX`) still order — by `_id`.+ #[test]+ fn rrf_falls_through_to_id_when_arrival_is_unknown() {+ let legs = vec![+ (vec![hit("zeta", 1.0)], 1.0),+ (vec![hit("alpha", 1.0)], 1.0),+ ];+ for _ in 0..64 {+ let fused = fuse_rrf(&legs, 60, &|_| u64::MAX);+ assert_eq!(ids(&fused), ["alpha", "zeta"]);+ }+ }++ /// `fuse_linear` had the same accumulator and the same score-only sort. A+ /// constant-score leg normalises to 0.0 for every document it returns, so+ /// the whole list ties.+ #[test]+ fn linear_constant_score_leg_resolves_by_arrival_order_on_every_call() {+ let constant: Vec<Hit> = [5, 11, 2, 9, 10, 3, 8, 1]+ .iter()+ .map(|n| hit(&format!("doc-{n}"), 1.0))+ .collect();+ let legs = vec![(constant, 1.0)];+ for call in 0..64 {+ let fused = fuse_linear(&legs, &arrival);+ assert!(fused.iter().all(|h| h.score == 0.0));+ assert_eq!(+ ids(&fused),+ ["doc-1", "doc-2", "doc-3", "doc-5", "doc-8", "doc-9", "doc-10", "doc-11"],+ "call {call}"+ );+ }+ }++ /// A NaN leg score must neither abort the sort nor float to the top of+ /// the page (`f32::total_cmp` alone would rank a positive NaN FIRST).+ #[test]+ fn linear_nan_score_sorts_last_and_does_not_panic() {+ let legs = vec![(+ vec![hit("doc-1", 4.0), hit("doc-2", f32::NAN), hit("doc-3", 1.0)],+ 1.0,+ )];+ let fused = fuse_linear(&legs, &arrival);+ assert_eq!(ids(&fused), ["doc-1", "doc-3", "doc-2"]);+ assert!(fused[2].score.is_nan());+ }++ /// Fusion still fuses: a document both legs rank highly beats one that+ /// only one leg found, and duplicates collapse to one hit.+ #[test]+ fn rrf_still_ranks_by_fused_score_first() {+ let legs = vec![+ (vec![hit("doc-9", 3.0), hit("doc-1", 2.0)], 1.0),+ (vec![hit("doc-9", 0.9), hit("doc-2", 0.8)], 1.0),+ ];+ let fused = fuse_rrf(&legs, 60, &arrival);+ assert_eq!(ids(&fused), ["doc-9", "doc-1", "doc-2"]);+ assert!(fused[0].score > fused[1].score);+ assert_eq!(fused[1].score.to_bits(), fused[2].score.to_bits());+ }+}+++++ /// #939 — the exact kNN scan ranks addresses and hydrates only the winners.+ ///+ /// The rewrite is meant to be invisible: same documents, same order, same+ /// scores to the bit. These tests pin that against `legacy_scan`, a transcript+ /// of the scan as it stood before the change (clone every live source, clone+ /// it again for the filter, re-walk the JSON numbers per passage), so the+ /// equivalence is checked in CI and not only by a one-off benchmark.+ #[cfg(test)]+ mod exact_scan_hydration_tests {+ use super::*;+ use crate::Engine;+ use tempfile::TempDir;++ const DIM: usize = 8;++ fn engine(dir: &TempDir) -> Engine {+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ Engine::new(config).expect("engine")+ }++ /// `compute_vector_similarity` as it stood before #939, verbatim. The+ /// oracle must not share code with what it checks.+ fn legacy_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {+ let dot: f64 = a+ .iter()+ .zip(b.iter())+ .map(|(x, y)| (*x as f64) * (*y as f64))+ .sum();+ let result: f64 = match sim {+ "l2_norm" => {+ let sq: f64 = a+ .iter()+ .zip(b.iter())+ .map(|(x, y)| {+ let d = (*x as f64) - (*y as f64);+ d * d+ })+ .sum();+ 1.0 / (1.0 + sq)+ }+ "dot_product" => (1.0 + dot) / 2.0,+ "max_inner_product" => {+ if dot < 0.0 {+ 1.0 / (1.0 - dot)+ } else {+ dot + 1.0+ }+ }+ _ => {+ let na: f64 = a+ .iter()+ .map(|x| (*x as f64) * (*x as f64))+ .sum::<f64>()+ .sqrt();+ let nb: f64 = b+ .iter()+ .map(|x| (*x as f64) * (*x as f64))+ .sum::<f64>()+ .sqrt();+ let denom = na * nb;+ let cos = if denom > 0.0 { dot / denom } else { 0.0 };+ (1.0 + cos) / 2.0+ }+ };+ result as f32+ }++ fn json_vector(value: &Value) -> Option<Vec<f32>> {+ match value {+ Value::Array(a) => Some(+ a.iter()+ .filter_map(|v| v.as_f64().map(|f| f as f32))+ .collect(),+ ),+ _ => None,+ }+ }++ /// The exact scan as it stood before #939. Returns the ranked, `k`-capped+ /// pool as `(id, score bits, passage ordinal, source)`.+ #[allow(clippy::too_many_arguments)]+ async fn legacy_scan(+ idx: &Index,+ field: &str,+ query_vec: &[f32],+ k: usize,+ filter: Option<&QueryNode>,+ similarity: &str,+ boost: Option<f32>,+ min_similarity: Option<f32>,+ ) -> Vec<(String, u32, Option<u32>, Value)> {+ let schema = idx.schema().await;+ let mut candidates: Vec<(String, Value)> = idx.memtable.all_docs_with_sources();+ let snap = idx.store.snapshot();+ let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();+ for meta in snap.segments.iter() {+ let Some(docs) = idx.stored_values_for(&meta.id) else {+ continue;+ };+ for doc in docs.iter() {+ let Some(id) = doc.get("_id").and_then(Value::as_str).map(str::to_string) else {+ continue;+ };+ if let Some(ver) = idx.store.version_map.get(&id) {+ if ver.deleted {+ continue;+ }+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {+ if doc_seq < ver.seq_no {+ continue;+ }+ }+ }+ if !seen.insert(id.clone()) {+ continue;+ }+ let src = doc.get("_source").cloned().unwrap_or_else(|| {+ let mut d = doc.clone();+ if let Some(obj) = d.as_object_mut() {+ obj.remove("_id");+ }+ d+ });+ candidates.push((id, src));+ }+ }+ let passes = |id: &str, src: &Value| -> bool {+ let Some(f) = filter else {+ return true;+ };+ let mut src_with_id = src.clone();+ if let Some(obj) = src_with_id.as_object_mut() {+ obj.insert("_id".to_string(), Value::String(id.to_string()));+ }+ doc_matches_query_typed(f, &src_with_id, &schema)+ };+ let use_sq8 = lookup_vector_quantization(&schema, field).as_deref() == Some("scalar8");+ let mut scored: Vec<(String, f32, Value, Option<u32>)> = Vec::new();+ if use_sq8 {+ let normalize = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");+ let dim = query_vec.len();+ let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::new();+ for (id, src) in candidates {+ if !passes(&id, &src) {+ continue;+ }+ let Some(mut doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)+ else {+ continue;+ };+ if doc_vec.len() != dim {+ continue;+ }+ if normalize {+ l2_normalize_vec(&mut doc_vec);+ }+ cand.push((id, src, doc_vec));+ }+ let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, _, v)| v.as_slice()), dim);+ let mut codes = vec![0u8; dim];+ let mut decoded = vec![0.0f32; dim];+ for (id, src, v) in cand {+ params.encode_into(&v, &mut codes);+ params.decode_into(&codes, &mut decoded);+ scored.push((+ id,+ legacy_similarity(similarity, query_vec, &decoded),+ src,+ None,+ ));+ }+ } else {+ let chunk_field = format!("{field}_chunks");+ for (id, src) in candidates {+ if !passes(&id, &src) {+ continue;+ }+ let (score, ordinal) = if let Some(Value::Array(chunks)) =+ get_field_value(&src, &chunk_field)+ {+ let mut best: Option<(f32, u32)> = None;+ for (chunk_position, cv) in chunks.iter().enumerate() {+ let Some(dv) = json_vector(cv) else {+ continue;+ };+ if dv.len() != query_vec.len() {+ continue;+ }+ let s = legacy_similarity(similarity, query_vec, &dv);+ if best.is_none_or(|(current, _)| s > current) {+ best = Some((s, chunk_position as u32));+ }+ }+ match best {+ Some((score, ordinal)) => (score, Some(ordinal)),+ None => continue,+ }+ } else {+ let Some(doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)+ else {+ continue;+ };+ if doc_vec.len() != query_vec.len() {+ continue;+ }+ (legacy_similarity(similarity, query_vec, &doc_vec), Some(0))+ };+ scored.push((id, score, src, ordinal));+ }+ }+ if let Some(raw) = min_similarity {+ let cut = raw_similarity_to_score(similarity, raw);+ scored.retain(|(_, score, _, _)| *score >= cut);+ }+ if let Some(b) = boost {+ if (b - 1.0).abs() > f32::EPSILON {+ for (_, score, _, _) in scored.iter_mut() {+ *score *= b;+ }+ }+ }+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ scored.truncate(k.max(1));+ scored+ .into_iter()+ .map(|(id, score, src, ordinal)| (id, score.to_bits(), ordinal, src))+ .collect()+ }++ /// Deterministic pseudo-random unit-ish vectors; no `rand` dependency.+ struct Lcg(u64);+ impl Lcg {+ fn next_f32(&mut self) -> f32 {+ self.0 = self+ .0+ .wrapping_mul(6364136223846793005)+ .wrapping_add(1442695040888963407);+ (((self.0 >> 33) as f64) / ((1u64 << 31) as f64) - 1.0) as f32+ }+ fn vector(&mut self) -> Vec<f32> {+ (0..DIM).map(|_| self.next_f32()).collect()+ }+ }++ fn as_json(vector: &[f32]) -> Value {+ Value::Array(vector.iter().map(|v| Value::from(*v as f64)).collect())+ }++ fn vector_schema(quantization: Option<&str>) -> Schema {+ let mut schema = Schema::empty();+ let mut body = FieldConfig::new("body", FieldType::Text);+ body.options.dimensions = Some(DIM);+ body.options.similarity = Some("cosine".into());+ body.embedding = Some(xerj_common::types::EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("body_vector".into()),+ });+ schema.add_field(body).unwrap();+ let mut companion = FieldConfig::new("body_vector", FieldType::Vector);+ companion.options.dimensions = Some(DIM);+ companion.options.similarity = Some("cosine".into());+ companion.options.quantization = quantization.map(str::to_string);+ schema.add_field(companion).unwrap();+ schema+ .add_field(FieldConfig::new("tag", FieldType::Keyword))+ .unwrap();+ schema+ .add_field(FieldConfig::new("n", FieldType::Long))+ .unwrap();+ schema+ .add_field(FieldConfig::new("title", FieldType::Text))+ .unwrap();+ schema+ }++ /// A document. Every third is multi-passage; a run of ids shares one+ /// vector so the rank has exact score ties to break.+ fn document(id: usize, rng: &mut Lcg, shared: &[f32]) -> Value {+ let pooled = if (10..16).contains(&id) {+ shared.to_vec()+ } else {+ rng.vector()+ };+ let mut doc = serde_json::json!({+ "title": format!("doc {id} {}", if id.is_multiple_of(4) { "cell" } else { "other" }),+ "tag": format!("t{}", id % 5),+ "n": (id * 37) % 1000,+ "body": format!("body text {id}"),+ "body_vector": as_json(&pooled),+ });+ if id.is_multiple_of(3) {+ let passages: Vec<Value> = (0..2 + id % 4).map(|_| as_json(&rng.vector())).collect();+ doc["body_vector_chunks"] = Value::Array(passages);+ }+ doc+ }++ /// Two segments (the second superseding and deleting part of the first)+ /// plus a live memtable, with every awkward document shape the scan has a+ /// rule for.+ async fn seed(idx: &Arc<Index>) {+ let mut rng = Lcg(0x939);+ let shared = rng.vector();+ for id in 0..150 {+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))+ .await+ .unwrap();+ }+ // Shapes with a rule of their own.+ idx.index_document_prepared(+ Some("malformed-passages".into()),+ serde_json::json!({+ "title": "cell malformed", "tag": "t1", "n": 5, "body": "x",+ "body_vector": as_json(&rng.vector()),+ // ordinal 1 is not a vector, ordinal 3 has the wrong length:+ // both skipped, neither renumbers ordinal 2.+ "body_vector_chunks": [as_json(&rng.vector()), "junk", as_json(&rng.vector()), [1.0, 2.0]],+ }),+ )+ .await+ .unwrap();+ idx.index_document_prepared(+ Some("empty-passages".into()),+ serde_json::json!({+ "title": "cell empty", "tag": "t1", "n": 6, "body": "x",+ "body_vector": as_json(&shared),+ "body_vector_chunks": [],+ }),+ )+ .await+ .unwrap();+ idx.index_document_prepared(+ Some("no-vector".into()),+ serde_json::json!({"title": "cell none", "tag": "t1", "n": 7, "body": "x"}),+ )+ .await+ .unwrap();+ idx.flush().await.unwrap();++ // Second segment: supersede some of the first, delete some, add more.+ for id in (0..150).step_by(9) {+ idx.index_document_prepared(+ Some(format!("d{id}")),+ document(id + 1000, &mut rng, &shared),+ )+ .await+ .unwrap();+ }+ for id in (5..150).step_by(17) {+ idx.delete_document(&format!("d{id}")).await.unwrap();+ }+ for id in 150..260 {+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();++ // Live memtable over both: updates of flushed documents and new ones.+ for id in (3..260).step_by(31) {+ idx.index_document_prepared(+ Some(format!("d{id}")),+ document(id + 2000, &mut rng, &shared),+ )+ .await+ .unwrap();+ }+ for id in 260..290 {+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))+ .await+ .unwrap();+ }+ assert!(+ idx.store.snapshot().segments.len() >= 2,+ "the fixture needs flushed segments beside the memtable"+ );+ }++ fn filters() -> Vec<(&'static str, Option<QueryNode>)> {+ let term = QueryNode::Term {+ field: "tag".into(),+ value: Value::from("t3"),+ boost: None,+ };+ let range = QueryNode::Range {+ field: "n".into(),+ gte: Some(Value::from(100)),+ gt: None,+ lte: None,+ lt: Some(Value::from(800)),+ boost: None,+ };+ vec![+ ("none", None),+ (+ "ids",+ Some(QueryNode::Ids {+ values: (0..290).step_by(7).map(|id| format!("d{id}")).collect(),+ }),+ ),+ ("term", Some(term.clone())),+ (+ "bool(term AND range, NOT exists)",+ Some(QueryNode::Bool {+ must: Vec::new(),+ should: Vec::new(),+ filter: vec![term.clone(), range.clone()],+ must_not: vec![QueryNode::Exists {+ field: "body_vector_chunks".into(),+ }],+ minimum_should_match: None,+ }),+ ),+ (+ // Reads the VECTOR fields themselves: the projection must+ // carry them rather than assume a filter never looks.+ "exists(vector)",+ Some(QueryNode::Exists {+ field: "body_vector".into(),+ }),+ ),+ (+ // Not on the projection whitelist: takes the full-source path.+ "match (full source)",+ Some(QueryNode::Match {+ field: "title".into(),+ query: "cell".into(),+ operator: xerj_query::ast::BoolOperator::Or,+ analyzer: None,+ boost: None,+ minimum_should_match: None,+ }),+ ),+ ]+ }++ async fn assert_scan_matches_legacy(idx: &Arc<Index>, field: &str, sq8: bool) {+ let mut rng = Lcg(0xBEEF);+ let mut compared = 0usize;+ let mut saw_later_passage = false;+ let similarities: &[&str] = if sq8 {+ &["cosine", "l2_norm"]+ } else {+ &["cosine", "dot_product", "l2_norm", "max_inner_product"]+ };+ for _ in 0..4 {+ let query = rng.vector();+ for similarity in similarities {+ for (filter_name, filter) in filters() {+ for (k, boost, min_similarity) in [+ (1usize, None, None),+ (10, Some(2.5f32), None),+ (25, None, Some(0.1f32)),+ (1000, None, None),+ ] {+ let expected = legacy_scan(+ idx,+ field,+ &query,+ k,+ filter.as_ref(),+ similarity,+ boost,+ min_similarity,+ )+ .await;+ let request = SearchRequest {+ size: k,+ fields: vec![PASSAGE_RESPONSE_FIELD.into()],+ ..SearchRequest::default()+ };+ let got = idx+ .run_knn_brute_force(+ &request,+ field,+ &query,+ k,+ filter.clone().map(Box::new),+ similarity,+ boost,+ min_similarity,+ )+ .await+ .unwrap();+ let context = format!("similarity={similarity} filter={filter_name} k={k}");+ assert_eq!(got.total.value, expected.len() as u64, "total: {context}");+ assert_eq!(+ got.hits+ .iter()+ .map(|h| (h.id.clone(), h.score.to_bits()))+ .collect::<Vec<_>>(),+ expected+ .iter()+ .map(|(id, bits, _, _)| (id.clone(), *bits))+ .collect::<Vec<_>>(),+ "ids, order and score BITS: {context}"+ );+ for (hit, (_, _, ordinal, source)) in got.hits.iter().zip(&expected) {+ // Hydrated from the right document: the user+ // fields are the legacy scan's, value for value.+ for key in ["title", "tag", "n", "body"] {+ assert_eq!(+ hit.source.get(key),+ source.get(key),+ "{key}: {context}"+ );+ }+ if let Some(ordinal) = ordinal {+ saw_later_passage |= *ordinal > 0;+ }+ }+ compared += got.hits.len();+ }+ }+ }+ }+ assert!(+ compared > 1000,+ "the comparison must have teeth: {compared}"+ );+ if !sq8 {+ assert!(+ saw_later_passage,+ "no query was won by a passage past the first"+ );+ }+ }++ #[tokio::test]+ async fn exact_scan_is_bit_identical_to_the_clone_everything_scan() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("scan-939", vector_schema(None))+ .unwrap();+ let idx = engine.get_index("scan-939").unwrap();+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;+ seed(&idx).await;+ // Cold (columns derived in the scan) and warm (columns cached) must+ // agree with the oracle and therefore with each other.+ assert_scan_matches_legacy(&idx, "body_vector", false).await;+ assert!(+ !idx.vector_column_cache.is_empty(),+ "columns were never published"+ );+ assert_scan_matches_legacy(&idx, "body_vector", false).await;+ }++ #[tokio::test]+ async fn scalar8_scan_is_bit_identical_to_the_clone_everything_scan() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("scan-939-sq8", vector_schema(Some("scalar8")))+ .unwrap();+ let idx = engine.get_index("scan-939-sq8").unwrap();+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;+ seed(&idx).await;+ assert_scan_matches_legacy(&idx, "body_vector", true).await;+ assert_scan_matches_legacy(&idx, "body_vector", true).await;+ }++ #[tokio::test]+ async fn vector_columns_are_per_segment_budgeted_and_retired_by_merge() {+ let dir = TempDir::new().unwrap();+ let engine = engine(&dir);+ engine+ .create_index("scan-939-cache", vector_schema(None))+ .unwrap();+ let idx = engine.get_index("scan-939-cache").unwrap();+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;+ seed(&idx).await;+ let query = Lcg(7).vector();+ let request = SearchRequest {+ size: 10,+ ..SearchRequest::default()+ };+ let scan = || {+ idx.run_knn_brute_force(+ &request,+ "body_vector",+ &query,+ 10,+ None,+ "cosine",+ None,+ None,+ )+ };++ let before = scan().await.unwrap();+ let segments: Vec<String> = idx+ .store+ .snapshot()+ .segments+ .iter()+ .map(|m| m.id.clone())+ .collect();+ // One column per segment that HAS stored documents. A segment holding+ // only tombstones has no stored section; the scan skips it, so there+ // is nothing to derive a column from.+ let mut with_stored = 0usize;+ for id in &segments {+ let has_stored = idx.stored_values_for(id).is_some();+ with_stored += usize::from(has_stored);+ assert_eq!(+ idx.vector_column_cache+ .contains_key(&format!("{id}\u{1}body_vector\u{1}c")),+ has_stored,+ "segment {id}: a column exists exactly when stored documents do"+ );+ }+ assert!(with_stored >= 2, "the fixture needs several real segments");+ assert_eq!(idx.vector_column_cache.len(), with_stored);+ let charged = idx.segment_hydration_budget.snapshot().category_current+ [SegmentCacheCategory::VectorColumn as usize];+ assert!(+ charged > 0,+ "a published column must be charged to the hydration budget"+ );++ // A merge retires the segments; their columns must go with them, and+ // the answer must not change.+ idx.flush().await.unwrap();+ idx.force_merge(1).await.unwrap();+ let live: HashSet<String> = idx+ .store+ .snapshot()+ .segments+ .iter()+ .map(|m| m.id.clone())+ .collect();+ for entry in idx.vector_column_cache.iter() {+ let segment = entry.key().split('\u{1}').next().unwrap();+ assert!(+ live.contains(segment),+ "column outlived its segment: {}",+ entry.key()+ );+ }+ let after = scan().await.unwrap();+ let rows = |r: &SearchResult| {+ r.hits+ .iter()+ .map(|h| (h.id.clone(), h.score.to_bits()))+ .collect::<Vec<_>>()+ };+ assert_eq!(+ rows(&before),+ rows(&after),+ "a merge changed an exact answer"+ );++ idx.release_memory();+ assert!(idx.vector_column_cache.is_empty());+ assert_eq!(rows(&before), rows(&scan().await.unwrap()));+ }++ #[test]+ fn query_scorer_is_bit_identical_to_compute_vector_similarity() {+ let mut rng = Lcg(42);+ let zero = vec![0.0f32; DIM];+ for similarity in [+ "cosine",+ "dot_product",+ "l2_norm",+ "max_inner_product",+ "anything-else",+ ] {+ for round in 0..500 {+ let query = if round == 0 {+ zero.clone()+ } else {+ rng.vector()+ };+ let stored = if round == 1 {+ zero.clone()+ } else {+ rng.vector()+ };+ let scorer = QueryScorer::new(similarity, &query);+ let got = scorer.score(crate::vector_column::StoredVector {+ values: &stored,+ ordinal: 0,+ norm: crate::vector_column::norm_f64(&stored),+ });+ let legacy = legacy_similarity(similarity, &query, &stored);+ assert_eq!(+ got.to_bits(),+ legacy.to_bits(),+ "{similarity} round {round}"+ );+ assert_eq!(+ compute_vector_similarity(similarity, &query, &stored).to_bits(),+ legacy.to_bits(),+ "refactored compute_vector_similarity drifted: {similarity} round {round}"+ );+ }+ }+ }++ #[test]+ fn knn_filter_projection_agrees_with_the_full_source() {+ let schema = vector_schema(None);+ let docs = [+ serde_json::json!({"tag": "t3", "n": 150, "a": {"b": {"c": 1}}, "a.b": {"c": 2}, "body_vector": [1.0], "_id": "shadowed"}),+ serde_json::json!({"tag": ["t1", "t3"], "n": 900, "a.b.c": 3, "_routing": "r"}),+ serde_json::json!({"tag": null, "n": "150", "a": [{"b": {"c": 4}}, {"b": {"c": 5}}]}),+ serde_json::json!({"title": "cell", "count": 2}),+ serde_json::json!({}),+ Value::Null,+ ];+ let term = |field: &str, value: Value| QueryNode::Term {+ field: field.into(),+ value,+ boost: None,+ };+ let exists = |field: &str| QueryNode::Exists {+ field: field.into(),+ };+ let projected = vec![+ QueryNode::MatchAll,+ QueryNode::MatchNone,+ QueryNode::Ids {+ values: vec!["id-0".into(), "id-2".into()],+ },+ term("tag", Value::from("t3")),+ term("_id", Value::from("id-1")),+ term("a.b.c", Value::from(1)),+ term("a.b.c", Value::from(3)),+ term("a.b.c", Value::from(4)),+ term("tag.keyword", Value::from("t3")),+ exists("a.b"),+ exists("a.b.c"),+ exists("_routing"),+ exists("_id"),+ exists("body_vector"),+ exists("missing"),+ QueryNode::Terms {+ field: "tag".into(),+ values: vec![Value::from("t1"), Value::from("zz")],+ boost: None,+ },+ QueryNode::Range {+ field: "n".into(),+ gte: Some(Value::from(100)),+ gt: None,+ lte: None,+ lt: Some(Value::from(800)),+ boost: None,+ },+ QueryNode::Prefix {+ field: "tag".into(),+ value: "t".into(),+ boost: None,+ constant_score: false,+ },+ QueryNode::Bool {+ must: vec![exists("tag")],+ should: vec![term("tag", Value::from("t3")), term("n", Value::from(900))],+ filter: Vec::new(),+ must_not: vec![term("_id", Value::from("id-0"))],+ minimum_should_match: Some(MinShouldMatch::Field("count".into())),+ },+ QueryNode::Constant {+ score: 1.0,+ query: Box::new(term("tag", Value::from("t3"))),+ },+ ];+ for filter in &projected {+ let plan = KnnFilterPlan::for_filter(filter);+ assert!(+ matches!(plan, KnnFilterPlan::Projected(_)),+ "should project: {filter:?}"+ );+ for (n, doc) in docs.iter().enumerate() {+ let id = format!("id-{n}");+ assert_eq!(+ plan.matches(filter, &id, doc, &schema),+ KnnFilterPlan::FullSource.matches(filter, &id, doc, &schema),+ "projection changed the answer: {filter:?} on {doc}"+ );+ }+ }+ // Declined: these read the document some other way.+ let declined = vec![+ term("ta*", Value::from("t3")),+ QueryNode::Script {+ source: "true".into(),+ params: None,+ },+ QueryNode::Bool {+ must: Vec::new(),+ should: vec![term("tag", Value::from("t3"))],+ filter: Vec::new(),+ must_not: Vec::new(),+ minimum_should_match: Some(MinShouldMatch::Script {+ source: "1".into(),+ params: None,+ }),+ },+ ];+ for filter in &declined {+ assert!(+ matches!(KnnFilterPlan::for_filter(filter), KnnFilterPlan::FullSource),+ "must not project: {filter:?}"+ );+ }+ }++ /// The full-source plan IS the old behaviour: the whole source with `_id`+ /// inserted over whatever the document carried under that key.+ #[test]+ fn full_source_plan_inserts_the_id_like_the_old_scan_did() {+ let schema = vector_schema(None);+ let filter = QueryNode::Ids {+ values: vec!["real".into()],+ };+ let doc = serde_json::json!({"_id": "shadowed", "tag": "t"});+ assert!(KnnFilterPlan::FullSource.matches(&filter, "real", &doc, &schema));+ assert!(!KnnFilterPlan::FullSource.matches(&filter, "other", &doc, &schema));+ }+ }