Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

merge main (979 hydrate-winners) into fix/issue-940-rrf-stable-ties

xerj-org23f467507a41
diff --cc engine/crates/xerj-engine/src/index.rs
index 7cc013eb9,ef4e9b9fc..9dc57aed0
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@@ -53439,175 -53821,828 +53889,1003 @@@ mod pinned_knn_fts_892_tests
}
}
++
+/// #940 — the fused order is total, and a function of the leg lists and the
+/// documents' arrival order alone.
+///
+/// These drive `fuse_rrf`/`fuse_linear` directly, so they exercise the
+/// accumulator on every call — the per-index result cache that made repeated
+/// in-process requests look stable in the issue's own runs is not in the way.
+#[cfg(test)]
+mod hybrid_fused_order_tests {
+ use super::*;
+
+ fn hit(id: &str, score: f32) -> Hit {
+ Hit {
+ id: id.to_string(),
+ score,
+ source: serde_json::json!({ "id": id }),
+ seq_no: None,
+ version: None,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ }
+ }
+
+ fn ids(hits: &[Hit]) -> Vec<&str> {
+ hits.iter().map(|h| h.id.as_str()).collect()
+ }
+
+ /// Arrival order used by every test here: the trailing number of the id,
+ /// scrambled against `_id` order on purpose (`doc-10` arrives before
+ /// `doc-9` but sorts after it as a string) so a test cannot pass by
+ /// tying on `_id` alone.
+ fn arrival(id: &str) -> u64 {
+ id.rsplit('-').next().unwrap().parse().unwrap()
+ }
+
+ /// Two disjoint legs of equal weight: the document at rank r of leg A and
+ /// the one at rank r of leg B both score exactly `1/(60+r)`. Twelve
+ /// structural ties — under the old `HashMap` drain the chance that one
+ /// call returns them all in arrival order is 2^-12, and the chance that
+ /// 64 calls agree with each other is nil.
+ fn disjoint_legs() -> Vec<(Vec<Hit>, f32)> {
+ // Leg A holds the ODD arrivals, leg B the EVEN ones, each already in
+ // rank order. Scores inside a leg are distinct, as a real leg's are.
+ let a: Vec<Hit> = (0..12)
+ .map(|r| hit(&format!("doc-{}", 2 * r + 1), 9.0 - r as f32 * 0.5))
+ .collect();
+ let b: Vec<Hit> = (0..12)
+ .map(|r| hit(&format!("doc-{}", 2 * r), 0.99 - r as f32 * 0.01))
+ .collect();
+ vec![(a, 1.0), (b, 1.0)]
+ }
+
+ #[test]
+ fn rrf_structural_ties_resolve_by_arrival_order_on_every_call() {
+ let legs = disjoint_legs();
+ // Rank r → the even arrival (leg B) precedes the odd one (leg A):
+ // doc-0, doc-1, doc-2, … — NOT the `_id` order, which would put
+ // doc-10 ahead of doc-2.
+ let expected: Vec<String> = (0..24).map(|n| format!("doc-{n}")).collect();
+ for call in 0..64 {
+ let fused = fuse_rrf(&legs, 60, &arrival);
+ assert_eq!(
+ ids(&fused),
+ expected.iter().map(String::as_str).collect::<Vec<_>>(),
+ "call {call}: tied fused scores must come back in arrival order"
+ );
+ // The ties are exact — otherwise this test proves nothing.
+ for pair in fused.chunks(2) {
+ assert_eq!(
+ pair[0].score.to_bits(),
+ pair[1].score.to_bits(),
+ "fixture must produce EXACT ties, got {} vs {}",
+ pair[0].score,
+ pair[1].score
+ );
+ }
+ }
+ }
+
+ /// The symmetric pair the issue names: two documents at swapped ranks
+ /// (1,2)/(2,1) both score `1/61 + 1/62`.
+ #[test]
+ fn rrf_swapped_rank_pair_resolves_by_arrival_order() {
+ let legs = vec![
+ (vec![hit("doc-7", 3.0), hit("doc-4", 2.0)], 1.0),
+ (vec![hit("doc-4", 0.9), hit("doc-7", 0.8)], 1.0),
+ ];
+ for _ in 0..64 {
+ let fused = fuse_rrf(&legs, 60, &arrival);
+ assert_eq!(fused[0].score.to_bits(), fused[1].score.to_bits());
+ assert_eq!(ids(&fused), ["doc-4", "doc-7"]);
+ }
+ }
+
+ /// RRF with equal weights is symmetric in its legs, so listing the legs
+ /// in the other order must not change the page. This is the property a
+ /// rank-derived tie-break ("the earlier-listed leg wins") cannot keep.
+ #[test]
+ fn rrf_order_does_not_depend_on_how_the_request_lists_its_legs() {
+ let legs = disjoint_legs();
+ let mut swapped = legs.clone();
+ swapped.reverse();
+ assert_eq!(
+ ids(&fuse_rrf(&legs, 60, &arrival)),
+ ids(&fuse_rrf(&swapped, 60, &arrival))
+ );
+ }
+
+ /// Two documents sharing an arrival key (both unknown to the version map
+ /// → `u64::MAX`) still order — by `_id`.
+ #[test]
+ fn rrf_falls_through_to_id_when_arrival_is_unknown() {
+ let legs = vec![
+ (vec![hit("zeta", 1.0)], 1.0),
+ (vec![hit("alpha", 1.0)], 1.0),
+ ];
+ for _ in 0..64 {
+ let fused = fuse_rrf(&legs, 60, &|_| u64::MAX);
+ assert_eq!(ids(&fused), ["alpha", "zeta"]);
+ }
+ }
+
+ /// `fuse_linear` had the same accumulator and the same score-only sort. A
+ /// constant-score leg normalises to 0.0 for every document it returns, so
+ /// the whole list ties.
+ #[test]
+ fn linear_constant_score_leg_resolves_by_arrival_order_on_every_call() {
+ let constant: Vec<Hit> = [5, 11, 2, 9, 10, 3, 8, 1]
+ .iter()
+ .map(|n| hit(&format!("doc-{n}"), 1.0))
+ .collect();
+ let legs = vec![(constant, 1.0)];
+ for call in 0..64 {
+ let fused = fuse_linear(&legs, &arrival);
+ assert!(fused.iter().all(|h| h.score == 0.0));
+ assert_eq!(
+ ids(&fused),
+ ["doc-1", "doc-2", "doc-3", "doc-5", "doc-8", "doc-9", "doc-10", "doc-11"],
+ "call {call}"
+ );
+ }
+ }
+
+ /// A NaN leg score must neither abort the sort nor float to the top of
+ /// the page (`f32::total_cmp` alone would rank a positive NaN FIRST).
+ #[test]
+ fn linear_nan_score_sorts_last_and_does_not_panic() {
+ let legs = vec![(
+ vec![hit("doc-1", 4.0), hit("doc-2", f32::NAN), hit("doc-3", 1.0)],
+ 1.0,
+ )];
+ let fused = fuse_linear(&legs, &arrival);
+ assert_eq!(ids(&fused), ["doc-1", "doc-3", "doc-2"]);
+ assert!(fused[2].score.is_nan());
+ }
+
+ /// Fusion still fuses: a document both legs rank highly beats one that
+ /// only one leg found, and duplicates collapse to one hit.
+ #[test]
+ fn rrf_still_ranks_by_fused_score_first() {
+ let legs = vec![
+ (vec![hit("doc-9", 3.0), hit("doc-1", 2.0)], 1.0),
+ (vec![hit("doc-9", 0.9), hit("doc-2", 0.8)], 1.0),
+ ];
+ let fused = fuse_rrf(&legs, 60, &arrival);
+ assert_eq!(ids(&fused), ["doc-9", "doc-1", "doc-2"]);
+ assert!(fused[0].score > fused[1].score);
+ assert_eq!(fused[1].score.to_bits(), fused[2].score.to_bits());
+ }
+}
++
++
+ /// #939 — the exact kNN scan ranks addresses and hydrates only the winners.
+ ///
+ /// The rewrite is meant to be invisible: same documents, same order, same
+ /// scores to the bit. These tests pin that against `legacy_scan`, a transcript
+ /// of the scan as it stood before the change (clone every live source, clone
+ /// it again for the filter, re-walk the JSON numbers per passage), so the
+ /// equivalence is checked in CI and not only by a one-off benchmark.
+ #[cfg(test)]
+ mod exact_scan_hydration_tests {
+ use super::*;
+ use crate::Engine;
+ use tempfile::TempDir;
+
+ const DIM: usize = 8;
+
+ fn engine(dir: &TempDir) -> Engine {
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ Engine::new(config).expect("engine")
+ }
+
+ /// `compute_vector_similarity` as it stood before #939, verbatim. The
+ /// oracle must not share code with what it checks.
+ fn legacy_similarity(sim: &str, a: &[f32], b: &[f32]) -> f32 {
+ let dot: f64 = a
+ .iter()
+ .zip(b.iter())
+ .map(|(x, y)| (*x as f64) * (*y as f64))
+ .sum();
+ let result: f64 = match sim {
+ "l2_norm" => {
+ let sq: f64 = a
+ .iter()
+ .zip(b.iter())
+ .map(|(x, y)| {
+ let d = (*x as f64) - (*y as f64);
+ d * d
+ })
+ .sum();
+ 1.0 / (1.0 + sq)
+ }
+ "dot_product" => (1.0 + dot) / 2.0,
+ "max_inner_product" => {
+ if dot < 0.0 {
+ 1.0 / (1.0 - dot)
+ } else {
+ dot + 1.0
+ }
+ }
+ _ => {
+ let na: f64 = a
+ .iter()
+ .map(|x| (*x as f64) * (*x as f64))
+ .sum::<f64>()
+ .sqrt();
+ let nb: f64 = b
+ .iter()
+ .map(|x| (*x as f64) * (*x as f64))
+ .sum::<f64>()
+ .sqrt();
+ let denom = na * nb;
+ let cos = if denom > 0.0 { dot / denom } else { 0.0 };
+ (1.0 + cos) / 2.0
+ }
+ };
+ result as f32
+ }
+
+ fn json_vector(value: &Value) -> Option<Vec<f32>> {
+ match value {
+ Value::Array(a) => Some(
+ a.iter()
+ .filter_map(|v| v.as_f64().map(|f| f as f32))
+ .collect(),
+ ),
+ _ => None,
+ }
+ }
+
+ /// The exact scan as it stood before #939. Returns the ranked, `k`-capped
+ /// pool as `(id, score bits, passage ordinal, source)`.
+ #[allow(clippy::too_many_arguments)]
+ async fn legacy_scan(
+ idx: &Index,
+ field: &str,
+ query_vec: &[f32],
+ k: usize,
+ filter: Option<&QueryNode>,
+ similarity: &str,
+ boost: Option<f32>,
+ min_similarity: Option<f32>,
+ ) -> Vec<(String, u32, Option<u32>, Value)> {
+ let schema = idx.schema().await;
+ let mut candidates: Vec<(String, Value)> = idx.memtable.all_docs_with_sources();
+ let snap = idx.store.snapshot();
+ let mut seen: HashSet<String> = candidates.iter().map(|(id, _)| id.clone()).collect();
+ for meta in snap.segments.iter() {
+ let Some(docs) = idx.stored_values_for(&meta.id) else {
+ continue;
+ };
+ for doc in docs.iter() {
+ let Some(id) = doc.get("_id").and_then(Value::as_str).map(str::to_string) else {
+ continue;
+ };
+ if let Some(ver) = idx.store.version_map.get(&id) {
+ if ver.deleted {
+ continue;
+ }
+ if let Some(doc_seq) = doc.get("_seq_no").and_then(Value::as_u64) {
+ if doc_seq < ver.seq_no {
+ continue;
+ }
+ }
+ }
+ if !seen.insert(id.clone()) {
+ continue;
+ }
+ let src = doc.get("_source").cloned().unwrap_or_else(|| {
+ let mut d = doc.clone();
+ if let Some(obj) = d.as_object_mut() {
+ obj.remove("_id");
+ }
+ d
+ });
+ candidates.push((id, src));
+ }
+ }
+ let passes = |id: &str, src: &Value| -> bool {
+ let Some(f) = filter else {
+ return true;
+ };
+ let mut src_with_id = src.clone();
+ if let Some(obj) = src_with_id.as_object_mut() {
+ obj.insert("_id".to_string(), Value::String(id.to_string()));
+ }
+ doc_matches_query_typed(f, &src_with_id, &schema)
+ };
+ let use_sq8 = lookup_vector_quantization(&schema, field).as_deref() == Some("scalar8");
+ let mut scored: Vec<(String, f32, Value, Option<u32>)> = Vec::new();
+ if use_sq8 {
+ let normalize = !matches!(similarity, "l2_norm" | "dot_product" | "max_inner_product");
+ let dim = query_vec.len();
+ let mut cand: Vec<(String, Value, Vec<f32>)> = Vec::new();
+ for (id, src) in candidates {
+ if !passes(&id, &src) {
+ continue;
+ }
+ let Some(mut doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)
+ else {
+ continue;
+ };
+ if doc_vec.len() != dim {
+ continue;
+ }
+ if normalize {
+ l2_normalize_vec(&mut doc_vec);
+ }
+ cand.push((id, src, doc_vec));
+ }
+ let params = Sq8Params::fit_borrowed(cand.iter().map(|(_, _, v)| v.as_slice()), dim);
+ let mut codes = vec![0u8; dim];
+ let mut decoded = vec![0.0f32; dim];
+ for (id, src, v) in cand {
+ params.encode_into(&v, &mut codes);
+ params.decode_into(&codes, &mut decoded);
+ scored.push((
+ id,
+ legacy_similarity(similarity, query_vec, &decoded),
+ src,
+ None,
+ ));
+ }
+ } else {
+ let chunk_field = format!("{field}_chunks");
+ for (id, src) in candidates {
+ if !passes(&id, &src) {
+ continue;
+ }
+ let (score, ordinal) = if let Some(Value::Array(chunks)) =
+ get_field_value(&src, &chunk_field)
+ {
+ let mut best: Option<(f32, u32)> = None;
+ for (chunk_position, cv) in chunks.iter().enumerate() {
+ let Some(dv) = json_vector(cv) else {
+ continue;
+ };
+ if dv.len() != query_vec.len() {
+ continue;
+ }
+ let s = legacy_similarity(similarity, query_vec, &dv);
+ if best.is_none_or(|(current, _)| s > current) {
+ best = Some((s, chunk_position as u32));
+ }
+ }
+ match best {
+ Some((score, ordinal)) => (score, Some(ordinal)),
+ None => continue,
+ }
+ } else {
+ let Some(doc_vec) = get_field_value(&src, field).as_ref().and_then(json_vector)
+ else {
+ continue;
+ };
+ if doc_vec.len() != query_vec.len() {
+ continue;
+ }
+ (legacy_similarity(similarity, query_vec, &doc_vec), Some(0))
+ };
+ scored.push((id, score, src, ordinal));
+ }
+ }
+ if let Some(raw) = min_similarity {
+ let cut = raw_similarity_to_score(similarity, raw);
+ scored.retain(|(_, score, _, _)| *score >= cut);
+ }
+ if let Some(b) = boost {
+ if (b - 1.0).abs() > f32::EPSILON {
+ for (_, score, _, _) in scored.iter_mut() {
+ *score *= b;
+ }
+ }
+ }
+ scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+ scored.truncate(k.max(1));
+ scored
+ .into_iter()
+ .map(|(id, score, src, ordinal)| (id, score.to_bits(), ordinal, src))
+ .collect()
+ }
+
+ /// Deterministic pseudo-random unit-ish vectors; no `rand` dependency.
+ struct Lcg(u64);
+ impl Lcg {
+ fn next_f32(&mut self) -> f32 {
+ self.0 = self
+ .0
+ .wrapping_mul(6364136223846793005)
+ .wrapping_add(1442695040888963407);
+ (((self.0 >> 33) as f64) / ((1u64 << 31) as f64) - 1.0) as f32
+ }
+ fn vector(&mut self) -> Vec<f32> {
+ (0..DIM).map(|_| self.next_f32()).collect()
+ }
+ }
+
+ fn as_json(vector: &[f32]) -> Value {
+ Value::Array(vector.iter().map(|v| Value::from(*v as f64)).collect())
+ }
+
+ fn vector_schema(quantization: Option<&str>) -> Schema {
+ let mut schema = Schema::empty();
+ let mut body = FieldConfig::new("body", FieldType::Text);
+ body.options.dimensions = Some(DIM);
+ body.options.similarity = Some("cosine".into());
+ body.embedding = Some(xerj_common::types::EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("body_vector".into()),
+ });
+ schema.add_field(body).unwrap();
+ let mut companion = FieldConfig::new("body_vector", FieldType::Vector);
+ companion.options.dimensions = Some(DIM);
+ companion.options.similarity = Some("cosine".into());
+ companion.options.quantization = quantization.map(str::to_string);
+ schema.add_field(companion).unwrap();
+ schema
+ .add_field(FieldConfig::new("tag", FieldType::Keyword))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("n", FieldType::Long))
+ .unwrap();
+ schema
+ .add_field(FieldConfig::new("title", FieldType::Text))
+ .unwrap();
+ schema
+ }
+
+ /// A document. Every third is multi-passage; a run of ids shares one
+ /// vector so the rank has exact score ties to break.
+ fn document(id: usize, rng: &mut Lcg, shared: &[f32]) -> Value {
+ let pooled = if (10..16).contains(&id) {
+ shared.to_vec()
+ } else {
+ rng.vector()
+ };
+ let mut doc = serde_json::json!({
+ "title": format!("doc {id} {}", if id.is_multiple_of(4) { "cell" } else { "other" }),
+ "tag": format!("t{}", id % 5),
+ "n": (id * 37) % 1000,
+ "body": format!("body text {id}"),
+ "body_vector": as_json(&pooled),
+ });
+ if id.is_multiple_of(3) {
+ let passages: Vec<Value> = (0..2 + id % 4).map(|_| as_json(&rng.vector())).collect();
+ doc["body_vector_chunks"] = Value::Array(passages);
+ }
+ doc
+ }
+
+ /// Two segments (the second superseding and deleting part of the first)
+ /// plus a live memtable, with every awkward document shape the scan has a
+ /// rule for.
+ async fn seed(idx: &Arc<Index>) {
+ let mut rng = Lcg(0x939);
+ let shared = rng.vector();
+ for id in 0..150 {
+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))
+ .await
+ .unwrap();
+ }
+ // Shapes with a rule of their own.
+ idx.index_document_prepared(
+ Some("malformed-passages".into()),
+ serde_json::json!({
+ "title": "cell malformed", "tag": "t1", "n": 5, "body": "x",
+ "body_vector": as_json(&rng.vector()),
+ // ordinal 1 is not a vector, ordinal 3 has the wrong length:
+ // both skipped, neither renumbers ordinal 2.
+ "body_vector_chunks": [as_json(&rng.vector()), "junk", as_json(&rng.vector()), [1.0, 2.0]],
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document_prepared(
+ Some("empty-passages".into()),
+ serde_json::json!({
+ "title": "cell empty", "tag": "t1", "n": 6, "body": "x",
+ "body_vector": as_json(&shared),
+ "body_vector_chunks": [],
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document_prepared(
+ Some("no-vector".into()),
+ serde_json::json!({"title": "cell none", "tag": "t1", "n": 7, "body": "x"}),
+ )
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+
+ // Second segment: supersede some of the first, delete some, add more.
+ for id in (0..150).step_by(9) {
+ idx.index_document_prepared(
+ Some(format!("d{id}")),
+ document(id + 1000, &mut rng, &shared),
+ )
+ .await
+ .unwrap();
+ }
+ for id in (5..150).step_by(17) {
+ idx.delete_document(&format!("d{id}")).await.unwrap();
+ }
+ for id in 150..260 {
+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ // Live memtable over both: updates of flushed documents and new ones.
+ for id in (3..260).step_by(31) {
+ idx.index_document_prepared(
+ Some(format!("d{id}")),
+ document(id + 2000, &mut rng, &shared),
+ )
+ .await
+ .unwrap();
+ }
+ for id in 260..290 {
+ idx.index_document_prepared(Some(format!("d{id}")), document(id, &mut rng, &shared))
+ .await
+ .unwrap();
+ }
+ assert!(
+ idx.store.snapshot().segments.len() >= 2,
+ "the fixture needs flushed segments beside the memtable"
+ );
+ }
+
+ fn filters() -> Vec<(&'static str, Option<QueryNode>)> {
+ let term = QueryNode::Term {
+ field: "tag".into(),
+ value: Value::from("t3"),
+ boost: None,
+ };
+ let range = QueryNode::Range {
+ field: "n".into(),
+ gte: Some(Value::from(100)),
+ gt: None,
+ lte: None,
+ lt: Some(Value::from(800)),
+ boost: None,
+ };
+ vec![
+ ("none", None),
+ (
+ "ids",
+ Some(QueryNode::Ids {
+ values: (0..290).step_by(7).map(|id| format!("d{id}")).collect(),
+ }),
+ ),
+ ("term", Some(term.clone())),
+ (
+ "bool(term AND range, NOT exists)",
+ Some(QueryNode::Bool {
+ must: Vec::new(),
+ should: Vec::new(),
+ filter: vec![term.clone(), range.clone()],
+ must_not: vec![QueryNode::Exists {
+ field: "body_vector_chunks".into(),
+ }],
+ minimum_should_match: None,
+ }),
+ ),
+ (
+ // Reads the VECTOR fields themselves: the projection must
+ // carry them rather than assume a filter never looks.
+ "exists(vector)",
+ Some(QueryNode::Exists {
+ field: "body_vector".into(),
+ }),
+ ),
+ (
+ // Not on the projection whitelist: takes the full-source path.
+ "match (full source)",
+ Some(QueryNode::Match {
+ field: "title".into(),
+ query: "cell".into(),
+ operator: xerj_query::ast::BoolOperator::Or,
+ analyzer: None,
+ boost: None,
+ minimum_should_match: None,
+ }),
+ ),
+ ]
+ }
+
+ async fn assert_scan_matches_legacy(idx: &Arc<Index>, field: &str, sq8: bool) {
+ let mut rng = Lcg(0xBEEF);
+ let mut compared = 0usize;
+ let mut saw_later_passage = false;
+ let similarities: &[&str] = if sq8 {
+ &["cosine", "l2_norm"]
+ } else {
+ &["cosine", "dot_product", "l2_norm", "max_inner_product"]
+ };
+ for _ in 0..4 {
+ let query = rng.vector();
+ for similarity in similarities {
+ for (filter_name, filter) in filters() {
+ for (k, boost, min_similarity) in [
+ (1usize, None, None),
+ (10, Some(2.5f32), None),
+ (25, None, Some(0.1f32)),
+ (1000, None, None),
+ ] {
+ let expected = legacy_scan(
+ idx,
+ field,
+ &query,
+ k,
+ filter.as_ref(),
+ similarity,
+ boost,
+ min_similarity,
+ )
+ .await;
+ let request = SearchRequest {
+ size: k,
+ fields: vec![PASSAGE_RESPONSE_FIELD.into()],
+ ..SearchRequest::default()
+ };
+ let got = idx
+ .run_knn_brute_force(
+ &request,
+ field,
+ &query,
+ k,
+ filter.clone().map(Box::new),
+ similarity,
+ boost,
+ min_similarity,
+ )
+ .await
+ .unwrap();
+ let context = format!("similarity={similarity} filter={filter_name} k={k}");
+ assert_eq!(got.total.value, expected.len() as u64, "total: {context}");
+ assert_eq!(
+ got.hits
+ .iter()
+ .map(|h| (h.id.clone(), h.score.to_bits()))
+ .collect::<Vec<_>>(),
+ expected
+ .iter()
+ .map(|(id, bits, _, _)| (id.clone(), *bits))
+ .collect::<Vec<_>>(),
+ "ids, order and score BITS: {context}"
+ );
+ for (hit, (_, _, ordinal, source)) in got.hits.iter().zip(&expected) {
+ // Hydrated from the right document: the user
+ // fields are the legacy scan's, value for value.
+ for key in ["title", "tag", "n", "body"] {
+ assert_eq!(
+ hit.source.get(key),
+ source.get(key),
+ "{key}: {context}"
+ );
+ }
+ if let Some(ordinal) = ordinal {
+ saw_later_passage |= *ordinal > 0;
+ }
+ }
+ compared += got.hits.len();
+ }
+ }
+ }
+ }
+ assert!(
+ compared > 1000,
+ "the comparison must have teeth: {compared}"
+ );
+ if !sq8 {
+ assert!(
+ saw_later_passage,
+ "no query was won by a passage past the first"
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn exact_scan_is_bit_identical_to_the_clone_everything_scan() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("scan-939", vector_schema(None))
+ .unwrap();
+ let idx = engine.get_index("scan-939").unwrap();
+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;
+ seed(&idx).await;
+ // Cold (columns derived in the scan) and warm (columns cached) must
+ // agree with the oracle and therefore with each other.
+ assert_scan_matches_legacy(&idx, "body_vector", false).await;
+ assert!(
+ !idx.vector_column_cache.is_empty(),
+ "columns were never published"
+ );
+ assert_scan_matches_legacy(&idx, "body_vector", false).await;
+ }
+
+ #[tokio::test]
+ async fn scalar8_scan_is_bit_identical_to_the_clone_everything_scan() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("scan-939-sq8", vector_schema(Some("scalar8")))
+ .unwrap();
+ let idx = engine.get_index("scan-939-sq8").unwrap();
+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;
+ seed(&idx).await;
+ assert_scan_matches_legacy(&idx, "body_vector", true).await;
+ assert_scan_matches_legacy(&idx, "body_vector", true).await;
+ }
+
+ #[tokio::test]
+ async fn vector_columns_are_per_segment_budgeted_and_retired_by_merge() {
+ let dir = TempDir::new().unwrap();
+ let engine = engine(&dir);
+ engine
+ .create_index("scan-939-cache", vector_schema(None))
+ .unwrap();
+ let idx = engine.get_index("scan-939-cache").unwrap();
+ idx.schema.write().await.dynamic = xerj_common::schema::DynamicMapping::Runtime;
+ seed(&idx).await;
+ let query = Lcg(7).vector();
+ let request = SearchRequest {
+ size: 10,
+ ..SearchRequest::default()
+ };
+ let scan = || {
+ idx.run_knn_brute_force(
+ &request,
+ "body_vector",
+ &query,
+ 10,
+ None,
+ "cosine",
+ None,
+ None,
+ )
+ };
+
+ let before = scan().await.unwrap();
+ let segments: Vec<String> = idx
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|m| m.id.clone())
+ .collect();
+ // One column per segment that HAS stored documents. A segment holding
+ // only tombstones has no stored section; the scan skips it, so there
+ // is nothing to derive a column from.
+ let mut with_stored = 0usize;
+ for id in &segments {
+ let has_stored = idx.stored_values_for(id).is_some();
+ with_stored += usize::from(has_stored);
+ assert_eq!(
+ idx.vector_column_cache
+ .contains_key(&format!("{id}\u{1}body_vector\u{1}c")),
+ has_stored,
+ "segment {id}: a column exists exactly when stored documents do"
+ );
+ }
+ assert!(with_stored >= 2, "the fixture needs several real segments");
+ assert_eq!(idx.vector_column_cache.len(), with_stored);
+ let charged = idx.segment_hydration_budget.snapshot().category_current
+ [SegmentCacheCategory::VectorColumn as usize];
+ assert!(
+ charged > 0,
+ "a published column must be charged to the hydration budget"
+ );
+
+ // A merge retires the segments; their columns must go with them, and
+ // the answer must not change.
+ idx.flush().await.unwrap();
+ idx.force_merge(1).await.unwrap();
+ let live: HashSet<String> = idx
+ .store
+ .snapshot()
+ .segments
+ .iter()
+ .map(|m| m.id.clone())
+ .collect();
+ for entry in idx.vector_column_cache.iter() {
+ let segment = entry.key().split('\u{1}').next().unwrap();
+ assert!(
+ live.contains(segment),
+ "column outlived its segment: {}",
+ entry.key()
+ );
+ }
+ let after = scan().await.unwrap();
+ let rows = |r: &SearchResult| {
+ r.hits
+ .iter()
+ .map(|h| (h.id.clone(), h.score.to_bits()))
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(
+ rows(&before),
+ rows(&after),
+ "a merge changed an exact answer"
+ );
+
+ idx.release_memory();
+ assert!(idx.vector_column_cache.is_empty());
+ assert_eq!(rows(&before), rows(&scan().await.unwrap()));
+ }
+
+ #[test]
+ fn query_scorer_is_bit_identical_to_compute_vector_similarity() {
+ let mut rng = Lcg(42);
+ let zero = vec![0.0f32; DIM];
+ for similarity in [
+ "cosine",
+ "dot_product",
+ "l2_norm",
+ "max_inner_product",
+ "anything-else",
+ ] {
+ for round in 0..500 {
+ let query = if round == 0 {
+ zero.clone()
+ } else {
+ rng.vector()
+ };
+ let stored = if round == 1 {
+ zero.clone()
+ } else {
+ rng.vector()
+ };
+ let scorer = QueryScorer::new(similarity, &query);
+ let got = scorer.score(crate::vector_column::StoredVector {
+ values: &stored,
+ ordinal: 0,
+ norm: crate::vector_column::norm_f64(&stored),
+ });
+ let legacy = legacy_similarity(similarity, &query, &stored);
+ assert_eq!(
+ got.to_bits(),
+ legacy.to_bits(),
+ "{similarity} round {round}"
+ );
+ assert_eq!(
+ compute_vector_similarity(similarity, &query, &stored).to_bits(),
+ legacy.to_bits(),
+ "refactored compute_vector_similarity drifted: {similarity} round {round}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn knn_filter_projection_agrees_with_the_full_source() {
+ let schema = vector_schema(None);
+ let docs = [
+ serde_json::json!({"tag": "t3", "n": 150, "a": {"b": {"c": 1}}, "a.b": {"c": 2}, "body_vector": [1.0], "_id": "shadowed"}),
+ serde_json::json!({"tag": ["t1", "t3"], "n": 900, "a.b.c": 3, "_routing": "r"}),
+ serde_json::json!({"tag": null, "n": "150", "a": [{"b": {"c": 4}}, {"b": {"c": 5}}]}),
+ serde_json::json!({"title": "cell", "count": 2}),
+ serde_json::json!({}),
+ Value::Null,
+ ];
+ let term = |field: &str, value: Value| QueryNode::Term {
+ field: field.into(),
+ value,
+ boost: None,
+ };
+ let exists = |field: &str| QueryNode::Exists {
+ field: field.into(),
+ };
+ let projected = vec![
+ QueryNode::MatchAll,
+ QueryNode::MatchNone,
+ QueryNode::Ids {
+ values: vec!["id-0".into(), "id-2".into()],
+ },
+ term("tag", Value::from("t3")),
+ term("_id", Value::from("id-1")),
+ term("a.b.c", Value::from(1)),
+ term("a.b.c", Value::from(3)),
+ term("a.b.c", Value::from(4)),
+ term("tag.keyword", Value::from("t3")),
+ exists("a.b"),
+ exists("a.b.c"),
+ exists("_routing"),
+ exists("_id"),
+ exists("body_vector"),
+ exists("missing"),
+ QueryNode::Terms {
+ field: "tag".into(),
+ values: vec![Value::from("t1"), Value::from("zz")],
+ boost: None,
+ },
+ QueryNode::Range {
+ field: "n".into(),
+ gte: Some(Value::from(100)),
+ gt: None,
+ lte: None,
+ lt: Some(Value::from(800)),
+ boost: None,
+ },
+ QueryNode::Prefix {
+ field: "tag".into(),
+ value: "t".into(),
+ boost: None,
+ constant_score: false,
+ },
+ QueryNode::Bool {
+ must: vec![exists("tag")],
+ should: vec![term("tag", Value::from("t3")), term("n", Value::from(900))],
+ filter: Vec::new(),
+ must_not: vec![term("_id", Value::from("id-0"))],
+ minimum_should_match: Some(MinShouldMatch::Field("count".into())),
+ },
+ QueryNode::Constant {
+ score: 1.0,
+ query: Box::new(term("tag", Value::from("t3"))),
+ },
+ ];
+ for filter in &projected {
+ let plan = KnnFilterPlan::for_filter(filter);
+ assert!(
+ matches!(plan, KnnFilterPlan::Projected(_)),
+ "should project: {filter:?}"
+ );
+ for (n, doc) in docs.iter().enumerate() {
+ let id = format!("id-{n}");
+ assert_eq!(
+ plan.matches(filter, &id, doc, &schema),
+ KnnFilterPlan::FullSource.matches(filter, &id, doc, &schema),
+ "projection changed the answer: {filter:?} on {doc}"
+ );
+ }
+ }
+ // Declined: these read the document some other way.
+ let declined = vec![
+ term("ta*", Value::from("t3")),
+ QueryNode::Script {
+ source: "true".into(),
+ params: None,
+ },
+ QueryNode::Bool {
+ must: Vec::new(),
+ should: vec![term("tag", Value::from("t3"))],
+ filter: Vec::new(),
+ must_not: Vec::new(),
+ minimum_should_match: Some(MinShouldMatch::Script {
+ source: "1".into(),
+ params: None,
+ }),
+ },
+ ];
+ for filter in &declined {
+ assert!(
+ matches!(KnnFilterPlan::for_filter(filter), KnnFilterPlan::FullSource),
+ "must not project: {filter:?}"
+ );
+ }
+ }
+
+ /// The full-source plan IS the old behaviour: the whole source with `_id`
+ /// inserted over whatever the document carried under that key.
+ #[test]
+ fn full_source_plan_inserts_the_id_like_the_old_scan_did() {
+ let schema = vector_schema(None);
+ let filter = QueryNode::Ids {
+ values: vec!["real".into()],
+ };
+ let doc = serde_json::json!({"_id": "shadowed", "tag": "t"});
+ assert!(KnnFilterPlan::FullSource.matches(&filter, "real", &doc, &schema));
+ assert!(!KnnFilterPlan::FullSource.matches(&filter, "other", &doc, &schema));
+ }
+ }