AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): match a numeric term by value in the memtable and _source paths (#2667)
Amartya Vikram Singh8ea684d986f7
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rsindex 3929d7604..37b0ee50e 100644--- a/engine/crates/xerj-engine/src/fast_aggs.rs+++ b/engine/crates/xerj-engine/src/fast_aggs.rs@@ -105,6 +105,8 @@ fn mem_preds_from_agg_filter(filter: &Value) -> Option<Vec<MemBoolPred>> {out.push(MemBoolPred::Term {field: field.clone(),value,+ // String needles only (numbers bail above).+ num: None,});Some(())}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 3f483d17d..0915123b7 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -49473,6 +49473,7 @@ fn mem_bool_preds(q: &QueryNode) -> Option<Vec<crate::memtable::MemBoolPred>> {preds.push(MemBoolPred::Term {field: field.clone(),value: val_str,+ num: value.as_f64(),});}QueryNode::Terms { field, values, .. } => {@@ -49498,6 +49499,7 @@ fn mem_bool_preds(q: &QueryNode) -> Option<Vec<crate::memtable::MemBoolPred>> {preds.push(MemBoolPred::Terms {field: field.clone(),values: lowered,+ nums: values.iter().filter_map(Value::as_f64).collect(),});}QueryNode::Range {@@ -49549,6 +49551,18 @@ fn try_doc_values_query(limit: usize,) -> Option<(Vec<(String, usize)>, u64)> {match q {+ // A JSON-number needle must match by value, which the keyword-column+ // walks below cannot do (they compare the needle's string form with+ // the document's own spelling, so `10` missed a stored `10.0`). The+ // fused walk carries the typed needle; it has the same array, dotted+ // path and analyzed-text bailouts.+ QueryNode::Term {+ value: Value::Number(_),+ ..+ } => mem.doc_values_bool_query(&mem_bool_preds(q)?, limit),+ QueryNode::Terms { values, .. } if values.iter().any(Value::is_number) => {+ mem.doc_values_bool_query(&mem_bool_preds(q)?, limit)+ }QueryNode::Term { field, value, .. } => {// Skip if the value is complex or the field uses CIDR notation.let val_str = match value {@@ -55241,6 +55255,17 @@ fn json_scalar_equal(dv: &Value, query_val: &Value) -> bool {}// Cross-type: number stored as string or vice versa.match (dv, query_val) {+ // Two JSON numbers compare by value: serde_json's `Number` equality+ // is representational, so a stored `10.0` was not equal to a queried+ // `10`. Integers compare exactly (above 2^53 an f64 round-trip would+ // merge neighbours); anything else compares as f64.+ (Value::Number(a), Value::Number(b)) => match (a.as_i64(), b.as_i64()) {+ (Some(x), Some(y)) => x == y,+ _ => match (a.as_u64(), b.as_u64()) {+ (Some(x), Some(y)) => x == y,+ _ => a.as_f64() == b.as_f64(),+ },+ },(Value::String(s), Value::Number(n)) => s.parse::<f64>().ok() == n.as_f64(),(Value::Number(n), Value::String(s)) => n.as_f64() == s.parse::<f64>().ok(),// Cross-type: a boolean and its string spelling name the samediff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex 1faea93fe..e6ab9521b 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -2991,22 +2991,38 @@ fn entry_no_clone<'a, V>(map.get_mut(key).expect("just inserted / already present")}+/// Whether a keyword-column value names the number `num`. The column holds+/// a JSON number as its source spelling (`10.0`, `1e1`), so a numeric needle+/// compares by value; `None` (a string needle) never matches here.+fn kw_equals_number(stored: &str, num: Option<f64>) -> bool {+ num.is_some_and(|n| stored.parse::<f64>().is_ok_and(|f| f == n))+}+/// One resolved predicate of a `bool { must/filter }` for the fused/// columnar memtable walk (`doc_values_bool_query`). Built by the engine/// from Term / Range query nodes; semantics per predicate mirror/// `doc_values_term_query` / `doc_values_range_query` exactly.pub enum MemBoolPred {+ /// `num` is the needle's value when the query spelled it as a JSON+ /// number. A numeric needle matches numerically, as on the flushed+ /// columnar path and the `_source` comparator: the keyword column renders+ /// a number the way its document spelled it, so `10` must also match a+ /// stored `10.0` (and `1e1`). A string needle stays an exact match.Term {field: String,value: String,+ num: Option<f64>,},/// Disjunction of exact values on one field. The catalog finalize-verify/// query is `bool.filter [terms file_key, term run_id]`; without this arm/// that bool fell through to `DocsForScan` and cloned every buffered/// catalog row, including the other run's (#1761).+ /// `nums` holds the values the query spelled as JSON numbers; see+ /// `Term::num`.Terms {field: String,values: Vec<String>,+ nums: Vec<f64>,},Range {field: String,@@ -4106,8 +4122,8 @@ impl FtsMemtable {limit: usize,) -> Option<(Vec<(String, usize)>, u64)> {enum Col<'a> {- Kw(&'a Vec<Option<String>>, &'a str),- KwAny(&'a Vec<Option<String>>, FxHashSet<String>),+ Kw(&'a Vec<Option<String>>, &'a str, Option<f64>),+ KwAny(&'a Vec<Option<String>>, FxHashSet<String>, FxHashSet<u64>),NumEq(&'a Vec<Option<f64>>, f64),NumAny(&'a Vec<Option<f64>>, FxHashSet<u64>),Num(@@ -4121,7 +4137,7 @@ impl FtsMemtable {let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());for p in preds {match p {- MemBoolPred::Term { field, value } => {+ MemBoolPred::Term { field, value, num } => {// Array-valued field: the column is lossy (first element// only) → bail so a later matching element can't be// dropped (#332; mirrors `filtered_docs_arc`).@@ -4147,7 +4163,7 @@ impl FtsMemtable {{return None;}- cols.push(Col::Kw(col, value.as_str()));+ cols.push(Col::Kw(col, value.as_str(), *num));} else {// Numeric term predicate: exact f64 equality (the// predicate value was lowered to its string form).@@ -4156,7 +4172,11 @@ impl FtsMemtable {cols.push(Col::NumEq(col, needle));}}- MemBoolPred::Terms { field, values } => {+ MemBoolPred::Terms {+ field,+ values,+ nums,+ } => {if self.doc_values.array_fields.contains(field.as_str()) {return None;}@@ -4179,7 +4199,9 @@ impl FtsMemtable {return None;}let set: FxHashSet<String> = values.iter().cloned().collect();- cols.push(Col::KwAny(col, set));+ let num_set: FxHashSet<u64> =+ nums.iter().map(|n| (n + 0.0).to_bits()).collect();+ cols.push(Col::KwAny(col, set, num_set));} else {let col = self.doc_values.numeric.get(field.as_str())?;let mut set: FxHashSet<u64> = FxHashSet::default();@@ -4215,11 +4237,18 @@ impl FtsMemtable {'doc: for idx in 0..n {for c in &cols {let ok = match c {- Col::Kw(col, want) => col.get(idx).and_then(|o| o.as_deref()) == Some(*want),- Col::KwAny(col, set) => col+ Col::Kw(col, want, num) => col.get(idx).and_then(|o| o.as_deref())- .is_some_and(|v| set.contains(v)),+ .is_some_and(|v| v == *want || kw_equals_number(v, *num)),+ Col::KwAny(col, set, nums) => {+ col.get(idx).and_then(|o| o.as_deref()).is_some_and(|v| {+ set.contains(v)+ || (!nums.is_empty()+ && v.parse::<f64>()+ .is_ok_and(|f| nums.contains(&(f + 0.0).to_bits())))+ })+ }Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),Col::NumAny(col, set) => col.get(idx)@@ -4284,8 +4313,8 @@ impl FtsMemtable {out: &mut Vec<(String, Arc<Value>)>,) -> Option<()> {enum Col<'a> {- Kw(&'a Vec<Option<String>>, &'a str),- KwAny(&'a Vec<Option<String>>, FxHashSet<String>),+ Kw(&'a Vec<Option<String>>, &'a str, Option<f64>),+ KwAny(&'a Vec<Option<String>>, FxHashSet<String>, FxHashSet<u64>),NumEq(&'a Vec<Option<f64>>, f64),NumAny(&'a Vec<Option<f64>>, FxHashSet<u64>),Num(@@ -4299,7 +4328,7 @@ impl FtsMemtable {let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());for p in preds {match p {- MemBoolPred::Term { field, value } => {+ MemBoolPred::Term { field, value, num } => {// Array-valued field: the column is lossy (first element// only) → bail so a later matching element can't be dropped.if self.doc_values.array_fields.contains(field.as_str()) {@@ -4319,14 +4348,18 @@ impl FtsMemtable {{return None;}- cols.push(Col::Kw(col, value.as_str()));+ cols.push(Col::Kw(col, value.as_str(), *num));} else {let col = self.doc_values.numeric.get(field.as_str())?;let needle = value.parse::<f64>().ok()?;cols.push(Col::NumEq(col, needle));}}- MemBoolPred::Terms { field, values } => {+ MemBoolPred::Terms {+ field,+ values,+ nums,+ } => {if self.doc_values.array_fields.contains(field.as_str()) {return None;}@@ -4346,7 +4379,9 @@ impl FtsMemtable {return None;}let set: FxHashSet<String> = values.iter().cloned().collect();- cols.push(Col::KwAny(col, set));+ let num_set: FxHashSet<u64> =+ nums.iter().map(|n| (n + 0.0).to_bits()).collect();+ cols.push(Col::KwAny(col, set, num_set));} else {let col = self.doc_values.numeric.get(field.as_str())?;let mut set: FxHashSet<u64> = FxHashSet::default();@@ -4380,11 +4415,18 @@ impl FtsMemtable {'doc: for idx in 0..n {for c in &cols {let ok = match c {- Col::Kw(col, want) => col.get(idx).and_then(|o| o.as_deref()) == Some(*want),- Col::KwAny(col, set) => col+ Col::Kw(col, want, num) => col.get(idx).and_then(|o| o.as_deref())- .is_some_and(|v| set.contains(v)),+ .is_some_and(|v| v == *want || kw_equals_number(v, *num)),+ Col::KwAny(col, set, nums) => {+ col.get(idx).and_then(|o| o.as_deref()).is_some_and(|v| {+ set.contains(v)+ || (!nums.is_empty()+ && v.parse::<f64>()+ .is_ok_and(|f| nums.contains(&(f + 0.0).to_bits())))+ })+ }Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),Col::NumAny(col, set) => col.get(idx)@@ -6338,6 +6380,7 @@ mod filtered_docs_arc_tests {MemBoolPred::Term {field: "status".to_string(),value: "s2".to_string(),+ num: None,},MemBoolPred::Range {field: "n".to_string(),@@ -6365,6 +6408,7 @@ mod filtered_docs_arc_tests {let preds = vec![MemBoolPred::Term {field: "status".to_string(),value: "s3".to_string(),+ num: None,}];let filter = json!({ "term": { "status": "s3" } });let brute = brute_ids(&mem, &filter);@@ -6381,6 +6425,7 @@ mod filtered_docs_arc_tests {let terms = vec![MemBoolPred::Terms {field: "status".to_string(),values: vec!["s1".to_string(), "s2".to_string()],+ nums: Vec::new(),}];let terms_filter = json!({ "terms": { "status": ["s1", "s2"] } });let terms_brute = brute_ids(&mem, &terms_filter);@@ -6392,10 +6437,12 @@ mod filtered_docs_arc_tests {MemBoolPred::Terms {field: "status".to_string(),values: vec!["s1".to_string(), "s2".to_string()],+ nums: Vec::new(),},MemBoolPred::Term {field: "status".to_string(),value: "s1".to_string(),+ num: None,},];let conj_filter = json!({diff --git a/engine/crates/xerj-engine/tests/numeric_term_value_equality.rs b/engine/crates/xerj-engine/tests/numeric_term_value_equality.rsnew file mode 100644index 000000000..5ab1eccbf--- /dev/null+++ b/engine/crates/xerj-engine/tests/numeric_term_value_equality.rs@@ -0,0 +1,123 @@+//! A `term` / `terms` with a JSON-number value matches a numeric field by+//! value, whether the document spelled the number `10` or `10.0`.+//!+//! Elasticsearch parses a numeric term through the field's number type, so+//! `term {price: 10}` matches a document indexed as `{"price": 10.0}` and the+//! reverse. XERJ answered that only after a flush:+//!+//! * buffered, the fused columnar walk (`doc_values_bool_hits`) and the+//! keyword-column walks compared the needle's string form (`"10"`) with+//! the document's own spelling (`"10.0"`) — `size: 10` found no hits while+//! the `size: 0` count shortcut, which compares f64 values, counted one;+//! * the `_source` comparator used serde_json's representational `Number`+//! equality, where `PosInt(10) != Float(10.0)`.+//!+//! Every case runs buffered and flushed, at `size: 0` and `size: 10`, and once+//! more with a sort (which reaches the keyword-column walk instead of the+//! fused one).++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::Engine;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++async fn seed(engine: &Engine) {+ let mut schema = Schema::empty();+ for (name, ty) in [+ ("price", FieldType::Double),+ ("n", FieldType::Long),+ ("code", FieldType::Keyword),+ ] {+ schema.fields.push(FieldConfig::new(name, ty));+ }+ engine.create_index("t", schema).expect("create");+ let idx = engine.get_index("t").expect("get");+ for (id, doc) in [+ (+ "a",+ json!({"price": 10.0, "n": 2.0, "code": "10.0", "rank": 1}),+ ),+ ("b", json!({"price": 10, "n": 2, "code": "10", "rank": 2})),+ ("c", json!({"price": 10.5, "n": 3, "code": "x", "rank": 3})),+ ] {+ idx.index_document(Some(id.to_string()), doc)+ .await+ .expect("index");+ }+}++async fn run(engine: &Engine, query: &Value) -> (u64, u64, Vec<String>, Vec<String>) {+ let idx = engine.get_index("t").expect("get");+ let search = |body: Value| {+ let req = parse_request(&body).unwrap();+ let idx = idx.clone();+ async move { idx.search(&req).await.expect("search") }+ };+ let count = search(json!({"query": query, "size": 0})).await.total.value;+ let hits = search(json!({"query": query, "size": 10})).await;+ let sorted = search(json!({"query": query, "size": 10, "sort": [{"rank": "asc"}]})).await;+ let mut ids: Vec<String> = hits.hits.iter().map(|h| h.id.clone()).collect();+ ids.sort();+ let sorted_ids = sorted.hits.iter().map(|h| h.id.clone()).collect();+ (count, hits.total.value, ids, sorted_ids)+}++fn cases() -> Vec<(Value, Vec<&'static str>)> {+ vec![+ (json!({"term": {"price": 10}}), vec!["a", "b"]),+ (json!({"term": {"price": 10.0}}), vec!["a", "b"]),+ (json!({"term": {"price": 10.5}}), vec!["c"]),+ (json!({"term": {"n": 2}}), vec!["a", "b"]),+ (json!({"term": {"n": 2.0}}), vec!["a", "b"]),+ (json!({"terms": {"price": [10, 99]}}), vec!["a", "b"]),+ (json!({"terms": {"n": [2.0, 3]}}), vec!["a", "b", "c"]),+ (+ json!({"bool": {"filter": [{"term": {"price": 10}}, {"term": {"n": 2.0}}]}}),+ vec!["a", "b"],+ ),+ // A STRING needle on a keyword field stays an exact match: ES+ // compares keyword terms as bytes, so "10" is not "10.0".+ (json!({"term": {"code": "10"}}), vec!["b"]),+ (json!({"term": {"code": "10.0"}}), vec!["a"]),+ (json!({"terms": {"code": ["10"]}}), vec!["b"]),+ ]+}++async fn assert_cases(engine: &Engine, phase: &str) {+ for (query, want) in cases() {+ let (count, total, got, sorted) = run(engine, &query).await;+ assert_eq!(got, want, "[{phase}] {query}: hit ids (size:10)");+ assert_eq!(total, want.len() as u64, "[{phase}] {query}: hits.total");+ assert_eq!(+ count,+ want.len() as u64,+ "[{phase}] {query}: count (size:0)"+ );+ assert_eq!(sorted, want, "[{phase}] {query}: sorted hit ids");+ }+}++/// FAIL-BEFORE (buffered): `term price 10` → count 1 (or 2), hits 1 (`b`+/// only), so `size: 0` and `size: 10` disagreed and doc `a` was missing.+#[tokio::test]+async fn numeric_term_matches_by_value_buffered_and_flushed() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine).await;+ assert_cases(&engine, "buffered").await;+ engine+ .get_index("t")+ .unwrap()+ .refresh()+ .await+ .expect("refresh");+ assert_cases(&engine, "flushed").await;+}