Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): match a numeric term by value in the memtable and _source paths (#2667)

Amartya Vikram Singh8ea684d986f7
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index 3929d7604..37b0ee50e 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -105,6 +105,8 @@ fn mem_preds_from_agg_filter(filter: &Value) -> Option<Vec<MemBoolPred>> {
out.push(MemBoolPred::Term {
field: field.clone(),
value,
+ // String needles only (numbers bail above).
+ num: None,
});
Some(())
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 3f483d17d..0915123b7 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -49473,6 +49473,7 @@ fn mem_bool_preds(q: &QueryNode) -> Option<Vec<crate::memtable::MemBoolPred>> {
preds.push(MemBoolPred::Term {
field: field.clone(),
value: val_str,
+ num: value.as_f64(),
});
}
QueryNode::Terms { field, values, .. } => {
@@ -49498,6 +49499,7 @@ fn mem_bool_preds(q: &QueryNode) -> Option<Vec<crate::memtable::MemBoolPred>> {
preds.push(MemBoolPred::Terms {
field: field.clone(),
values: lowered,
+ nums: values.iter().filter_map(Value::as_f64).collect(),
});
}
QueryNode::Range {
@@ -49549,6 +49551,18 @@ fn try_doc_values_query(
limit: usize,
) -> Option<(Vec<(String, usize)>, u64)> {
match q {
+ // A JSON-number needle must match by value, which the keyword-column
+ // walks below cannot do (they compare the needle's string form with
+ // the document's own spelling, so `10` missed a stored `10.0`). The
+ // fused walk carries the typed needle; it has the same array, dotted
+ // path and analyzed-text bailouts.
+ QueryNode::Term {
+ value: Value::Number(_),
+ ..
+ } => mem.doc_values_bool_query(&mem_bool_preds(q)?, limit),
+ QueryNode::Terms { values, .. } if values.iter().any(Value::is_number) => {
+ mem.doc_values_bool_query(&mem_bool_preds(q)?, limit)
+ }
QueryNode::Term { field, value, .. } => {
// Skip if the value is complex or the field uses CIDR notation.
let val_str = match value {
@@ -55241,6 +55255,17 @@ fn json_scalar_equal(dv: &Value, query_val: &Value) -> bool {
}
// Cross-type: number stored as string or vice versa.
match (dv, query_val) {
+ // Two JSON numbers compare by value: serde_json's `Number` equality
+ // is representational, so a stored `10.0` was not equal to a queried
+ // `10`. Integers compare exactly (above 2^53 an f64 round-trip would
+ // merge neighbours); anything else compares as f64.
+ (Value::Number(a), Value::Number(b)) => match (a.as_i64(), b.as_i64()) {
+ (Some(x), Some(y)) => x == y,
+ _ => match (a.as_u64(), b.as_u64()) {
+ (Some(x), Some(y)) => x == y,
+ _ => a.as_f64() == b.as_f64(),
+ },
+ },
(Value::String(s), Value::Number(n)) => s.parse::<f64>().ok() == n.as_f64(),
(Value::Number(n), Value::String(s)) => n.as_f64() == s.parse::<f64>().ok(),
// Cross-type: a boolean and its string spelling name the same
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 1faea93fe..e6ab9521b 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -2991,22 +2991,38 @@ fn entry_no_clone<'a, V>(
map.get_mut(key).expect("just inserted / already present")
}
+/// Whether a keyword-column value names the number `num`. The column holds
+/// a JSON number as its source spelling (`10.0`, `1e1`), so a numeric needle
+/// compares by value; `None` (a string needle) never matches here.
+fn kw_equals_number(stored: &str, num: Option<f64>) -> bool {
+ num.is_some_and(|n| stored.parse::<f64>().is_ok_and(|f| f == n))
+}
+
/// One resolved predicate of a `bool { must/filter }` for the fused
/// columnar memtable walk (`doc_values_bool_query`). Built by the engine
/// from Term / Range query nodes; semantics per predicate mirror
/// `doc_values_term_query` / `doc_values_range_query` exactly.
pub enum MemBoolPred {
+ /// `num` is the needle's value when the query spelled it as a JSON
+ /// number. A numeric needle matches numerically, as on the flushed
+ /// columnar path and the `_source` comparator: the keyword column renders
+ /// a number the way its document spelled it, so `10` must also match a
+ /// stored `10.0` (and `1e1`). A string needle stays an exact match.
Term {
field: String,
value: String,
+ num: Option<f64>,
},
/// Disjunction of exact values on one field. The catalog finalize-verify
/// query is `bool.filter [terms file_key, term run_id]`; without this arm
/// that bool fell through to `DocsForScan` and cloned every buffered
/// catalog row, including the other run's (#1761).
+ /// `nums` holds the values the query spelled as JSON numbers; see
+ /// `Term::num`.
Terms {
field: String,
values: Vec<String>,
+ nums: Vec<f64>,
},
Range {
field: String,
@@ -4106,8 +4122,8 @@ impl FtsMemtable {
limit: usize,
) -> Option<(Vec<(String, usize)>, u64)> {
enum Col<'a> {
- Kw(&'a Vec<Option<String>>, &'a str),
- KwAny(&'a Vec<Option<String>>, FxHashSet<String>),
+ Kw(&'a Vec<Option<String>>, &'a str, Option<f64>),
+ KwAny(&'a Vec<Option<String>>, FxHashSet<String>, FxHashSet<u64>),
NumEq(&'a Vec<Option<f64>>, f64),
NumAny(&'a Vec<Option<f64>>, FxHashSet<u64>),
Num(
@@ -4121,7 +4137,7 @@ impl FtsMemtable {
let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());
for p in preds {
match p {
- MemBoolPred::Term { field, value } => {
+ MemBoolPred::Term { field, value, num } => {
// Array-valued field: the column is lossy (first element
// only) → bail so a later matching element can't be
// dropped (#332; mirrors `filtered_docs_arc`).
@@ -4147,7 +4163,7 @@ impl FtsMemtable {
{
return None;
}
- cols.push(Col::Kw(col, value.as_str()));
+ cols.push(Col::Kw(col, value.as_str(), *num));
} else {
// Numeric term predicate: exact f64 equality (the
// predicate value was lowered to its string form).
@@ -4156,7 +4172,11 @@ impl FtsMemtable {
cols.push(Col::NumEq(col, needle));
}
}
- MemBoolPred::Terms { field, values } => {
+ MemBoolPred::Terms {
+ field,
+ values,
+ nums,
+ } => {
if self.doc_values.array_fields.contains(field.as_str()) {
return None;
}
@@ -4179,7 +4199,9 @@ impl FtsMemtable {
return None;
}
let set: FxHashSet<String> = values.iter().cloned().collect();
- cols.push(Col::KwAny(col, set));
+ let num_set: FxHashSet<u64> =
+ nums.iter().map(|n| (n + 0.0).to_bits()).collect();
+ cols.push(Col::KwAny(col, set, num_set));
} else {
let col = self.doc_values.numeric.get(field.as_str())?;
let mut set: FxHashSet<u64> = FxHashSet::default();
@@ -4215,11 +4237,18 @@ impl FtsMemtable {
'doc: for idx in 0..n {
for c in &cols {
let ok = match c {
- Col::Kw(col, want) => col.get(idx).and_then(|o| o.as_deref()) == Some(*want),
- Col::KwAny(col, set) => col
+ Col::Kw(col, want, num) => col
.get(idx)
.and_then(|o| o.as_deref())
- .is_some_and(|v| set.contains(v)),
+ .is_some_and(|v| v == *want || kw_equals_number(v, *num)),
+ Col::KwAny(col, set, nums) => {
+ col.get(idx).and_then(|o| o.as_deref()).is_some_and(|v| {
+ set.contains(v)
+ || (!nums.is_empty()
+ && v.parse::<f64>()
+ .is_ok_and(|f| nums.contains(&(f + 0.0).to_bits())))
+ })
+ }
Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),
Col::NumAny(col, set) => col
.get(idx)
@@ -4284,8 +4313,8 @@ impl FtsMemtable {
out: &mut Vec<(String, Arc<Value>)>,
) -> Option<()> {
enum Col<'a> {
- Kw(&'a Vec<Option<String>>, &'a str),
- KwAny(&'a Vec<Option<String>>, FxHashSet<String>),
+ Kw(&'a Vec<Option<String>>, &'a str, Option<f64>),
+ KwAny(&'a Vec<Option<String>>, FxHashSet<String>, FxHashSet<u64>),
NumEq(&'a Vec<Option<f64>>, f64),
NumAny(&'a Vec<Option<f64>>, FxHashSet<u64>),
Num(
@@ -4299,7 +4328,7 @@ impl FtsMemtable {
let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());
for p in preds {
match p {
- MemBoolPred::Term { field, value } => {
+ MemBoolPred::Term { field, value, num } => {
// Array-valued field: the column is lossy (first element
// only) → bail so a later matching element can't be dropped.
if self.doc_values.array_fields.contains(field.as_str()) {
@@ -4319,14 +4348,18 @@ impl FtsMemtable {
{
return None;
}
- cols.push(Col::Kw(col, value.as_str()));
+ cols.push(Col::Kw(col, value.as_str(), *num));
} else {
let col = self.doc_values.numeric.get(field.as_str())?;
let needle = value.parse::<f64>().ok()?;
cols.push(Col::NumEq(col, needle));
}
}
- MemBoolPred::Terms { field, values } => {
+ MemBoolPred::Terms {
+ field,
+ values,
+ nums,
+ } => {
if self.doc_values.array_fields.contains(field.as_str()) {
return None;
}
@@ -4346,7 +4379,9 @@ impl FtsMemtable {
return None;
}
let set: FxHashSet<String> = values.iter().cloned().collect();
- cols.push(Col::KwAny(col, set));
+ let num_set: FxHashSet<u64> =
+ nums.iter().map(|n| (n + 0.0).to_bits()).collect();
+ cols.push(Col::KwAny(col, set, num_set));
} else {
let col = self.doc_values.numeric.get(field.as_str())?;
let mut set: FxHashSet<u64> = FxHashSet::default();
@@ -4380,11 +4415,18 @@ impl FtsMemtable {
'doc: for idx in 0..n {
for c in &cols {
let ok = match c {
- Col::Kw(col, want) => col.get(idx).and_then(|o| o.as_deref()) == Some(*want),
- Col::KwAny(col, set) => col
+ Col::Kw(col, want, num) => col
.get(idx)
.and_then(|o| o.as_deref())
- .is_some_and(|v| set.contains(v)),
+ .is_some_and(|v| v == *want || kw_equals_number(v, *num)),
+ Col::KwAny(col, set, nums) => {
+ col.get(idx).and_then(|o| o.as_deref()).is_some_and(|v| {
+ set.contains(v)
+ || (!nums.is_empty()
+ && v.parse::<f64>()
+ .is_ok_and(|f| nums.contains(&(f + 0.0).to_bits())))
+ })
+ }
Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),
Col::NumAny(col, set) => col
.get(idx)
@@ -6338,6 +6380,7 @@ mod filtered_docs_arc_tests {
MemBoolPred::Term {
field: "status".to_string(),
value: "s2".to_string(),
+ num: None,
},
MemBoolPred::Range {
field: "n".to_string(),
@@ -6365,6 +6408,7 @@ mod filtered_docs_arc_tests {
let preds = vec![MemBoolPred::Term {
field: "status".to_string(),
value: "s3".to_string(),
+ num: None,
}];
let filter = json!({ "term": { "status": "s3" } });
let brute = brute_ids(&mem, &filter);
@@ -6381,6 +6425,7 @@ mod filtered_docs_arc_tests {
let terms = vec![MemBoolPred::Terms {
field: "status".to_string(),
values: vec!["s1".to_string(), "s2".to_string()],
+ nums: Vec::new(),
}];
let terms_filter = json!({ "terms": { "status": ["s1", "s2"] } });
let terms_brute = brute_ids(&mem, &terms_filter);
@@ -6392,10 +6437,12 @@ mod filtered_docs_arc_tests {
MemBoolPred::Terms {
field: "status".to_string(),
values: vec!["s1".to_string(), "s2".to_string()],
+ nums: Vec::new(),
},
MemBoolPred::Term {
field: "status".to_string(),
value: "s1".to_string(),
+ num: None,
},
];
let conj_filter = json!({
diff --git a/engine/crates/xerj-engine/tests/numeric_term_value_equality.rs b/engine/crates/xerj-engine/tests/numeric_term_value_equality.rs
new file mode 100644
index 000000000..5ab1eccbf
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/numeric_term_value_equality.rs
@@ -0,0 +1,123 @@
+//! A `term` / `terms` with a JSON-number value matches a numeric field by
+//! value, whether the document spelled the number `10` or `10.0`.
+//!
+//! Elasticsearch parses a numeric term through the field's number type, so
+//! `term {price: 10}` matches a document indexed as `{"price": 10.0}` and the
+//! reverse. XERJ answered that only after a flush:
+//!
+//! * buffered, the fused columnar walk (`doc_values_bool_hits`) and the
+//! keyword-column walks compared the needle's string form (`"10"`) with
+//! the document's own spelling (`"10.0"`) — `size: 10` found no hits while
+//! the `size: 0` count shortcut, which compares f64 values, counted one;
+//! * the `_source` comparator used serde_json's representational `Number`
+//! equality, where `PosInt(10) != Float(10.0)`.
+//!
+//! Every case runs buffered and flushed, at `size: 0` and `size: 10`, and once
+//! more with a sort (which reaches the keyword-column walk instead of the
+//! fused one).
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+async fn seed(engine: &Engine) {
+ let mut schema = Schema::empty();
+ for (name, ty) in [
+ ("price", FieldType::Double),
+ ("n", FieldType::Long),
+ ("code", FieldType::Keyword),
+ ] {
+ schema.fields.push(FieldConfig::new(name, ty));
+ }
+ engine.create_index("t", schema).expect("create");
+ let idx = engine.get_index("t").expect("get");
+ for (id, doc) in [
+ (
+ "a",
+ json!({"price": 10.0, "n": 2.0, "code": "10.0", "rank": 1}),
+ ),
+ ("b", json!({"price": 10, "n": 2, "code": "10", "rank": 2})),
+ ("c", json!({"price": 10.5, "n": 3, "code": "x", "rank": 3})),
+ ] {
+ idx.index_document(Some(id.to_string()), doc)
+ .await
+ .expect("index");
+ }
+}
+
+async fn run(engine: &Engine, query: &Value) -> (u64, u64, Vec<String>, Vec<String>) {
+ let idx = engine.get_index("t").expect("get");
+ let search = |body: Value| {
+ let req = parse_request(&body).unwrap();
+ let idx = idx.clone();
+ async move { idx.search(&req).await.expect("search") }
+ };
+ let count = search(json!({"query": query, "size": 0})).await.total.value;
+ let hits = search(json!({"query": query, "size": 10})).await;
+ let sorted = search(json!({"query": query, "size": 10, "sort": [{"rank": "asc"}]})).await;
+ let mut ids: Vec<String> = hits.hits.iter().map(|h| h.id.clone()).collect();
+ ids.sort();
+ let sorted_ids = sorted.hits.iter().map(|h| h.id.clone()).collect();
+ (count, hits.total.value, ids, sorted_ids)
+}
+
+fn cases() -> Vec<(Value, Vec<&'static str>)> {
+ vec![
+ (json!({"term": {"price": 10}}), vec!["a", "b"]),
+ (json!({"term": {"price": 10.0}}), vec!["a", "b"]),
+ (json!({"term": {"price": 10.5}}), vec!["c"]),
+ (json!({"term": {"n": 2}}), vec!["a", "b"]),
+ (json!({"term": {"n": 2.0}}), vec!["a", "b"]),
+ (json!({"terms": {"price": [10, 99]}}), vec!["a", "b"]),
+ (json!({"terms": {"n": [2.0, 3]}}), vec!["a", "b", "c"]),
+ (
+ json!({"bool": {"filter": [{"term": {"price": 10}}, {"term": {"n": 2.0}}]}}),
+ vec!["a", "b"],
+ ),
+ // A STRING needle on a keyword field stays an exact match: ES
+ // compares keyword terms as bytes, so "10" is not "10.0".
+ (json!({"term": {"code": "10"}}), vec!["b"]),
+ (json!({"term": {"code": "10.0"}}), vec!["a"]),
+ (json!({"terms": {"code": ["10"]}}), vec!["b"]),
+ ]
+}
+
+async fn assert_cases(engine: &Engine, phase: &str) {
+ for (query, want) in cases() {
+ let (count, total, got, sorted) = run(engine, &query).await;
+ assert_eq!(got, want, "[{phase}] {query}: hit ids (size:10)");
+ assert_eq!(total, want.len() as u64, "[{phase}] {query}: hits.total");
+ assert_eq!(
+ count,
+ want.len() as u64,
+ "[{phase}] {query}: count (size:0)"
+ );
+ assert_eq!(sorted, want, "[{phase}] {query}: sorted hit ids");
+ }
+}
+
+/// FAIL-BEFORE (buffered): `term price 10` → count 1 (or 2), hits 1 (`b`
+/// only), so `size: 0` and `size: 10` disagreed and doc `a` was missing.
+#[tokio::test]
+async fn numeric_term_matches_by_value_buffered_and_flushed() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine).await;
+ assert_cases(&engine, "buffered").await;
+ engine
+ .get_index("t")
+ .unwrap()
+ .refresh()
+ .await
+ .expect("refresh");
+ assert_cases(&engine, "flushed").await;
+}