Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): keep doc-values text flush-invariant (#1517)

Lenin Mookiah2e495f0c5f56
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 757b8248f..6adf9604e 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -21537,6 +21537,8 @@ impl Index {
.clone()
};
+ let unindexed_dv_text_match = has_unindexed_doc_values_text_match(query, &dmq_schema);
+
let mem_snapshot = {
let mem = &*self.memtable;
if mem_doc_count == 0 {
@@ -21631,6 +21633,12 @@ impl Index {
let extra = total.saturating_sub(ids.len() as u64);
MemSnapshot::AllDocIds(ids, extra)
}
+ } else if unindexed_dv_text_match {
+ // These fields have no memtable postings. Route direct Match,
+ // normalized bool-Match, and DisMax-of-Match shapes through
+ // the mapping-aware scan so pre-flush answers use the same
+ // whole-value column semantics as flushed documents.
+ MemSnapshot::DocsForScan(mem.all_docs_with_seq_arc())
} else if let Some((dm_subs, dm_tie)) = dis_max_mem_fts_subs(query) {
// #834: a `dis_max` of single-field text `match` subs used
// to fall through to the stored-doc scan, whose scorer has
@@ -41527,6 +41535,45 @@ fn field_is_analyzed_text(schema: &Schema, field: &str) -> bool {
.is_some_and(|f| matches!(f.field_type, FieldType::Text) && f.options.indexed)
}
+/// Does `q` contain a `match` leaf for declared doc-values-only text?
+fn has_unindexed_doc_values_text_match(q: &QueryNode, schema: &Schema) -> bool {
+ match q {
+ QueryNode::Match { field, .. } => declared_field(schema, field).is_some_and(|f| {
+ matches!(f.field_type, FieldType::Text) && !f.options.indexed && f.options.doc_values
+ }),
+ QueryNode::Bool {
+ must,
+ should,
+ must_not,
+ filter,
+ ..
+ } => must
+ .iter()
+ .chain(should)
+ .chain(must_not)
+ .chain(filter)
+ .any(|child| has_unindexed_doc_values_text_match(child, schema)),
+ QueryNode::DisMax { queries, .. } => queries
+ .iter()
+ .any(|child| has_unindexed_doc_values_text_match(child, schema)),
+ QueryNode::Constant { query, .. }
+ | QueryNode::Boosted { query, .. }
+ | QueryNode::Named { query, .. }
+ | QueryNode::Nested { query, .. }
+ | QueryNode::FunctionScore { query, .. } => {
+ has_unindexed_doc_values_text_match(query, schema)
+ }
+ QueryNode::Pinned { organic, .. } => has_unindexed_doc_values_text_match(organic, schema),
+ QueryNode::Boosting {
+ positive, negative, ..
+ } => {
+ has_unindexed_doc_values_text_match(positive, schema)
+ || has_unindexed_doc_values_text_match(negative, schema)
+ }
+ _ => false,
+ }
+}
+
/// Does `q` carry a `term`/`terms` leaf naming an analyzed `text` field?
///
/// The doc-values shapes [`field_is_analyzed_text`] cannot resolve are
@@ -45738,6 +45785,15 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo
if values.iter().any(|v| json_values_equal(&doc_val, v)) {
return true;
}
+ // A declared unindexed text field has no analyzed token stream.
+ // With doc values, ES answers term-level queries from whole raw
+ // column values; do not manufacture a whitespace-token match.
+ // Undeclared fields retain the legacy fallback below.
+ if declared_field(schema, field)
+ .is_some_and(|f| matches!(f.field_type, FieldType::Text) && !f.options.indexed)
+ {
+ return false;
+ }
// Text-field fallback: if the doc has a string with whitespace
// (i.e. looks tokenized), treat each space-separated token as a
// candidate match. This mirrors ES's behavior of running `terms`
@@ -46185,6 +46241,50 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo
// default (standard/simple/lowercase etc.) lowercases both
// sides like the existing path.
let preserve_case = matches!(analyzer.as_deref(), Some("whitespace") | Some("keyword"));
+ let unindexed_doc_values_text = declared_field(schema, field).is_some_and(|f| {
+ matches!(f.field_type, FieldType::Text)
+ && !f.options.indexed
+ && f.options.doc_values
+ });
+
+ if unindexed_doc_values_text {
+ // ES analyzes only the query for this mapping, then compares
+ // its tokens against whole, case-sensitive doc-value strings.
+ // The field's `_source` value is never tokenized.
+ let q_tokens: Vec<String> = match analyzer.as_deref() {
+ Some("keyword") => vec![query.clone()],
+ Some("whitespace") => query.split_whitespace().map(str::to_string).collect(),
+ _ => phrase_tokens(query),
+ };
+ if q_tokens.is_empty() {
+ return false;
+ }
+ let need = match operator {
+ xerj_query::ast::BoolOperator::And => q_tokens.len(),
+ xerj_query::ast::BoolOperator::Or => match minimum_should_match {
+ None => 1,
+ Some(MinShouldMatch::Fixed(n)) => *n as usize,
+ Some(MinShouldMatch::Percentage(pct)) => {
+ ((q_tokens.len() as f32 * (*pct as f32 / 100.0)).floor() as usize)
+ .max(1)
+ }
+ Some(other) => other.required(q_tokens.len()).unwrap_or(1),
+ },
+ };
+ let raw_values: Vec<String> = match get_field_value(source, field) {
+ Some(Value::String(value)) => vec![value],
+ Some(Value::Array(values)) => values
+ .into_iter()
+ .filter_map(|value| value.as_str().map(str::to_string))
+ .collect(),
+ _ => Vec::new(),
+ };
+ let matched = q_tokens
+ .iter()
+ .filter(|token| raw_values.iter().any(|value| value == *token))
+ .count();
+ return matched >= need;
+ }
// When the query string parses as a date AND the field value
// is a date (or array of dates), switch to date-aware equality:
// the field value is coerced to epoch ms so
@@ -60310,6 +60410,18 @@ mod exact_scan_hydration_tests {
}
}
+ #[test]
+ fn terms_on_undeclared_strings_keeps_legacy_word_matching() {
+ let query = QueryNode::Terms {
+ field: "text".into(),
+ values: vec![Value::from("brown")],
+ boost: None,
+ };
+ let source = serde_json::json!({ "text": "quick brown fox" });
+
+ assert!(doc_matches_query_typed(&query, &source, &Schema::empty()));
+ }
+
/// The full-source plan IS the old behaviour: the whole source with `_id`
/// inserted over whatever the document carried under that key.
#[test]
diff --git a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs
index 4e9fdc41a..07820a680 100644
--- a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs
+++ b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs
@@ -151,20 +151,109 @@ async fn terms_on_unindexed_text_with_doc_values_stays_whole_value() {
schema.fields.push(text);
engine.create_index("dvn", schema).unwrap();
let idx: Arc<Index> = engine.get_index("dvn").unwrap();
- for (id, t) in [("d1", "foo bar"), ("d2", "foo bar"), ("d3", "baz qux")] {
- idx.index_document(Some(id.into()), json!({"text": t}))
+ for (id, value) in [
+ ("d1", json!("quick brown fox")),
+ ("d2", json!("lazy dog")),
+ ("d3", json!("fox")),
+ ("d4", json!("Fox")),
+ ("d5", json!(["fox", "lazy dog"])),
+ ] {
+ idx.index_document(Some(id.into()), json!({ "text": value }))
.await
.unwrap();
}
- idx.flush().await.unwrap();
- let whole = json!({"terms": {"text": ["foo bar", "baz qux"]}});
- assert_eq!(search(&idx, &whole).await.0, 3, "whole values match");
- let token = json!({"terms": {"text": ["foo"]}});
- assert_eq!(
- search(&idx, &token).await,
- (0, set(&[])),
- "unindexed doc-values text answers whole values, never tokens"
- );
- assert_eq!(count_total(&idx, &token).await, 0);
+ let cases = [
+ (
+ "match token absent",
+ json!({"match": {"text": "quick"}}),
+ set(&[]),
+ ),
+ (
+ "match whole token",
+ json!({"match": {"text": "fox"}}),
+ set(&["d3", "d5"]),
+ ),
+ (
+ "match default or",
+ json!({"match": {"text": "quick brown fox"}}),
+ set(&["d3", "d5"]),
+ ),
+ (
+ "match and",
+ json!({"match": {"text": {"query": "quick brown fox", "operator": "and"}}}),
+ set(&[]),
+ ),
+ (
+ "match keyword",
+ json!({"match": {"text": {"query": "quick brown fox", "analyzer": "keyword"}}}),
+ set(&["d1"]),
+ ),
+ (
+ "match uppercase",
+ json!({"match": {"text": "FOX"}}),
+ set(&["d3", "d5"]),
+ ),
+ (
+ "match whitespace",
+ json!({"match": {"text": {"query": "Fox", "analyzer": "whitespace"}}}),
+ set(&["d4"]),
+ ),
+ (
+ "match msm",
+ json!({"match": {"text": {"query": "fox lazy", "minimum_should_match": 2}}}),
+ set(&[]),
+ ),
+ (
+ "terms token absent",
+ json!({"terms": {"text": ["quick"]}}),
+ set(&[]),
+ ),
+ (
+ "terms whole",
+ json!({"terms": {"text": ["quick brown fox"]}}),
+ set(&["d1"]),
+ ),
+ (
+ "term whole",
+ json!({"term": {"text": "fox"}}),
+ set(&["d3", "d5"]),
+ ),
+ (
+ "bool filter",
+ json!({"bool": {"filter": [{"terms": {"text": ["quick"]}}]}}),
+ set(&[]),
+ ),
+ (
+ "bool must",
+ json!({"bool": {"must": [{"match": {"text": "fox"}}]}}),
+ set(&["d3", "d5"]),
+ ),
+ (
+ "constant score",
+ json!({"constant_score": {"filter": {"match": {"text": "fox"}}}}),
+ set(&["d3", "d5"]),
+ ),
+ (
+ "dis max",
+ json!({"dis_max": {"queries": [{"match": {"text": "fox"}}]}}),
+ set(&["d3", "d5"]),
+ ),
+ ];
+
+ for phase in ["before", "after"] {
+ for (name, query, expected) in &cases {
+ let (total, hits) = search(&idx, query).await;
+ assert_eq!(&hits, expected, "{phase} flush: {name}");
+ assert_eq!(total, expected.len() as u64, "{phase} flush total: {name}");
+ assert_eq!(
+ count_total(&idx, query).await,
+ expected.len() as u64,
+ "{phase} flush size:0 total: {name}"
+ );
+ }
+ if phase == "before" {
+ idx.flush().await.unwrap();
+ }
+ }
}