AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): keep doc-values text flush-invariant (#1517)
Lenin Mookiah2e495f0c5f56
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 757b8248f..6adf9604e 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -21537,6 +21537,8 @@ impl Index {.clone()};+ let unindexed_dv_text_match = has_unindexed_doc_values_text_match(query, &dmq_schema);+let mem_snapshot = {let mem = &*self.memtable;if mem_doc_count == 0 {@@ -21631,6 +21633,12 @@ impl Index {let extra = total.saturating_sub(ids.len() as u64);MemSnapshot::AllDocIds(ids, extra)}+ } else if unindexed_dv_text_match {+ // These fields have no memtable postings. Route direct Match,+ // normalized bool-Match, and DisMax-of-Match shapes through+ // the mapping-aware scan so pre-flush answers use the same+ // whole-value column semantics as flushed documents.+ MemSnapshot::DocsForScan(mem.all_docs_with_seq_arc())} else if let Some((dm_subs, dm_tie)) = dis_max_mem_fts_subs(query) {// #834: a `dis_max` of single-field text `match` subs used// to fall through to the stored-doc scan, whose scorer has@@ -41527,6 +41535,45 @@ fn field_is_analyzed_text(schema: &Schema, field: &str) -> bool {.is_some_and(|f| matches!(f.field_type, FieldType::Text) && f.options.indexed)}+/// Does `q` contain a `match` leaf for declared doc-values-only text?+fn has_unindexed_doc_values_text_match(q: &QueryNode, schema: &Schema) -> bool {+ match q {+ QueryNode::Match { field, .. } => declared_field(schema, field).is_some_and(|f| {+ matches!(f.field_type, FieldType::Text) && !f.options.indexed && f.options.doc_values+ }),+ QueryNode::Bool {+ must,+ should,+ must_not,+ filter,+ ..+ } => must+ .iter()+ .chain(should)+ .chain(must_not)+ .chain(filter)+ .any(|child| has_unindexed_doc_values_text_match(child, schema)),+ QueryNode::DisMax { queries, .. } => queries+ .iter()+ .any(|child| has_unindexed_doc_values_text_match(child, schema)),+ QueryNode::Constant { query, .. }+ | QueryNode::Boosted { query, .. }+ | QueryNode::Named { query, .. }+ | QueryNode::Nested { query, .. }+ | QueryNode::FunctionScore { query, .. } => {+ has_unindexed_doc_values_text_match(query, schema)+ }+ QueryNode::Pinned { organic, .. } => has_unindexed_doc_values_text_match(organic, schema),+ QueryNode::Boosting {+ positive, negative, ..+ } => {+ has_unindexed_doc_values_text_match(positive, schema)+ || has_unindexed_doc_values_text_match(negative, schema)+ }+ _ => false,+ }+}+/// Does `q` carry a `term`/`terms` leaf naming an analyzed `text` field?////// The doc-values shapes [`field_is_analyzed_text`] cannot resolve are@@ -45738,6 +45785,15 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> boif values.iter().any(|v| json_values_equal(&doc_val, v)) {return true;}+ // A declared unindexed text field has no analyzed token stream.+ // With doc values, ES answers term-level queries from whole raw+ // column values; do not manufacture a whitespace-token match.+ // Undeclared fields retain the legacy fallback below.+ if declared_field(schema, field)+ .is_some_and(|f| matches!(f.field_type, FieldType::Text) && !f.options.indexed)+ {+ return false;+ }// Text-field fallback: if the doc has a string with whitespace// (i.e. looks tokenized), treat each space-separated token as a// candidate match. This mirrors ES's behavior of running `terms`@@ -46185,6 +46241,50 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo// default (standard/simple/lowercase etc.) lowercases both// sides like the existing path.let preserve_case = matches!(analyzer.as_deref(), Some("whitespace") | Some("keyword"));+ let unindexed_doc_values_text = declared_field(schema, field).is_some_and(|f| {+ matches!(f.field_type, FieldType::Text)+ && !f.options.indexed+ && f.options.doc_values+ });++ if unindexed_doc_values_text {+ // ES analyzes only the query for this mapping, then compares+ // its tokens against whole, case-sensitive doc-value strings.+ // The field's `_source` value is never tokenized.+ let q_tokens: Vec<String> = match analyzer.as_deref() {+ Some("keyword") => vec![query.clone()],+ Some("whitespace") => query.split_whitespace().map(str::to_string).collect(),+ _ => phrase_tokens(query),+ };+ if q_tokens.is_empty() {+ return false;+ }+ let need = match operator {+ xerj_query::ast::BoolOperator::And => q_tokens.len(),+ xerj_query::ast::BoolOperator::Or => match minimum_should_match {+ None => 1,+ Some(MinShouldMatch::Fixed(n)) => *n as usize,+ Some(MinShouldMatch::Percentage(pct)) => {+ ((q_tokens.len() as f32 * (*pct as f32 / 100.0)).floor() as usize)+ .max(1)+ }+ Some(other) => other.required(q_tokens.len()).unwrap_or(1),+ },+ };+ let raw_values: Vec<String> = match get_field_value(source, field) {+ Some(Value::String(value)) => vec![value],+ Some(Value::Array(values)) => values+ .into_iter()+ .filter_map(|value| value.as_str().map(str::to_string))+ .collect(),+ _ => Vec::new(),+ };+ let matched = q_tokens+ .iter()+ .filter(|token| raw_values.iter().any(|value| value == *token))+ .count();+ return matched >= need;+ }// When the query string parses as a date AND the field value// is a date (or array of dates), switch to date-aware equality:// the field value is coerced to epoch ms so@@ -60310,6 +60410,18 @@ mod exact_scan_hydration_tests {}}+ #[test]+ fn terms_on_undeclared_strings_keeps_legacy_word_matching() {+ let query = QueryNode::Terms {+ field: "text".into(),+ values: vec![Value::from("brown")],+ boost: None,+ };+ let source = serde_json::json!({ "text": "quick brown fox" });++ assert!(doc_matches_query_typed(&query, &source, &Schema::empty()));+ }+/// The full-source plan IS the old behaviour: the whole source with `_id`/// inserted over whatever the document carried under that key.#[test]diff --git a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rsindex 4e9fdc41a..07820a680 100644--- a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs+++ b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs@@ -151,20 +151,109 @@ async fn terms_on_unindexed_text_with_doc_values_stays_whole_value() {schema.fields.push(text);engine.create_index("dvn", schema).unwrap();let idx: Arc<Index> = engine.get_index("dvn").unwrap();- for (id, t) in [("d1", "foo bar"), ("d2", "foo bar"), ("d3", "baz qux")] {- idx.index_document(Some(id.into()), json!({"text": t}))+ for (id, value) in [+ ("d1", json!("quick brown fox")),+ ("d2", json!("lazy dog")),+ ("d3", json!("fox")),+ ("d4", json!("Fox")),+ ("d5", json!(["fox", "lazy dog"])),+ ] {+ idx.index_document(Some(id.into()), json!({ "text": value })).await.unwrap();}- idx.flush().await.unwrap();- let whole = json!({"terms": {"text": ["foo bar", "baz qux"]}});- assert_eq!(search(&idx, &whole).await.0, 3, "whole values match");- let token = json!({"terms": {"text": ["foo"]}});- assert_eq!(- search(&idx, &token).await,- (0, set(&[])),- "unindexed doc-values text answers whole values, never tokens"- );- assert_eq!(count_total(&idx, &token).await, 0);+ let cases = [+ (+ "match token absent",+ json!({"match": {"text": "quick"}}),+ set(&[]),+ ),+ (+ "match whole token",+ json!({"match": {"text": "fox"}}),+ set(&["d3", "d5"]),+ ),+ (+ "match default or",+ json!({"match": {"text": "quick brown fox"}}),+ set(&["d3", "d5"]),+ ),+ (+ "match and",+ json!({"match": {"text": {"query": "quick brown fox", "operator": "and"}}}),+ set(&[]),+ ),+ (+ "match keyword",+ json!({"match": {"text": {"query": "quick brown fox", "analyzer": "keyword"}}}),+ set(&["d1"]),+ ),+ (+ "match uppercase",+ json!({"match": {"text": "FOX"}}),+ set(&["d3", "d5"]),+ ),+ (+ "match whitespace",+ json!({"match": {"text": {"query": "Fox", "analyzer": "whitespace"}}}),+ set(&["d4"]),+ ),+ (+ "match msm",+ json!({"match": {"text": {"query": "fox lazy", "minimum_should_match": 2}}}),+ set(&[]),+ ),+ (+ "terms token absent",+ json!({"terms": {"text": ["quick"]}}),+ set(&[]),+ ),+ (+ "terms whole",+ json!({"terms": {"text": ["quick brown fox"]}}),+ set(&["d1"]),+ ),+ (+ "term whole",+ json!({"term": {"text": "fox"}}),+ set(&["d3", "d5"]),+ ),+ (+ "bool filter",+ json!({"bool": {"filter": [{"terms": {"text": ["quick"]}}]}}),+ set(&[]),+ ),+ (+ "bool must",+ json!({"bool": {"must": [{"match": {"text": "fox"}}]}}),+ set(&["d3", "d5"]),+ ),+ (+ "constant score",+ json!({"constant_score": {"filter": {"match": {"text": "fox"}}}}),+ set(&["d3", "d5"]),+ ),+ (+ "dis max",+ json!({"dis_max": {"queries": [{"match": {"text": "fox"}}]}}),+ set(&["d3", "d5"]),+ ),+ ];++ for phase in ["before", "after"] {+ for (name, query, expected) in &cases {+ let (total, hits) = search(&idx, query).await;+ assert_eq!(&hits, expected, "{phase} flush: {name}");+ assert_eq!(total, expected.len() as u64, "{phase} flush total: {name}");+ assert_eq!(+ count_total(&idx, query).await,+ expected.len() as u64,+ "{phase} flush size:0 total: {name}"+ );+ }+ if phase == "before" {+ idx.flush().await.unwrap();+ }+ }}