AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): inferred text fields take the ES dynamic-mapping doc-values default
Tong-bit-art2207328d02ba
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 9c031ac5d..6d720289f 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -26717,8 +26717,10 @@ impl Index {////// Derived from the SCHEMA rather than from the documents, so it is stable/// across flush and merge — a force-merge must not resurrect a column that-/// flush correctly skipped. Dynamic (unmapped) fields are absent from the-/// schema and keep today's behaviour of getting a column.+/// flush correctly skipped. Inferred fields carry their own flag too+/// (`dynamic_field_config*`: `Text` gets `false`, mirroring ES's dynamic+/// mapping and the declared default), so they take this skip set rather than+/// the builder's per-document default.pub fn doc_values_skip_set(schema: &Schema) -> std::collections::HashSet<String> {let mut skip = std::collections::HashSet::new();for field in &schema.fields {@@ -44153,6 +44155,19 @@ fn dynamic_field_config_at_depth(let mut fc = FieldConfig::new(key.to_string(), ft);match fc.field_type {FieldType::Text => {+ // The ES dynamic-mapping default for an inferred string's parent+ // field is NO doc-values — only the `keyword` sub-field below+ // carries them — and `es_properties_to_fields` already applies+ // exactly that default to a DECLARED `text` mapping+ // (`doc_values_default`). `FieldConfig::new` defaults the flag to+ // true, so an INFERRED text field used to keep a doc-values column+ // a declared one does not: the segment then answered a `terms`+ // query from that column's whole raw values while the memtable+ // scan answers per analyzed token, so the same query matched+ // before `_refresh` and missed after it (and the `.dv` footprint+ // rc.12's skip set removed for declared fields was re-paid for+ // inferred ones).+ fc.options.doc_values = false;let mut kw = FieldConfig::new("keyword".to_string(), FieldType::Keyword);kw.options.ignore_above = Some(DYNAMIC_KEYWORD_IGNORE_ABOVE);fc.fields.push(kw);diff --git a/engine/crates/xerj-engine/tests/dynamic_text_terms_flush_parity.rs b/engine/crates/xerj-engine/tests/dynamic_text_terms_flush_parity.rsnew file mode 100644index 000000000..6e847b7b4--- /dev/null+++ b/engine/crates/xerj-engine/tests/dynamic_text_terms_flush_parity.rs@@ -0,0 +1,143 @@+//! Regression tests for the dynamic-mapping `doc_values` default.+//!+//! ES's dynamic mapping gives an inferred string field a `text` parent with a+//! `keyword` sub-field, and only the sub-field carries doc-values. The+//! engine's `es_properties_to_fields` already applies that default to a+//! DECLARED `text` mapping, but the dynamic-discovery path builds the parent+//! through `FieldConfig::new`, whose `FieldOptions` default the flag to+//! `true`. With the column present, the flushed segment answered a `terms`+//! query from the column's whole raw values while the memtable scan answered+//! per analyzed token — so the same query matched before `_refresh` and+//! returned nothing after it.++use std::collections::BTreeSet;+use std::sync::Arc;++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++/// What `POST /_search` runs: a real page of hits.+async fn search(idx: &Index, q: &Value) -> (u64, BTreeSet<String>) {+ let req = parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request");+ let r = idx.search(&req).await.unwrap();+ (r.total.value, r.hits.iter().map(|h| h.id.clone()).collect())+}++/// What `POST /_count` runs: `size: 0`, total only.+async fn count_total(idx: &Index, q: &Value) -> u64 {+ let req = parse_request(&json!({ "query": q, "size": 0, "from": 0 })).expect("parse_request");+ idx.search(&req).await.unwrap().total.value+}++/// `_count` must answer the same question `_search` answers — the count lane+/// takes a different shortcut on both sides of the flush.+async fn assert_count_agrees_with_search(idx: &Index, q: &Value) {+ let count = count_total(idx, q).await;+ let (total, hits) = search(idx, q).await;+ assert_eq!(+ count, total,+ "_count total {count} != _search total {total}"+ );+ assert_eq!(+ count,+ hits.len() as u64,+ "_count says {count} but _search materialises {} document(s)",+ hits.len()+ );+}++fn set(ids: &[&str]) -> BTreeSet<String> {+ ids.iter().map(|s| s.to_string()).collect()+}++/// The inferred schema mirrors ES's dynamic mapping: `text` parent with the+/// flag OFF, `keyword` sub-field with its own doc-values, and a numeric+/// sibling keeping theirs.+#[tokio::test]+async fn inferred_text_has_no_doc_values_but_its_keyword_subfield_and_numerics_do() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("dyn_schema", Schema::empty()).unwrap();+ let idx = engine.get_index("dyn_schema").unwrap();+ idx.index_document(+ Some("d1".into()),+ json!({"text": "quick brown fox", "n": 42}),+ )+ .await+ .unwrap();++ let schema = idx.schema().await;+ let text = schema+ .fields+ .iter()+ .find(|f| f.name == "text")+ .expect("text field");+ assert!(matches!(text.field_type, FieldType::Text));+ assert!(+ !text.options.doc_values,+ "ES dynamic mapping puts doc-values on the keyword sub-field, \+ not on the text parent"+ );+ let kw = text+ .fields+ .iter()+ .find(|f| f.name == "keyword")+ .expect("keyword sub-field");+ assert!(matches!(kw.field_type, FieldType::Keyword));+ assert!(kw.options.doc_values);+ assert_eq!(kw.options.ignore_above, Some(256));+ let n = schema+ .fields+ .iter()+ .find(|f| f.name == "n")+ .expect("n field");+ assert!(+ n.options.doc_values,+ "numeric doc-values default is unchanged"+ );+}++/// A `terms` query on a dynamically mapped text field answers the analyzed+/// tokens before and after the flush, with `_count` agreeing at both ends,+/// and the `.keyword` sub-field keeps answering the whole value.+#[tokio::test]+async fn dynamic_text_terms_matches_tokens_flush_invariantly() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ engine.create_index("dyn_terms", Schema::empty()).unwrap();+ let idx: Arc<Index> = engine.get_index("dyn_terms").unwrap();+ for (id, text) in [("d1", "quick brown fox"), ("d2", "lazy dog")] {+ idx.index_document(Some(id.into()), json!({"text": text}))+ .await+ .unwrap();+ }++ let token_query = json!({"terms": {"text": ["brown"]}});+ let (pre_total, pre_hits) = search(&idx, &token_query).await;+ assert_eq!(pre_hits, set(&["d1"]), "pre-flush (memtable) token match");+ assert_eq!(pre_total, 1);+ assert_count_agrees_with_search(&idx, &token_query).await;++ idx.flush().await.unwrap();++ let (post_total, post_hits) = search(&idx, &token_query).await;+ assert_eq!(+ post_hits, pre_hits,+ "terms on a dynamically mapped text field changed at flush"+ );+ assert_eq!(post_total, pre_total, "hit total changed at flush");+ assert_count_agrees_with_search(&idx, &token_query).await;++ let keyword_query = json!({"term": {"text.keyword": "quick brown fox"}});+ assert_eq!(search(&idx, &keyword_query).await.1, set(&["d1"]));+}diff --git a/engine/tests/es-compat-yaml/yaml/search/750_dynamic_text_terms_tokens.yml b/engine/tests/es-compat-yaml/yaml/search/750_dynamic_text_terms_tokens.ymlnew file mode 100644index 000000000..ea52e81ad--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/search/750_dynamic_text_terms_tokens.yml@@ -0,0 +1,39 @@+setup:+ - do:+ indices.create:+ index: dynamic_text_terms+ - do:+ bulk:+ body:+ - '{"index": {"_index": "dynamic_text_terms", "_id": "1"}}'+ - '{"text": "quick brown fox"}'+ - '{"index": {"_index": "dynamic_text_terms", "_id": "2"}}'+ - '{"text": "lazy dog"}'++---+"terms on a dynamically mapped text field matches analyzed tokens before refresh":+ - do:+ search:+ index: dynamic_text_terms+ body: { query: { terms: { text: [ "brown" ] } } }+ - match: { hits.total.value: 1 }+ - match: { hits.hits.0._id: "1" }++---+"terms on a dynamically mapped text field matches analyzed tokens after refresh":+ - do:+ indices.refresh: {}+ - do:+ search:+ index: dynamic_text_terms+ body: { query: { terms: { text: [ "brown" ] } } }+ - match: { hits.total.value: 1 }+ - match: { hits.hits.0._id: "1" }++ # The keyword sub-field keeps answering the whole value.+ - do:+ search:+ index: dynamic_text_terms+ body: { query: { term: { text.keyword: "quick brown fox" } } }+ - match: { hits.total.value: 1 }+ - match: { hits.hits.0._id: "1" }