Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): inferred text fields take the ES dynamic-mapping doc-values default

Tong-bit-art2207328d02ba
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 9c031ac5d..6d720289f 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -26717,8 +26717,10 @@ impl Index {
///
/// Derived from the SCHEMA rather than from the documents, so it is stable
/// across flush and merge — a force-merge must not resurrect a column that
-/// flush correctly skipped. Dynamic (unmapped) fields are absent from the
-/// schema and keep today's behaviour of getting a column.
+/// flush correctly skipped. Inferred fields carry their own flag too
+/// (`dynamic_field_config*`: `Text` gets `false`, mirroring ES's dynamic
+/// mapping and the declared default), so they take this skip set rather than
+/// the builder's per-document default.
pub fn doc_values_skip_set(schema: &Schema) -> std::collections::HashSet<String> {
let mut skip = std::collections::HashSet::new();
for field in &schema.fields {
@@ -44153,6 +44155,19 @@ fn dynamic_field_config_at_depth(
let mut fc = FieldConfig::new(key.to_string(), ft);
match fc.field_type {
FieldType::Text => {
+ // The ES dynamic-mapping default for an inferred string's parent
+ // field is NO doc-values — only the `keyword` sub-field below
+ // carries them — and `es_properties_to_fields` already applies
+ // exactly that default to a DECLARED `text` mapping
+ // (`doc_values_default`). `FieldConfig::new` defaults the flag to
+ // true, so an INFERRED text field used to keep a doc-values column
+ // a declared one does not: the segment then answered a `terms`
+ // query from that column's whole raw values while the memtable
+ // scan answers per analyzed token, so the same query matched
+ // before `_refresh` and missed after it (and the `.dv` footprint
+ // rc.12's skip set removed for declared fields was re-paid for
+ // inferred ones).
+ fc.options.doc_values = false;
let mut kw = FieldConfig::new("keyword".to_string(), FieldType::Keyword);
kw.options.ignore_above = Some(DYNAMIC_KEYWORD_IGNORE_ABOVE);
fc.fields.push(kw);
diff --git a/engine/crates/xerj-engine/tests/dynamic_text_terms_flush_parity.rs b/engine/crates/xerj-engine/tests/dynamic_text_terms_flush_parity.rs
new file mode 100644
index 000000000..6e847b7b4
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/dynamic_text_terms_flush_parity.rs
@@ -0,0 +1,143 @@
+//! Regression tests for the dynamic-mapping `doc_values` default.
+//!
+//! ES's dynamic mapping gives an inferred string field a `text` parent with a
+//! `keyword` sub-field, and only the sub-field carries doc-values. The
+//! engine's `es_properties_to_fields` already applies that default to a
+//! DECLARED `text` mapping, but the dynamic-discovery path builds the parent
+//! through `FieldConfig::new`, whose `FieldOptions` default the flag to
+//! `true`. With the column present, the flushed segment answered a `terms`
+//! query from the column's whole raw values while the memtable scan answered
+//! per analyzed token — so the same query matched before `_refresh` and
+//! returned nothing after it.
+
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+/// What `POST /_search` runs: a real page of hits.
+async fn search(idx: &Index, q: &Value) -> (u64, BTreeSet<String>) {
+ let req = parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request");
+ let r = idx.search(&req).await.unwrap();
+ (r.total.value, r.hits.iter().map(|h| h.id.clone()).collect())
+}
+
+/// What `POST /_count` runs: `size: 0`, total only.
+async fn count_total(idx: &Index, q: &Value) -> u64 {
+ let req = parse_request(&json!({ "query": q, "size": 0, "from": 0 })).expect("parse_request");
+ idx.search(&req).await.unwrap().total.value
+}
+
+/// `_count` must answer the same question `_search` answers — the count lane
+/// takes a different shortcut on both sides of the flush.
+async fn assert_count_agrees_with_search(idx: &Index, q: &Value) {
+ let count = count_total(idx, q).await;
+ let (total, hits) = search(idx, q).await;
+ assert_eq!(
+ count, total,
+ "_count total {count} != _search total {total}"
+ );
+ assert_eq!(
+ count,
+ hits.len() as u64,
+ "_count says {count} but _search materialises {} document(s)",
+ hits.len()
+ );
+}
+
+fn set(ids: &[&str]) -> BTreeSet<String> {
+ ids.iter().map(|s| s.to_string()).collect()
+}
+
+/// The inferred schema mirrors ES's dynamic mapping: `text` parent with the
+/// flag OFF, `keyword` sub-field with its own doc-values, and a numeric
+/// sibling keeping theirs.
+#[tokio::test]
+async fn inferred_text_has_no_doc_values_but_its_keyword_subfield_and_numerics_do() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ engine.create_index("dyn_schema", Schema::empty()).unwrap();
+ let idx = engine.get_index("dyn_schema").unwrap();
+ idx.index_document(
+ Some("d1".into()),
+ json!({"text": "quick brown fox", "n": 42}),
+ )
+ .await
+ .unwrap();
+
+ let schema = idx.schema().await;
+ let text = schema
+ .fields
+ .iter()
+ .find(|f| f.name == "text")
+ .expect("text field");
+ assert!(matches!(text.field_type, FieldType::Text));
+ assert!(
+ !text.options.doc_values,
+ "ES dynamic mapping puts doc-values on the keyword sub-field, \
+ not on the text parent"
+ );
+ let kw = text
+ .fields
+ .iter()
+ .find(|f| f.name == "keyword")
+ .expect("keyword sub-field");
+ assert!(matches!(kw.field_type, FieldType::Keyword));
+ assert!(kw.options.doc_values);
+ assert_eq!(kw.options.ignore_above, Some(256));
+ let n = schema
+ .fields
+ .iter()
+ .find(|f| f.name == "n")
+ .expect("n field");
+ assert!(
+ n.options.doc_values,
+ "numeric doc-values default is unchanged"
+ );
+}
+
+/// A `terms` query on a dynamically mapped text field answers the analyzed
+/// tokens before and after the flush, with `_count` agreeing at both ends,
+/// and the `.keyword` sub-field keeps answering the whole value.
+#[tokio::test]
+async fn dynamic_text_terms_matches_tokens_flush_invariantly() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ engine.create_index("dyn_terms", Schema::empty()).unwrap();
+ let idx: Arc<Index> = engine.get_index("dyn_terms").unwrap();
+ for (id, text) in [("d1", "quick brown fox"), ("d2", "lazy dog")] {
+ idx.index_document(Some(id.into()), json!({"text": text}))
+ .await
+ .unwrap();
+ }
+
+ let token_query = json!({"terms": {"text": ["brown"]}});
+ let (pre_total, pre_hits) = search(&idx, &token_query).await;
+ assert_eq!(pre_hits, set(&["d1"]), "pre-flush (memtable) token match");
+ assert_eq!(pre_total, 1);
+ assert_count_agrees_with_search(&idx, &token_query).await;
+
+ idx.flush().await.unwrap();
+
+ let (post_total, post_hits) = search(&idx, &token_query).await;
+ assert_eq!(
+ post_hits, pre_hits,
+ "terms on a dynamically mapped text field changed at flush"
+ );
+ assert_eq!(post_total, pre_total, "hit total changed at flush");
+ assert_count_agrees_with_search(&idx, &token_query).await;
+
+ let keyword_query = json!({"term": {"text.keyword": "quick brown fox"}});
+ assert_eq!(search(&idx, &keyword_query).await.1, set(&["d1"]));
+}
diff --git a/engine/tests/es-compat-yaml/yaml/search/750_dynamic_text_terms_tokens.yml b/engine/tests/es-compat-yaml/yaml/search/750_dynamic_text_terms_tokens.yml
new file mode 100644
index 000000000..ea52e81ad
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/750_dynamic_text_terms_tokens.yml
@@ -0,0 +1,39 @@
+setup:
+ - do:
+ indices.create:
+ index: dynamic_text_terms
+ - do:
+ bulk:
+ body:
+ - '{"index": {"_index": "dynamic_text_terms", "_id": "1"}}'
+ - '{"text": "quick brown fox"}'
+ - '{"index": {"_index": "dynamic_text_terms", "_id": "2"}}'
+ - '{"text": "lazy dog"}'
+
+---
+"terms on a dynamically mapped text field matches analyzed tokens before refresh":
+ - do:
+ search:
+ index: dynamic_text_terms
+ body: { query: { terms: { text: [ "brown" ] } } }
+ - match: { hits.total.value: 1 }
+ - match: { hits.hits.0._id: "1" }
+
+---
+"terms on a dynamically mapped text field matches analyzed tokens after refresh":
+ - do:
+ indices.refresh: {}
+ - do:
+ search:
+ index: dynamic_text_terms
+ body: { query: { terms: { text: [ "brown" ] } } }
+ - match: { hits.total.value: 1 }
+ - match: { hits.hits.0._id: "1" }
+
+ # The keyword sub-field keeps answering the whole value.
+ - do:
+ search:
+ index: dynamic_text_terms
+ body: { query: { term: { text.keyword: "quick brown fox" } } }
+ - match: { hits.total.value: 1 }
+ - match: { hits.hits.0._id: "1" }