AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): keep the doc-values term prefilter for unindexed text; drop the non-ES YAML case
xerj-orga970f45a484c
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 292e888d8..7088e2433 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -41472,8 +41472,14 @@ fn declared_field<'a>(schema: &'a Schema, field: &str) -> Option<&'a FieldConfig/// (the term/terms prefilters, the doc-values count shortcuts) silently drops/// the token-only matches once the document flushes. Those paths must decline/// and let the stored scan answer.+///+/// Only INDEXED text qualifies. An `"index": false` text field with doc-values+/// has no token stream to match against: ES answers `term`/`terms` on it from+/// the column, by whole value (`aggregations/terms_text_docvalues.yml`), so the+/// column-derived set is exactly right there and must keep being used.fn field_is_analyzed_text(schema: &Schema, field: &str) -> bool {- declared_field(schema, field).is_some_and(|f| matches!(f.field_type, FieldType::Text))+ declared_field(schema, field)+ .is_some_and(|f| matches!(f.field_type, FieldType::Text) && f.options.indexed)}/// Does `q` carry a `term`/`terms` leaf naming an analyzed `text` field?diff --git a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rsindex e68f15cba..4e9fdc41a 100644--- a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs+++ b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs@@ -57,7 +57,9 @@ fn doc_values_text_schema() -> Schema {async fn terms_on_text_with_doc_values_is_flush_invariant() {let dir = TempDir::new().unwrap();let engine = make_engine(&dir);- engine.create_index("dvt", doc_values_text_schema()).unwrap();+ engine+ .create_index("dvt", doc_values_text_schema())+ .unwrap();let idx: Arc<Index> = engine.get_index("dvt").unwrap();for (id, t) in [("d1", "quick brown fox"),@@ -80,7 +82,9 @@ async fn terms_on_text_with_doc_values_is_flush_invariant() {assert_eq!(pre_total, 2);assert_eq!(count_total(&idx, &token_q).await, 2, "count before flush");- // The whole-value query both sides agree on.+ // The whole-value query both sides agree on. NOTE: this pins flush+ // invariance of XERJ's current answer, not ES semantics — ES's term-level+ // `terms` on indexed text would match no token "quick brown fox" (0 hits).let whole_q = json!({"terms": {"text": ["quick brown fox"]}});assert_eq!(search(&idx, &whole_q).await.1, set(&["d1", "d2"]));@@ -131,3 +135,36 @@ async fn terms_on_text_with_doc_values_is_flush_invariant() {"term on text stays whole-value");}++/// `"index": false` + doc-values text (the ES `mapper.text.doc_values` preview+/// pinned by `aggregations/terms_text_docvalues.yml`): there are no postings,+/// so ES answers `term`/`terms` from the doc-values column — WHOLE values only.+/// A token-only value must not match once the segment answers.+#[tokio::test]+async fn terms_on_unindexed_text_with_doc_values_stays_whole_value() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let mut schema = Schema::empty();+ let mut text = FieldConfig::new("text", FieldType::Text);+ text.options.doc_values = true;+ text.options.indexed = false;+ schema.fields.push(text);+ engine.create_index("dvn", schema).unwrap();+ let idx: Arc<Index> = engine.get_index("dvn").unwrap();+ for (id, t) in [("d1", "foo bar"), ("d2", "foo bar"), ("d3", "baz qux")] {+ idx.index_document(Some(id.into()), json!({"text": t}))+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();++ let whole = json!({"terms": {"text": ["foo bar", "baz qux"]}});+ assert_eq!(search(&idx, &whole).await.0, 3, "whole values match");+ let token = json!({"terms": {"text": ["foo"]}});+ assert_eq!(+ search(&idx, &token).await,+ (0, set(&[])),+ "unindexed doc-values text answers whole values, never tokens"+ );+ assert_eq!(count_total(&idx, &token).await, 0);+}diff --git a/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml b/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.ymlindex 4b89770e0..e8c415cec 100644--- a/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml+++ b/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml@@ -22,11 +22,3 @@ setup:index: text_dv_termsbody: { query: { terms: { text: [ "brown" ] } } }- match: { hits.total.value: 1 }------"terms on an analyzed text field with doc_values still matches the whole value":- - do:- search:- index: text_dv_terms- body: { query: { terms: { text: [ "quick brown fox" ] } } }- - match: { hits.total.value: 1 }