Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): keep the doc-values term prefilter for unindexed text; drop the non-ES YAML case

xerj-orga970f45a484c
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 292e888d8..7088e2433 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -41472,8 +41472,14 @@ fn declared_field<'a>(schema: &'a Schema, field: &str) -> Option<&'a FieldConfig
/// (the term/terms prefilters, the doc-values count shortcuts) silently drops
/// the token-only matches once the document flushes. Those paths must decline
/// and let the stored scan answer.
+///
+/// Only INDEXED text qualifies. An `"index": false` text field with doc-values
+/// has no token stream to match against: ES answers `term`/`terms` on it from
+/// the column, by whole value (`aggregations/terms_text_docvalues.yml`), so the
+/// column-derived set is exactly right there and must keep being used.
fn field_is_analyzed_text(schema: &Schema, field: &str) -> bool {
- declared_field(schema, field).is_some_and(|f| matches!(f.field_type, FieldType::Text))
+ declared_field(schema, field)
+ .is_some_and(|f| matches!(f.field_type, FieldType::Text) && f.options.indexed)
}
/// Does `q` carry a `term`/`terms` leaf naming an analyzed `text` field?
diff --git a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs
index e68f15cba..4e9fdc41a 100644
--- a/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs
+++ b/engine/crates/xerj-engine/tests/text_doc_values_terms_flush_parity.rs
@@ -57,7 +57,9 @@ fn doc_values_text_schema() -> Schema {
async fn terms_on_text_with_doc_values_is_flush_invariant() {
let dir = TempDir::new().unwrap();
let engine = make_engine(&dir);
- engine.create_index("dvt", doc_values_text_schema()).unwrap();
+ engine
+ .create_index("dvt", doc_values_text_schema())
+ .unwrap();
let idx: Arc<Index> = engine.get_index("dvt").unwrap();
for (id, t) in [
("d1", "quick brown fox"),
@@ -80,7 +82,9 @@ async fn terms_on_text_with_doc_values_is_flush_invariant() {
assert_eq!(pre_total, 2);
assert_eq!(count_total(&idx, &token_q).await, 2, "count before flush");
- // The whole-value query both sides agree on.
+ // The whole-value query both sides agree on. NOTE: this pins flush
+ // invariance of XERJ's current answer, not ES semantics — ES's term-level
+ // `terms` on indexed text would match no token "quick brown fox" (0 hits).
let whole_q = json!({"terms": {"text": ["quick brown fox"]}});
assert_eq!(search(&idx, &whole_q).await.1, set(&["d1", "d2"]));
@@ -131,3 +135,36 @@ async fn terms_on_text_with_doc_values_is_flush_invariant() {
"term on text stays whole-value"
);
}
+
+/// `"index": false` + doc-values text (the ES `mapper.text.doc_values` preview
+/// pinned by `aggregations/terms_text_docvalues.yml`): there are no postings,
+/// so ES answers `term`/`terms` from the doc-values column — WHOLE values only.
+/// A token-only value must not match once the segment answers.
+#[tokio::test]
+async fn terms_on_unindexed_text_with_doc_values_stays_whole_value() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let mut schema = Schema::empty();
+ let mut text = FieldConfig::new("text", FieldType::Text);
+ text.options.doc_values = true;
+ text.options.indexed = false;
+ schema.fields.push(text);
+ engine.create_index("dvn", schema).unwrap();
+ let idx: Arc<Index> = engine.get_index("dvn").unwrap();
+ for (id, t) in [("d1", "foo bar"), ("d2", "foo bar"), ("d3", "baz qux")] {
+ idx.index_document(Some(id.into()), json!({"text": t}))
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+
+ let whole = json!({"terms": {"text": ["foo bar", "baz qux"]}});
+ assert_eq!(search(&idx, &whole).await.0, 3, "whole values match");
+ let token = json!({"terms": {"text": ["foo"]}});
+ assert_eq!(
+ search(&idx, &token).await,
+ (0, set(&[])),
+ "unindexed doc-values text answers whole values, never tokens"
+ );
+ assert_eq!(count_total(&idx, &token).await, 0);
+}
diff --git a/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml b/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml
index 4b89770e0..e8c415cec 100644
--- a/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml
+++ b/engine/tests/es-compat-yaml/yaml/search/760_text_doc_values_terms_after_refresh.yml
@@ -22,11 +22,3 @@ setup:
index: text_dv_terms
body: { query: { terms: { text: [ "brown" ] } } }
- match: { hits.total.value: 1 }
-
----
-"terms on an analyzed text field with doc_values still matches the whole value":
- - do:
- search:
- index: text_dv_terms
- body: { query: { terms: { text: [ "quick brown fox" ] } } }
- - match: { hits.total.value: 1 }