AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): #408 `term` on a keyword array is flush-invariant for counts
xerj-org1a0e4e2c9f8b
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex b55c7cc43..158eac6a9 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -21055,6 +21055,18 @@ impl Index {_ => return None,};+ // #423/#408: a keyword ARRAY (or whitespace keyword) stores only its+ // FIRST element in the memtable's single-valued count column (see+ // `push_field`), so counting `term` on any non-first element here would+ // undercount — a confident, wrong 0 while the doc is buffered, then N+ // after flush (size:0 = 0 while size:5 returned the doc). Abandon to the+ // stored-source scan, which matches array membership via+ // `doc_matches_query`. Mirrors the identical guard in+ // `doc_values_term_query` / `doc_values_bool_hits`.+ if self.memtable.term_count_needs_source_scan(field) {+ return None;+ }+// Memtable side — M5.1 sharded count via per-shard count maps.// Each shard lazily rebuilds its own count map on first query// after an ingest; the sharded `doc_values_*_count` helpersdiff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex b1f5098fb..257facdc3 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -1804,6 +1804,19 @@ impl ShardedFtsMemtable {})}+ /// True when the memtable's single-valued doc-values column for `field`+ /// cannot authoritatively answer a `term`/count query: the field has+ /// carried an ARRAY (only the first element is stored — see `push_field`)+ /// or a whitespace keyword (the column holds analyzed-text source). Callers+ /// that count via the column MUST fall back to the stored-source scan,+ /// which matches array membership. This is the same guard `doc_values_term_query`+ /// and `doc_values_bool_hits` already apply; exposed so the `try_shortcut_count`+ /// bare-`term` count path can bail identically (#423/#408: a `term` on a+ /// keyword array counted 0 while buffered but N after flush).+ pub fn term_count_needs_source_scan(&self, field: &str) -> bool {+ self.dv_column_unusable(field)+ }+/// DocValues term query — aggregates hits across all shards./// Returns `Some(Vec<(doc_id, local_idx)>)` if any shard matched./// The `local_idx` is shard-local; callers use the doc_id todiff --git a/engine/crates/xerj-engine/tests/term_on_keyword_array_flush_parity.rs b/engine/crates/xerj-engine/tests/term_on_keyword_array_flush_parity.rsnew file mode 100644index 000000000..38d3eb3aa--- /dev/null+++ b/engine/crates/xerj-engine/tests/term_on_keyword_array_flush_parity.rs@@ -0,0 +1,81 @@+//! Issue #423 (symptom #408): a `term` query on a `keyword` ARRAY field must+//! return the same hit set before and after a flush. A document with+//! `"tags": ["red", "green", "blue"]` answers `term: {tags: "green"}` with 1+//! hit once flushed, but 0 hits while still buffered — only the FIRST array+//! element ("red") matches pre-flush. Same document, same query, two answers,+//! decided by a background flush the caller cannot see.+//!+//! ROOT CAUSE (confirmed): NOT `doc_matches_query` — `json_values_equal` already+//! matches any array element, and `size:5` returned the doc correctly. The miss+//! was in the size:0 COUNT path: `try_shortcut_count`'s bare-`term` keyword+//! shortcut counted the memtable via `doc_values_keyword_count`, whose+//! single-valued column stores only the FIRST array element (see `push_field`),+//! and — unlike `doc_values_term_query` / `doc_values_bool_hits` — it lacked the+//! array bailout. It returned a confident `Some(0)` that overwrote the correct+//! scan count. The fix adds that bailout (`term_count_needs_source_scan`) so the+//! shortcut abandons to the stored-source scan for array/whitespace fields.++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::Engine;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++async fn term_tag_hits(engine: &Engine, tag: &str) -> u64 {+ let idx = engine.get_index("docs").expect("get index");+ let req = parse_request(&json!({+ "query": { "term": { "tags": tag } },+ "size": 0,+ "from": 0+ }))+ .expect("parse_request");+ idx.search(&req).await.expect("search").total.value+}++/// `term` on a `keyword` array is flush-invariant: every element is matchable+/// whether the document is buffered or flushed. FAIL-BEFORE: the second array+/// element ("green") matches only after flush, so pre-flush=0, post-flush=1.+#[tokio::test]+async fn term_on_keyword_array_is_flush_invariant() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("tags", FieldType::Keyword));+ engine.create_index("docs", schema).expect("create");++ let idx = engine.get_index("docs").expect("get");+ idx.index_document(+ Some("d0".to_string()),+ json!({ "tags": ["red", "green", "blue"] }),+ )+ .await+ .expect("index");++ // Second element, document still buffered (no refresh): pre-flush answer.+ let pre = term_tag_hits(&engine, "green").await;++ // Flush to segments: post-flush answer.+ idx.refresh().await.expect("refresh");+ let post = term_tag_hits(&engine, "green").await;++ assert_eq!(+ pre, post,+ "#423/#408: `term: green` on `[\"red\",\"green\",\"blue\"]` must be \+ flush-invariant (pre-flush={pre}, post-flush={post}); the memtable term \+ path is matching only the first array element"+ );+ assert_eq!(+ post, 1,+ "sanity: the flushed segment matches the array element"+ );+}