AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
wip(fts): index each value of a multi-valued keyword field (#332)
xerj-orgb1376e4e8bbc
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 20aa8611d..7ce8d25f1 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -8837,7 +8837,11 @@ impl Index {// for the doc-values pass. Drops `dv_sources` entirely.// Halves merge working memory and frees the Arc<Value>// immediately after both passes complete.- let mut fts_input: Vec<(String, HashMap<String, String>, Value)> = Vec::new();+ let mut fts_input: Vec<(+ String,+ HashMap<String, xerj_fts::index::FieldValues>,+ Value,+ )> = Vec::new();// (seq_no, doc_id) for every surviving doc — kept exactly// aligned with the stored-section byte copy (pushed right// after `live_doc_count += 1`, BEFORE the per-doc Value@@ -19407,53 +19411,27 @@ fn build_fts_field_configs(schema: &Schema) -> HashMap<String, xerj_fts::index::out}-/// Mirror of `FtsMemtable::extract_text_value` — converts any JSON value-/// into a flattened string suitable for tokenization, so the merge path-/// indexes fields the same way the memtable does. Kept as a free function-/// here because the memtable's helper is private.-fn extract_field_text(val: &Value) -> String {- match val {- Value::String(s) => s.clone(),- Value::Number(n) => n.to_string(),- Value::Bool(b) => b.to_string(),- Value::Array(arr) => arr- .iter()- .map(extract_field_text)- .collect::<Vec<_>>()- .join(" "),- Value::Object(_) => serde_json::to_string(val).unwrap_or_default(),- Value::Null => String::new(),- }-}-+/// FTS input extraction for the MERGE path.+///+/// Delegates to the flush path's own walker+/// (`crate::memtable::extract_field_values_excluding`) rather than mirroring+/// it. Until #332 this file carried a private copy (`extract_field_text` ++/// `extract_fts_fields_excluding`) that had to be kept in sync by hand with+/// `memtable::extract_text_value`; both copies flattened a JSON array with+/// `join(" ")`, and a merged segment reproduced the bug even after a flushed+/// one stopped having it. One walker, one behaviour.+///+/// The shared walker also carries #328's nested-exclusion pruning: an+/// object-valued field with an excluded descendant (a `dense_vector` at+/// `passages.vec`) is flattened from a PRUNED copy, so the merge does not+/// resurrect postings the flush path skipped. That pruning used to live inline+/// here; it now lives once, in `extract_field_values_excluding`, so both the+/// flush and the merge path get it.fn extract_fts_fields_excluding(source: &Value,excluded: &std::collections::HashSet<String>,-) -> HashMap<String, String> {- let mut fields = HashMap::new();- if let Some(obj) = source.as_object() {- for (key, val) in obj {- if excluded.contains(key) {- continue;- }- // This is the MERGE path, and a merge must not resurrect postings- // that flush correctly skipped (#328). A root key is caught above;- // an object-valued key is flattened whole by `extract_field_text`,- // so a `dense_vector` nested under it (`passages.vec`) would come- // back here even though flush dropped it. Prune it out of the- // flattened blob exactly the way `collect_text_fields` does.- let text = if val.is_object() && crate::memtable::has_excluded_descendant(key, excluded)- {- crate::memtable::extract_text_value_excluding(val, key, excluded)- } else {- extract_field_text(val)- };- if !text.is_empty() {- fields.insert(key.clone(), text);- }- }- }- fields+) -> HashMap<String, xerj_fts::index::FieldValues> {+ crate::memtable::extract_field_values_excluding(source, excluded)}// ── Short-circuit count helper ───────────────────────────────────────────────@@ -24513,7 +24491,7 @@ async fn do_flush_shard(let drained_opt: Option<(Vec<(String,- std::collections::HashMap<String, String>,+ std::collections::HashMap<String, xerj_fts::index::FieldValues>,std::sync::Arc<serde_json::Value>,)>,xerj_storage::index_store::DrainedMemtable,@@ -24576,7 +24554,7 @@ async fn do_flush_shard(} else {std::sync::Arc::new(serde_json::Value::Null)};- let fields = crate::memtable::extract_text_fields_from_excluding(+ let fields = crate::memtable::extract_field_values_excluding(&val,&excluded_fts_fields,);@@ -24588,7 +24566,7 @@ async fn do_flush_shard(// doesn't queue ahead of concurrent search/agg par_iters.let drained_fts: Vec<(String,- std::collections::HashMap<String, String>,+ std::collections::HashMap<String, xerj_fts::index::FieldValues>,std::sync::Arc<serde_json::Value>,)> = if prep_offload {crate::background_pool().install(build_drained_fts)@@ -33180,18 +33158,19 @@ fn phrase_prefix_positions_in_tokens(/// ones, so a split query and an analyzed segment clause ask different/// questions and the hit set changes at `_flush`.///-/// `field_text_lc` is the field's text as the rest of the `multi_match` arms-/// see it — an array-valued field arrives already joined with a space.-/// That means a phrase CAN span two array elements, which ES prevents with a-/// `position_increment_gap`. XERJ has no such gap anywhere: the indexing-/// path flattens an array to one space-joined string before the FTS layer-/// ever sees it (`extract_field_text`), so the segment's positions have no-/// gap either. Evaluating per element here would therefore FIX the memtable-/// and leave the segment as it is — reintroducing the flush-variant hit set-/// this fix exists to remove. The gap belongs in the indexing path; it is-/// deliberately not attempted here.+/// `field_values_lc` is the field's values, ONE ENTRY PER ARRAY ELEMENT, and+/// the phrase must be satisfied inside a single element — never across two.+///+/// This used to take one already-space-joined string, so a phrase COULD span+/// two array elements. That was deliberate at the time: the indexing path+/// flattened an array to one space-joined string before the FTS layer saw it,+/// so the segment's positions had no gap either, and evaluating per element+/// here would have fixed only the memtable and left the hit set flush-variant.+/// #332 put the gap where it belongs — `xerj_fts::index::POSITION_INCREMENT_GAP`+/// now separates array elements in the segment postings, matching Lucene — so+/// per-element evaluation here is what KEEPS the two sides equal.fn multi_match_phrase_hit(- field_text_lc: &str,+ field_values_lc: &[String],query_tokens: &[String],is_prefix: bool,slop: u32,@@ -33199,11 +33178,42 @@ fn multi_match_phrase_hit(if query_tokens.is_empty() {return true;}- let field_tokens = phrase_tokens(field_text_lc);- if is_prefix {- phrase_prefix_positions_in_tokens(&field_tokens, query_tokens, slop)- } else {- phrase_positions_in_tokens(&field_tokens, query_tokens, slop)+ field_values_lc.iter().any(|value| {+ let field_tokens = phrase_tokens(value);+ if is_prefix {+ phrase_prefix_positions_in_tokens(&field_tokens, query_tokens, slop)+ } else {+ phrase_positions_in_tokens(&field_tokens, query_tokens, slop)+ }+ })+}++/// Per-element lowercased values of `field` in one stored source document, as+/// the `multi_match` arms need them.+///+/// An array yields one entry per element (#332); a scalar yields a single+/// entry. The non-phrase arms join the result back with a space, which is+/// byte-identical to what they extracted before; the phrase arms consume the+/// elements separately so a phrase cannot straddle the boundary.+fn multi_match_field_values_lc(source: &Value, field: &str) -> Option<Vec<String>> {+ match get_field_value(source, field) {+ Some(Value::String(s)) => Some(vec![s.to_lowercase()]),+ Some(Value::Array(arr)) => {+ let values: Vec<String> = arr+ .iter()+ .map(|v| match v {+ Value::String(s) => s.to_lowercase(),+ other => other.to_string(),+ })+ .collect();+ if values.is_empty() {+ None+ } else {+ Some(values)+ }+ }+ Some(other) => Some(vec![other.to_string().to_lowercase()]),+ None => None,}}@@ -33767,31 +33777,22 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {} else {Vec::new()};- let field_texts: Vec<String> = fields+ // Per-field values, one entry per array element (#332). The+ // non-phrase arms below join them back with a space — byte-+ // identical to the extraction they had before — while the phrase+ // arm evaluates each element on its own, matching the+ // `POSITION_INCREMENT_GAP` the segment postings now carry.+ let field_value_lists: Vec<Vec<String>> = fields.iter().filter_map(|field_spec| {let (field, _) = parse_field_boost(field_spec);- match get_field_value(source, field) {- Some(Value::String(s)) => Some(s.to_lowercase()),- Some(Value::Array(arr)) => {- let joined: Vec<String> = arr- .iter()- .map(|v| match v {- Value::String(s) => s.to_lowercase(),- other => other.to_string(),- })- .collect();- if joined.is_empty() {- None- } else {- Some(joined.join(" "))- }- }- Some(other) => Some(other.to_string().to_lowercase()),- None => None,- }+ multi_match_field_values_lc(source, field)}).collect();+ let field_texts: Vec<String> = field_value_lists+ .iter()+ .map(|values| values.join(" "))+ .collect();if is_phrase && !phrase_q_tokens.is_empty() {// phrase / phrase_prefix: POSITIONAL, per field (issue// #230). ES lowers these types to a dis_max over per-field@@ -33818,9 +33819,9 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {// consults it), and pre-fix `{"type":"phrase","operator":// "and"}` fell into the token-AND branch below and silently// stopped being a phrase at all.- field_texts- .iter()- .any(|ft| multi_match_phrase_hit(ft, &phrase_q_tokens, is_phrase_prefix, *slop))+ field_value_lists.iter().any(|values| {+ multi_match_phrase_hit(values, &phrase_q_tokens, is_phrase_prefix, *slop)+ })} else if is_cross && is_and && !tokens.is_empty() {// cross_fields + operator AND: every token must appear in// at least one listed field (combined perspective).@@ -35530,15 +35531,16 @@ fn score_query_against_doc(q: &QueryNode, source: &Value) -> f32 {} else {Vec::new()};- let field_hit = |text_lc: &str| -> bool {+ let field_hit = |values_lc: &[String]| -> bool {if is_phrase && !phrase_q_tokens.is_empty() {return multi_match_phrase_hit(- text_lc,+ values_lc,&phrase_q_tokens,is_phrase_prefix,*slop,);}+ let text_lc = &values_lc.join(" ");if q_tokens.is_empty() {return text_lc.contains(&q_lower);}@@ -35557,30 +35559,13 @@ fn score_query_against_doc(q: &QueryNode, source: &Value) -> f32 {let mut matched = false;for field_spec in fields {let (field, field_boost) = parse_field_boost(field_spec);- // Same per-field text extraction as `doc_matches_query`:- // strings, arrays (joined), and other scalars — an array-- // valued field admitted by membership must not score 0.0.- let text_lc: Option<String> = match get_field_value(source, field) {- Some(Value::String(s)) => Some(s.to_lowercase()),- Some(Value::Array(arr)) => {- let joined: Vec<String> = arr- .iter()- .map(|v| match v {- Value::String(s) => s.to_lowercase(),- other => other.to_string(),- })- .collect();- if joined.is_empty() {- None- } else {- Some(joined.join(" "))- }- }- Some(other) => Some(other.to_string().to_lowercase()),- None => None,- };- if let Some(text) = text_lc {- if field_hit(&text) {+ // Same per-field value extraction as `doc_matches_query`:+ // strings, arrays (one entry per element, #332), and other+ // scalars — an array-valued field admitted by membership must+ // not score 0.0.+ let values_lc: Option<Vec<String>> = multi_match_field_values_lc(source, field);+ if let Some(values) = values_lc {+ if field_hit(&values) {sum_score += field_boost;if field_boost > max_score {max_score = field_boost;diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex 201eab085..22c96c03b 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -183,8 +183,9 @@ struct MemEntry {/// Reconstruct the (field_name → flattened text) map that pre-M4.9/// `MemEntry` used to cache eagerly at ingest time. Called only by-/// flush (`drain_with_sources`, `drain`) and by the rare-/// `get_source` query path — neither is on the hot ingest loop.+/// the legacy `drain_with_sources` / `drain` / `get_source` paths, which+/// are NOT the segment-FTS input path — use+/// [`extract_field_values_excluding`] for anything that will be indexed.pub fn extract_text_fields_from(source: &Value) -> HashMap<String, String> {extract_text_fields_from_excluding(source, &std::collections::HashSet::new())}@@ -193,21 +194,114 @@ pub fn extract_text_fields_from_excluding(source: &Value,excluded: &std::collections::HashSet<String>,) -> HashMap<String, String> {+ extract_field_values_excluding(source, excluded)+ .into_iter()+ .map(|(k, v)| (k, v.iter().collect::<Vec<_>>().join(" ")))+ .collect()+}++/// Extract the top-level source fields as the segment-FTS writer wants them:+/// one [`FieldValues`] per field, with a JSON **array preserved as N separate+/// values** rather than joined into one string.+///+/// This is the single walker feeding both segment-build paths — the flush path+/// (`do_flush_shard` → `add_documents_parallel`) and the merge path+/// (`run_merge_once`). Before #332 each path had its own copy+/// (`memtable::extract_text_value` and `index::extract_field_text`), and both+/// copies did `arr.join(" ")`, so `{"tags":["red","blue"]}` was indexed by the+/// keyword analyzer as the single term `"red blue"` — matching neither `red`+/// nor `blue`.+///+/// It also carries #328's nested-exclusion pruning, which the merge path's own+/// copy used to do inline: a field whose value is an OBJECT with an excluded+/// descendant (a `dense_vector` mapped at `passages.vec`) is flattened from a+/// PRUNED copy of that object, so the segment builder never resurrects the+/// vector's decimal components as a term under the parent's name. Unifying the+/// two paths onto this walker would otherwise have dropped that pruning on the+/// merge path (`.passages.fst` back to ~398 KB); doing it here gives it to the+/// flush path too, matching the live-insert path (`collect_text_fields`). It+/// only fires for objects, so #332's array handling is untouched.+pub fn extract_field_values_excluding(+ source: &Value,+ excluded: &std::collections::HashSet<String>,+) -> HashMap<String, xerj_fts::index::FieldValues> {+ use xerj_fts::index::FieldValues;let mut out = HashMap::new();if let Some(obj) = source.as_object() {for (key, val) in obj {if excluded.contains(key) {continue;}- let text = extract_text_value(val);- if !text.is_empty() {- out.insert(key.clone(), text);+ // #328 — object field with an excluded descendant: flatten a pruned+ // copy so a nested `dense_vector` cannot leak into the parent+ // object's term dictionary. Arrays never enter this arm, so the+ // #332 split below still owns every multi-valued keyword field.+ if val.is_object() && has_excluded_descendant(key, excluded) {+ let text = extract_text_value_excluding(val, key, excluded);+ if !text.is_empty() {+ out.insert(key.clone(), FieldValues::One(text));+ }+ continue;+ }+ if let Some(values) = extract_field_values(val) {+ out.insert(key.clone(), values);}}}out}+/// Convert one JSON value into the values the FTS writer should index for it.+///+/// * scalars → `One`+/// * arrays → one entry per element, recursively flattened (a nested array is+/// itself multi-valued in ES; an object element is JSON-encoded as before)+/// * objects → `One(json)`, unchanged: the root-level JSON blob that+/// flattened-style whole-object queries rely on+/// * `null` / empty → `None`, i.e. the field is not indexed for this doc+///+/// Empty strings are dropped INSIDE an array too — the pre-#332 code produced+/// them only as a by-product of `join(" ")` on `[null, "x"]`, and an empty+/// keyword token would otherwise become a real, matchable term.+fn extract_field_values(val: &Value) -> Option<xerj_fts::index::FieldValues> {+ use xerj_fts::index::FieldValues;+ match val {+ Value::Array(arr) => {+ let mut values: Vec<String> = Vec::with_capacity(arr.len());+ collect_array_values(arr, &mut values);+ if values.is_empty() {+ None+ } else {+ Some(FieldValues::from_values(values))+ }+ }+ _ => {+ let text = extract_text_value(val);+ if text.is_empty() {+ None+ } else {+ Some(FieldValues::One(text))+ }+ }+ }+}++fn collect_array_values(arr: &[Value], out: &mut Vec<String>) {+ for element in arr {+ match element {+ // A nested array contributes its own elements as separate values —+ // ES flattens arrays of arrays for indexing purposes.+ Value::Array(inner) => collect_array_values(inner, out),+ _ => {+ let text = extract_text_value(element);+ if !text.is_empty() {+ out.push(text);+ }+ }+ }+ }+}+/// Interned document identifier.////// One `Arc<str>` is allocated per document at insert time and its@@ -1422,7 +1516,12 @@ impl ShardedFtsMemtable {/// path. Newer writes must be filtered by the caller before restoration.pub(crate) fn restore_failed_flush(&self,- entries: Vec<(u64, String, HashMap<String, String>, Arc<Value>)>,+ entries: Vec<(+ u64,+ String,+ HashMap<String, xerj_fts::index::FieldValues>,+ Arc<Value>,+ )>,version_map: &xerj_storage::version_map::VersionMap,) {for (seq_no, doc_id, fields, source) in entries {@@ -1446,9 +1545,20 @@ impl ShardedFtsMemtable {.get_analyzer("default").or_else(|| shard.registry.get_analyzer("standard")).expect("standard analyzer always present");+ // Each value of a multi-valued field is analyzed on its own and the+ // token streams concatenated (#332). The memtable's postings carry+ // term frequencies only — no positions — so no position gap is+ // needed here; the gap lives in the segment writer, which is where+ // positions exist.let analyzed: Vec<(String, Vec<Token>)> = fields.into_iter()- .map(|(field, text)| (field, analyzer.analyze(&text)))+ .map(|(field, values)| {+ let tokens = values+ .iter()+ .flat_map(|value| analyzer.analyze(value))+ .collect();+ (field, tokens)+ }).collect();let size = (source.to_string().len() + doc_id.len()) * 3 + 64;shard.insert_analyzed(seq_no, doc_id, source, &analyzed, size);diff --git a/engine/crates/xerj-engine/tests/multi_valued_keyword_terms.rs b/engine/crates/xerj-engine/tests/multi_valued_keyword_terms.rsnew file mode 100644index 000000000..99b8d605a--- /dev/null+++ b/engine/crates/xerj-engine/tests/multi_valued_keyword_terms.rs@@ -0,0 +1,276 @@+//! Regression tests for issue #332: a multi-valued keyword field was+//! flattened into ONE FTS token.+//!+//! `{"tags": ["red", "blue"]}` on a `keyword` field used to be joined into the+//! single string `"red blue"` before the FTS layer saw it+//! (`memtable::extract_text_value` / `index::extract_field_text`, both doing+//! `arr.join(" ")`). The keyword analyzer emits its whole input as one token,+//! so the segment carried the term `"red blue"` and neither `"red"` nor+//! `"blue"` existed as a posting. Every clause that projects to a whole-value+//! `FtsQuery::Term` therefore missed the document once it was flushed —+//! visible in the live engine as `multi_match: {"query": "red"}` returning 0+//! hits while `multi_match: {"query": "red blue"}` returned 1, the exact+//! inverse of Elasticsearch.+//!+//! Every test asserts the hit set is the same BEFORE and AFTER `flush()`:+//! this class of defect is flush-dependent, and a post-flush-only assertion+//! would pass on a build that had simply stopped indexing the field.++use std::collections::BTreeSet;++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn req(q: Value) -> xerj_query::ast::SearchRequest {+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")+}++async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {+ idx.search(&req(q.clone()))+ .await+ .unwrap()+ .hits+ .iter()+ .map(|h| h.id.clone())+ .collect()+}++fn expect(ids: &[&str]) -> BTreeSet<String> {+ ids.iter().map(|s| s.to_string()).collect()+}++/// Two docs, one index:+/// * `1` — multi-valued keyword `tags: ["red","blue"]`, multi-valued text+/// `notes: ["alpha bravo","charlie delta"]`+/// * `2` — the single-valued control, `tags: "red"`, `notes: "alpha bravo"`+async fn seed(engine: &Engine, name: &str) -> std::sync::Arc<Index> {+ let mut schema = Schema::empty();+ schema.fields.push(FieldConfig::new("body", FieldType::Text));+ schema+ .fields+ .push(FieldConfig::new("notes", FieldType::Text));+ schema+ .fields+ .push(FieldConfig::new("tags", FieldType::Keyword));+ engine.create_index(name, schema).unwrap();+ let idx = engine.get_index(name).unwrap();+ idx.index_document(+ Some("1".into()),+ json!({+ "body": "hello",+ "tags": ["red", "blue"],+ "notes": ["alpha bravo", "charlie delta"]+ }),+ )+ .await+ .unwrap();+ idx.index_document(+ Some("2".into()),+ json!({"body": "hello", "tags": "red", "notes": "alpha bravo"}),+ )+ .await+ .unwrap();+ idx+}++/// Run every case pre-flush and post-flush and require the same hit set.+async fn assert_flush_parity(idx: &std::sync::Arc<Index>, cases: &[(Value, &[&str], &str)]) {+ for (q, exp, label) in cases {+ assert_eq!(+ ids(idx, q).await,+ expect(exp),+ "{label}: PRE-flush (memtable) hit set wrong for {q}"+ );+ }+ idx.flush().await.unwrap();+ for (q, exp, label) in cases {+ assert_eq!(+ ids(idx, q).await,+ expect(exp),+ "{label}: POST-flush (segment) hit set wrong for {q}"+ );+ }+}++#[tokio::test]+async fn term_match_multi_match_see_every_keyword_array_element() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mvk_terms").await;++ assert_flush_parity(+ &idx,+ &[+ // The headline: an exact `term` for one element of the array.+ (json!({"term": {"tags": "red"}}), &["1", "2"], "term red"),+ (json!({"term": {"tags": "blue"}}), &["1"], "term blue"),+ // `match` on a keyword field lowers to the same whole-value term.+ (json!({"match": {"tags": "red"}}), &["1", "2"], "match red"),+ (json!({"match": {"tags": "blue"}}), &["1"], "match blue"),+ // `multi_match` — the clause that visibly missed doc 1 on `main`.+ (+ json!({"multi_match": {"query": "red", "fields": ["tags", "body"]}}),+ &["1", "2"],+ "multi_match red",+ ),+ (+ json!({"multi_match": {"query": "blue", "fields": ["tags", "body"]}}),+ &["1"],+ "multi_match blue",+ ),+ // `terms` takes the stored-scan route and always agreed with ES;+ // keep it as the in-binary control that the two routes now match.+ (+ json!({"terms": {"tags": ["blue"]}}),+ &["1"],+ "terms control",+ ),+ (+ json!({"bool": {"must": [{"match": {"body": "hello"}}],+ "filter": [{"terms": {"tags": ["red"]}}]}}),+ &["1", "2"],+ "bool must+filter control",+ ),+ ],+ )+ .await;+}++/// The joined string must NOT survive as a term. Pre-fix `"red blue"` was+/// the ONLY term the segment held for doc 1, so this query matched it —+/// exactly backwards from Elasticsearch, where no keyword value equals+/// `"red blue"`.+#[tokio::test]+async fn joined_array_is_not_a_term() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mvk_joined").await;++ assert_flush_parity(+ &idx,+ &[+ (json!({"term": {"tags": "red blue"}}), &[], "term joined"),+ (+ json!({"multi_match": {"query": "red blue", "fields": ["tags"]}}),+ &[],+ "multi_match joined",+ ),+ ],+ )+ .await;+}++/// A phrase must not span two elements of an array — Lucene separates them by+/// `position_increment_gap` (100), and so does the segment writer now.+/// Pre-fix the memtable said 0 hits and the flushed segment said 1: the+/// joined string put `bravo` and `charlie` at adjacent positions.+#[tokio::test]+async fn phrase_does_not_span_array_elements() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mvk_phrase").await;++ assert_flush_parity(+ &idx,+ &[+ (+ json!({"match_phrase": {"notes": "bravo charlie"}}),+ &[],+ "match_phrase across the boundary",+ ),+ (+ json!({"multi_match": {"query": "bravo charlie",+ "fields": ["notes"], "type": "phrase"}}),+ &[],+ "multi_match phrase across the boundary",+ ),+ // ... but a phrase WITHIN one element still matches, in both docs+ // for the first element and only doc 1 for the second.+ (+ json!({"match_phrase": {"notes": "alpha bravo"}}),+ &["1", "2"],+ "match_phrase inside element 0",+ ),+ (+ json!({"multi_match": {"query": "charlie delta",+ "fields": ["notes"], "type": "phrase"}}),+ &["1"],+ "multi_match phrase inside element 1",+ ),+ ],+ )+ .await;+}++/// A `terms` aggregation over a multi-valued keyword field buckets EVERY+/// value, and the answer does not change at flush.+#[tokio::test]+async fn terms_agg_buckets_every_array_element() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mvk_agg").await;++ let body = json!({+ "query": {"match_all": {}},+ "size": 0,+ "aggs": {"t": {"terms": {"field": "tags"}}}+ });+ let expected = json!([+ {"key": "red", "doc_count": 2},+ {"key": "blue", "doc_count": 1}+ ]);++ for state in ["PRE-flush", "POST-flush"] {+ if state == "POST-flush" {+ idx.flush().await.unwrap();+ }+ let res = idx.search(&parse_request(&body).unwrap()).await.unwrap();+ let buckets = res.aggs.as_ref().and_then(|a| a["t"]["buckets"].as_array())+ .cloned()+ .unwrap_or_default();+ let trimmed: Vec<Value> = buckets+ .iter()+ .map(|b| json!({"key": b["key"], "doc_count": b["doc_count"]}))+ .collect();+ assert_eq!(+ Value::Array(trimmed),+ expected,+ "{state}: terms agg over a multi-valued keyword field"+ );+ }+}++/// `_source` is stored verbatim and must come back as the array it went in+/// as — the fix touches the inverted index, never the stored document.+#[tokio::test]+async fn source_round_trips_unchanged() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "mvk_source").await;++ for state in ["PRE-flush", "POST-flush"] {+ if state == "POST-flush" {+ idx.flush().await.unwrap();+ }+ let res = idx+ .search(&req(json!({"term": {"tags": "blue"}})))+ .await+ .unwrap();+ assert_eq!(res.hits.len(), 1, "{state}: expected exactly doc 1");+ assert_eq!(+ res.hits[0].source["tags"],+ json!(["red", "blue"]),+ "{state}: _source must keep the original array"+ );+ }+}diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex 02a0c083d..1a2046afe 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -540,6 +540,143 @@ fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {})}+// ── Multi-valued field input ──────────────────────────────────────────────────++/// Position gap inserted between two values of the same multi-valued field.+///+/// Mirrors Lucene/Elasticsearch's default `position_increment_gap` of 100.+/// Without it, `["alpha bravo", "charlie"]` would put `bravo` at position 1 and+/// `charlie` at position 2, so `match_phrase: "bravo charlie"` would match a+/// phrase that exists in NEITHER value. 100 is large enough that no realistic+/// `slop` bridges the boundary, and small enough to stay in the VByte fast path.+pub const POSITION_INCREMENT_GAP: u32 = 100;++/// The values a single document supplies for a single field.+///+/// A JSON document field is either a scalar or an array, and Elasticsearch+/// treats the array as N independent values of the field — not as one+/// concatenated value. That distinction is invisible for an analyzed `text`+/// field only by accident (the standard tokenizer re-splits on the joining+/// space); for a `keyword` field, which is indexed with the whole input as one+/// token, joining `["red","blue"]` into `"red blue"` produces a term that+/// matches neither `red` nor `blue`. See issue #332.+///+/// `One` exists so the overwhelmingly common single-valued case costs no extra+/// allocation on the flush/merge path — the `String` is moved in as-is.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum FieldValues {+ /// Exactly one value (a JSON scalar, or a one-element array).+ One(String),+ /// Two or more values (a JSON array). Never constructed with < 2 elements+ /// by [`FieldValues::from_values`], but tolerated if built directly.+ Many(Vec<String>),+}++impl FieldValues {+ /// Build from an iterator, collapsing the single-value case to `One`.+ pub fn from_values<I: IntoIterator<Item = String>>(values: I) -> Self {+ let mut v: Vec<String> = values.into_iter().collect();+ if v.len() == 1 {+ FieldValues::One(v.pop().unwrap())+ } else {+ FieldValues::Many(v)+ }+ }++ /// Iterate every value in document order.+ pub fn iter(&self) -> impl Iterator<Item = &str> {+ match self {+ FieldValues::One(s) => std::slice::from_ref(s).iter(),+ FieldValues::Many(v) => v.iter(),+ }+ .map(|s| s.as_str())+ }++ /// Number of values.+ pub fn len(&self) -> usize {+ match self {+ FieldValues::One(_) => 1,+ FieldValues::Many(v) => v.len(),+ }+ }++ /// True when the field carries no value at all (only reachable via a+ /// directly-constructed `Many(vec![])`).+ pub fn is_empty(&self) -> bool {+ match self {+ FieldValues::One(s) => s.is_empty(),+ FieldValues::Many(v) => v.iter().all(|s| s.is_empty()),+ }+ }+}++impl From<String> for FieldValues {+ fn from(s: String) -> Self {+ FieldValues::One(s)+ }+}++impl From<&str> for FieldValues {+ fn from(s: &str) -> Self {+ FieldValues::One(s.to_owned())+ }+}++impl From<Vec<String>> for FieldValues {+ fn from(v: Vec<String>) -> Self {+ FieldValues::from_values(v)+ }+}++/// Analyze every value of one field for one document and fold the tokens into+/// `field_data`.+///+/// Shared by [`FtsIndexWriter::add_document`] and+/// [`FtsIndexWriter::add_documents_parallel`] so the serial and parallel paths+/// cannot drift — the two used to hold independent copies of this loop.+///+/// Semantics per value:+/// * positions restart from `base`, which advances past the previous value's+/// last position plus [`POSITION_INCREMENT_GAP`];+/// * the norm (field length) is the SUM of every value's token count, matching+/// Lucene, so BM25 length-normalisation sees the whole field;+/// * exactly one norm entry and one `total_docs` increment per (doc, field),+/// however many values the field has.+fn index_field_values<'a>(+ field_data: &mut FieldData,+ analyzer: &crate::analyzer::AnalyzerPipeline,+ doc_ord: u32,+ values: impl Iterator<Item = &'a str>,+) {+ let mut field_len: u64 = 0;+ let mut base: u32 = 0;+ let mut first = true;+ for value in values {+ if !first {+ base = base.saturating_add(POSITION_INCREMENT_GAP);+ }+ first = false;+ let tokens = analyzer.analyze(value);+ let mut max_position = 0u32;+ for token in &tokens {+ field_data+ .postings+ .add_occurrence(&token.text, doc_ord, base.saturating_add(token.position));+ max_position = max_position.max(token.position);+ }+ if !tokens.is_empty() {+ base = base.saturating_add(max_position).saturating_add(1);+ }+ field_len += tokens.len() as u64;+ }++ field_data+ .norms+ .push((doc_ord, field_len.min(u16::MAX as u64) as u16));+ field_data.stats.total_docs += 1;+ field_data.stats.total_field_length += field_len;+}+// ── FtsIndexWriter ────────────────────────────────────────────────────────────/// Builds the FTS inverted index for one segment.@@ -686,11 +823,15 @@ impl FtsIndexWriter {/// Index all text fields of one document.///- /// `fields` is a map of field name → field text value.+ /// `fields` is a map of field name → that field's [`FieldValues`]. Each+ /// value is analyzed independently and separated from the next by+ /// [`POSITION_INCREMENT_GAP`] — a JSON array is N values of the field, not+ /// one joined value (issue #332).+ ////// Fields not previously registered via `configure_field` are indexed/// with the default configuration (standard analyzer, positions on).- pub fn add_document(&mut self, doc_id: u32, fields: &HashMap<String, String>) {- for (field_name, text) in fields {+ pub fn add_document(&mut self, doc_id: u32, fields: &HashMap<String, FieldValues>) {+ for (field_name, values) in fields {let registry = Arc::clone(&self.registry);// Ensure field entry exists@@ -715,30 +856,13 @@ impl FtsIndexWriter {.or_else(|| registry.get_analyzer("standard")).unwrap();- let tokens = analyzer.analyze(text);- let field_len = tokens.len() as u64;-- // Record norms (capped at u16::MAX)- field_data- .norms- .push((doc_id, field_len.min(u16::MAX as u64) as u16));-- // Update field stats- field_data.stats.total_docs += 1;- field_data.stats.total_field_length += field_len;-- // Accumulate postings- for token in &tokens {- field_data- .postings- .add_occurrence(&token.text, doc_id, token.position);- }+ index_field_values(field_data, &analyzer, doc_id, values.iter());}}/// V4 M4 — **parallel batch** add for flush time.///- /// Reshapes `(doc_id, field, text)` from row-major (per-doc) into+ /// Reshapes `(doc_id, field, values)` from row-major (per-doc) into/// column-major (per-field) then tokenises + builds per-field/// postings in parallel via rayon. The underlying PostingsWriter/// state is still single-threaded per field, but fields run in@@ -758,24 +882,24 @@ impl FtsIndexWriter {/// `Arc<serde_json::Value>`, …): this method never reads it.pub fn add_documents_parallel<S: Sync>(&mut self,- docs: &[(String, HashMap<String, String>, S)],+ docs: &[(String, HashMap<String, FieldValues>, S)],) {use rayon::prelude::*;use std::collections::HashMap as StdHashMap;- // Column-major reshape: field_name → Vec<(doc_ordinal, text)>.+ // Column-major reshape: field_name → Vec<(doc_ordinal, values)>.// Lookup-first so the common case (field already seen) skips the// per-doc-field `field_name.clone()` the `entry()` API forced.- let mut per_field: StdHashMap<String, Vec<(u32, &str)>> = StdHashMap::new();+ let mut per_field: StdHashMap<String, Vec<(u32, &FieldValues)>> = StdHashMap::new();for (ord, (_id, fields, _src)) in docs.iter().enumerate() {- for (field_name, text) in fields {+ for (field_name, values) in fields {if let Some(v) = per_field.get_mut(field_name) {- v.push((ord as u32, text.as_str()));+ v.push((ord as u32, values));} else {per_field.entry(field_name.clone()).or_default()- .push((ord as u32, text.as_str()));+ .push((ord as u32, values));}}}@@ -806,7 +930,7 @@ impl FtsIndexWriter {.map(|(k, v)| (k.clone(), v.config.clone())).collect();- let per_field_vec: Vec<(String, Vec<(u32, &str)>)> = per_field.into_iter().collect();+ let per_field_vec: Vec<(String, Vec<(u32, &FieldValues)>)> = per_field.into_iter().collect();let built: Vec<(String, FieldData)> = per_field_vec.into_par_iter()@@ -829,17 +953,8 @@ impl FtsIndexWriter {norms: Vec::with_capacity(entries.len()),};- for (doc_ord, text) in entries {- let tokens = analyzer.analyze(text);- let field_len = tokens.len() as u64;- fd.norms- .push((doc_ord, field_len.min(u16::MAX as u64) as u16));- fd.stats.total_docs += 1;- fd.stats.total_field_length += field_len;- for token in &tokens {- fd.postings- .add_occurrence(&token.text, doc_ord, token.position);- }+ for (doc_ord, values) in entries {+ index_field_values(&mut fd, &analyzer, doc_ord, values.iter());}(field_name, fd)})@@ -1741,14 +1856,14 @@ mod tests {let mut writer = FtsIndexWriter::new(dir.path(), "seg0", registry);- let docs: Vec<HashMap<String, String>> = vec![- [("body".to_owned(), "the quick brown fox".to_owned())]+ let docs: Vec<HashMap<String, FieldValues>> = vec![+ [("body".to_owned(), FieldValues::from("the quick brown fox"))].into_iter().collect(),- [("body".to_owned(), "the lazy dog".to_owned())]+ [("body".to_owned(), FieldValues::from("the lazy dog"))].into_iter().collect(),- [("body".to_owned(), "quick fox lazy dog".to_owned())]+ [("body".to_owned(), FieldValues::from("quick fox lazy dog"))].into_iter().collect(),];@@ -1804,10 +1919,10 @@ mod tests {writer.configure_field("title", cfg);let docs = [- [("title".to_owned(), "hello world".to_owned())]+ [("title".to_owned(), FieldValues::from("hello world"))].into_iter().collect(),- [("title".to_owned(), "hello rust".to_owned())]+ [("title".to_owned(), FieldValues::from("hello rust"))].into_iter().collect(),];@@ -1842,10 +1957,10 @@ mod tests {let dir = TempDir::new().unwrap();let mut writer = FtsIndexWriter::new(dir.path(), "seg-safe", make_registry());let fields = ["title.body-2", "@timestamp", "space field", "配置.名字"];- let document: HashMap<String, String> = fields+ let document: HashMap<String, FieldValues> = fields.into_iter().enumerate()- .map(|(index, field)| (field.to_owned(), format!("needle{index}")))+ .map(|(index, field)| (field.to_owned(), FieldValues::One(format!("needle{index}")))).collect();writer.add_document(0, &document);writer.finish().unwrap();@@ -1864,7 +1979,7 @@ mod tests {fn write_legacy_raw_field_fixture(segment_dir: &Path, segment_id: &str, legacy_field: &str) {let source_field = "legacy_fixture_source";let mut writer = FtsIndexWriter::new(segment_dir, segment_id, make_registry());- let document = [(source_field.to_owned(), "legacyneedle".to_owned())]+ let document = [(source_field.to_owned(), FieldValues::from("legacyneedle"))].into_iter().collect();writer.add_document(0, &document);@@ -1926,9 +2041,9 @@ mod tests {let (unsafe_name, legacy_alias) = collision_fields();let source_fields = ["source_unsafe", "source_alias"];let mut writer = FtsIndexWriter::new(segment_dir, segment_id, make_registry());- let document = [- (source_fields[0].to_owned(), "unsafeneedle".to_owned()),- (source_fields[1].to_owned(), "aliasneedle".to_owned()),+ let document: HashMap<String, FieldValues> = [+ (source_fields[0].to_owned(), FieldValues::from("unsafeneedle")),+ (source_fields[1].to_owned(), FieldValues::from("aliasneedle")),].into_iter().collect();@@ -1994,9 +2109,9 @@ mod tests {let segment_id = "v2-collision";let (unsafe_name, legacy_alias) = collision_fields();let mut writer = FtsIndexWriter::new(&segment_dir, segment_id, make_registry());- let document = [- (unsafe_name.clone(), "unsafeneedle".to_owned()),- (legacy_alias.clone(), "aliasneedle".to_owned()),+ let document: HashMap<String, FieldValues> = [+ (unsafe_name.clone(), FieldValues::from("unsafeneedle")),+ (legacy_alias.clone(), FieldValues::from("aliasneedle")),].into_iter().collect();@@ -2066,7 +2181,7 @@ mod tests {let mut writer = FtsIndexWriter::new(&segment_dir, segment_id, make_registry());writer.add_document(0,- &[(field.to_owned(), "encodedneedle".to_owned())]+ &[(field.to_owned(), FieldValues::from("encodedneedle"))].into_iter().collect(),);@@ -2204,10 +2319,12 @@ mod tests {assert!(legacy_field_sidecar_path(&segment_dir, "seg-unsafe", field, "fst").is_none());}let mut writer = FtsIndexWriter::new(&segment_dir, "seg-unsafe", make_registry());- let document: HashMap<String, String> = fields+ let document: HashMap<String, FieldValues> = fields.iter().enumerate()- .map(|(index, field)| ((*field).to_owned(), format!("needle{index}")))+ .map(|(index, field)| {+ ((*field).to_owned(), FieldValues::One(format!("needle{index}")))+ }).collect();writer.add_document(0, &document);writer.publish_encoded_filename_layout().unwrap();diff --git a/engine/crates/xerj-fts/src/lib.rs b/engine/crates/xerj-fts/src/lib.rsindex d37239bb3..f3d4cd533 100644--- a/engine/crates/xerj-fts/src/lib.rs+++ b/engine/crates/xerj-fts/src/lib.rs@@ -32,7 +32,7 @@//! use std::sync::Arc;//! use xerj_fts::{//! analyzer::AnalyzerRegistry,-//! index::{FtsIndexWriter, FtsIndexReader},+//! index::{FieldValues, FtsIndexWriter, FtsIndexReader},//! search::{FtsSearcher, Query, TermQuery},//! };//!@@ -42,7 +42,7 @@//! let mut writer = FtsIndexWriter::new(dir.path(), "seg0", Arc::clone(®istry));//!//! let mut doc = HashMap::new();-//! doc.insert("body".to_owned(), "the quick brown fox".to_owned());+//! doc.insert("body".to_owned(), FieldValues::from("the quick brown fox"));//! writer.add_document(0, &doc);//! writer.finish().unwrap();//!@@ -75,7 +75,9 @@ pub use bm25::{DEFAULT_K1,};-pub use index::{FieldIndexConfig, FtsIndexReader, FtsIndexWriter};+pub use index::{+ FieldIndexConfig, FieldValues, FtsIndexReader, FtsIndexWriter, POSITION_INCREMENT_GAP,+};pub use postings::{DecodedPosting, PostingsReader, PostingsWriter, TermPostings, BLOCK_SIZE, SKIP_INTERVAL,diff --git a/engine/crates/xerj-fts/src/search.rs b/engine/crates/xerj-fts/src/search.rsindex 15001d454..56abaa731 100644--- a/engine/crates/xerj-fts/src/search.rs+++ b/engine/crates/xerj-fts/src/search.rs@@ -1555,7 +1555,7 @@ mod tests {use super::*;use crate::{analyzer::AnalyzerRegistry,- index::{FieldIndexConfig, FtsIndexReader, FtsIndexWriter},+ index::{FieldIndexConfig, FieldValues, FtsIndexReader, FtsIndexWriter},};use std::collections::HashMap;use tempfile::TempDir;@@ -1581,9 +1581,10 @@ mod tests {];for (i, text) in docs.iter().enumerate() {- let fields: HashMap<String, String> = [("body".to_owned(), text.to_string())]- .into_iter()- .collect();+ let fields: HashMap<String, FieldValues> =+ [("body".to_owned(), FieldValues::from(text.to_string()))]+ .into_iter()+ .collect();writer.add_document(i as u32, &fields);}writer.finish().unwrap();@@ -1749,9 +1750,9 @@ mod tests {("morning news", "golf scores from sunday"),];for (i, (title, body)) in docs.iter().enumerate() {- let fields: HashMap<String, String> = [- ("title".to_owned(), title.to_string()),- ("body".to_owned(), body.to_string()),+ let fields: HashMap<String, FieldValues> = [+ ("title".to_owned(), FieldValues::from(title.to_string())),+ ("body".to_owned(), FieldValues::from(body.to_string())),].into_iter().collect();@@ -2004,7 +2005,7 @@ mod tests {writer.configure_field("body", cfg);writer.add_document(0,- &[("body".to_owned(), "Rust Systems".to_owned())]+ &[("body".to_owned(), FieldValues::from("Rust Systems"))].into_iter().collect(),);@@ -2080,7 +2081,12 @@ mod tests {.map(|i| format!("term{i:05}")).collect::<Vec<_>>().join(" ");- writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());+ writer.add_document(+ 0,+ &[("body".to_owned(), FieldValues::from(text))]+ .into_iter()+ .collect(),+ );writer.finish().unwrap();let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());@@ -2128,7 +2134,12 @@ mod tests {.map(|i| format!("term{i:05}")).collect::<Vec<_>>().join(" ");- writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());+ writer.add_document(+ 0,+ &[("body".to_owned(), FieldValues::from(text))]+ .into_iter()+ .collect(),+ );writer.finish().unwrap();let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());@@ -2213,7 +2224,12 @@ mod tests {for i in 0..DICTIONARY_TERMS {text.push_str(&format!(" term{i:06}"));}- writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());+ writer.add_document(+ 0,+ &[("body".to_owned(), FieldValues::from(text))]+ .into_iter()+ .collect(),+ );writer.finish().unwrap();let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());