Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

wip(fts): index each value of a multi-valued keyword field (#332)

xerj-orgb1376e4e8bbc
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 20aa8611d..7ce8d25f1 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -8837,7 +8837,11 @@ impl Index {
// for the doc-values pass. Drops `dv_sources` entirely.
// Halves merge working memory and frees the Arc<Value>
// immediately after both passes complete.
- let mut fts_input: Vec<(String, HashMap<String, String>, Value)> = Vec::new();
+ let mut fts_input: Vec<(
+ String,
+ HashMap<String, xerj_fts::index::FieldValues>,
+ Value,
+ )> = Vec::new();
// (seq_no, doc_id) for every surviving doc — kept exactly
// aligned with the stored-section byte copy (pushed right
// after `live_doc_count += 1`, BEFORE the per-doc Value
@@ -19407,53 +19411,27 @@ fn build_fts_field_configs(schema: &Schema) -> HashMap<String, xerj_fts::index::
out
}
-/// Mirror of `FtsMemtable::extract_text_value` — converts any JSON value
-/// into a flattened string suitable for tokenization, so the merge path
-/// indexes fields the same way the memtable does. Kept as a free function
-/// here because the memtable's helper is private.
-fn extract_field_text(val: &Value) -> String {
- match val {
- Value::String(s) => s.clone(),
- Value::Number(n) => n.to_string(),
- Value::Bool(b) => b.to_string(),
- Value::Array(arr) => arr
- .iter()
- .map(extract_field_text)
- .collect::<Vec<_>>()
- .join(" "),
- Value::Object(_) => serde_json::to_string(val).unwrap_or_default(),
- Value::Null => String::new(),
- }
-}
-
+/// FTS input extraction for the MERGE path.
+///
+/// Delegates to the flush path's own walker
+/// (`crate::memtable::extract_field_values_excluding`) rather than mirroring
+/// it. Until #332 this file carried a private copy (`extract_field_text` +
+/// `extract_fts_fields_excluding`) that had to be kept in sync by hand with
+/// `memtable::extract_text_value`; both copies flattened a JSON array with
+/// `join(" ")`, and a merged segment reproduced the bug even after a flushed
+/// one stopped having it. One walker, one behaviour.
+///
+/// The shared walker also carries #328's nested-exclusion pruning: an
+/// object-valued field with an excluded descendant (a `dense_vector` at
+/// `passages.vec`) is flattened from a PRUNED copy, so the merge does not
+/// resurrect postings the flush path skipped. That pruning used to live inline
+/// here; it now lives once, in `extract_field_values_excluding`, so both the
+/// flush and the merge path get it.
fn extract_fts_fields_excluding(
source: &Value,
excluded: &std::collections::HashSet<String>,
-) -> HashMap<String, String> {
- let mut fields = HashMap::new();
- if let Some(obj) = source.as_object() {
- for (key, val) in obj {
- if excluded.contains(key) {
- continue;
- }
- // This is the MERGE path, and a merge must not resurrect postings
- // that flush correctly skipped (#328). A root key is caught above;
- // an object-valued key is flattened whole by `extract_field_text`,
- // so a `dense_vector` nested under it (`passages.vec`) would come
- // back here even though flush dropped it. Prune it out of the
- // flattened blob exactly the way `collect_text_fields` does.
- let text = if val.is_object() && crate::memtable::has_excluded_descendant(key, excluded)
- {
- crate::memtable::extract_text_value_excluding(val, key, excluded)
- } else {
- extract_field_text(val)
- };
- if !text.is_empty() {
- fields.insert(key.clone(), text);
- }
- }
- }
- fields
+) -> HashMap<String, xerj_fts::index::FieldValues> {
+ crate::memtable::extract_field_values_excluding(source, excluded)
}
// ── Short-circuit count helper ───────────────────────────────────────────────
@@ -24513,7 +24491,7 @@ async fn do_flush_shard(
let drained_opt: Option<(
Vec<(
String,
- std::collections::HashMap<String, String>,
+ std::collections::HashMap<String, xerj_fts::index::FieldValues>,
std::sync::Arc<serde_json::Value>,
)>,
xerj_storage::index_store::DrainedMemtable,
@@ -24576,7 +24554,7 @@ async fn do_flush_shard(
} else {
std::sync::Arc::new(serde_json::Value::Null)
};
- let fields = crate::memtable::extract_text_fields_from_excluding(
+ let fields = crate::memtable::extract_field_values_excluding(
&val,
&excluded_fts_fields,
);
@@ -24588,7 +24566,7 @@ async fn do_flush_shard(
// doesn't queue ahead of concurrent search/agg par_iters.
let drained_fts: Vec<(
String,
- std::collections::HashMap<String, String>,
+ std::collections::HashMap<String, xerj_fts::index::FieldValues>,
std::sync::Arc<serde_json::Value>,
)> = if prep_offload {
crate::background_pool().install(build_drained_fts)
@@ -33180,18 +33158,19 @@ fn phrase_prefix_positions_in_tokens(
/// ones, so a split query and an analyzed segment clause ask different
/// questions and the hit set changes at `_flush`.
///
-/// `field_text_lc` is the field's text as the rest of the `multi_match` arms
-/// see it — an array-valued field arrives already joined with a space.
-/// That means a phrase CAN span two array elements, which ES prevents with a
-/// `position_increment_gap`. XERJ has no such gap anywhere: the indexing
-/// path flattens an array to one space-joined string before the FTS layer
-/// ever sees it (`extract_field_text`), so the segment's positions have no
-/// gap either. Evaluating per element here would therefore FIX the memtable
-/// and leave the segment as it is — reintroducing the flush-variant hit set
-/// this fix exists to remove. The gap belongs in the indexing path; it is
-/// deliberately not attempted here.
+/// `field_values_lc` is the field's values, ONE ENTRY PER ARRAY ELEMENT, and
+/// the phrase must be satisfied inside a single element — never across two.
+///
+/// This used to take one already-space-joined string, so a phrase COULD span
+/// two array elements. That was deliberate at the time: the indexing path
+/// flattened an array to one space-joined string before the FTS layer saw it,
+/// so the segment's positions had no gap either, and evaluating per element
+/// here would have fixed only the memtable and left the hit set flush-variant.
+/// #332 put the gap where it belongs — `xerj_fts::index::POSITION_INCREMENT_GAP`
+/// now separates array elements in the segment postings, matching Lucene — so
+/// per-element evaluation here is what KEEPS the two sides equal.
fn multi_match_phrase_hit(
- field_text_lc: &str,
+ field_values_lc: &[String],
query_tokens: &[String],
is_prefix: bool,
slop: u32,
@@ -33199,11 +33178,42 @@ fn multi_match_phrase_hit(
if query_tokens.is_empty() {
return true;
}
- let field_tokens = phrase_tokens(field_text_lc);
- if is_prefix {
- phrase_prefix_positions_in_tokens(&field_tokens, query_tokens, slop)
- } else {
- phrase_positions_in_tokens(&field_tokens, query_tokens, slop)
+ field_values_lc.iter().any(|value| {
+ let field_tokens = phrase_tokens(value);
+ if is_prefix {
+ phrase_prefix_positions_in_tokens(&field_tokens, query_tokens, slop)
+ } else {
+ phrase_positions_in_tokens(&field_tokens, query_tokens, slop)
+ }
+ })
+}
+
+/// Per-element lowercased values of `field` in one stored source document, as
+/// the `multi_match` arms need them.
+///
+/// An array yields one entry per element (#332); a scalar yields a single
+/// entry. The non-phrase arms join the result back with a space, which is
+/// byte-identical to what they extracted before; the phrase arms consume the
+/// elements separately so a phrase cannot straddle the boundary.
+fn multi_match_field_values_lc(source: &Value, field: &str) -> Option<Vec<String>> {
+ match get_field_value(source, field) {
+ Some(Value::String(s)) => Some(vec![s.to_lowercase()]),
+ Some(Value::Array(arr)) => {
+ let values: Vec<String> = arr
+ .iter()
+ .map(|v| match v {
+ Value::String(s) => s.to_lowercase(),
+ other => other.to_string(),
+ })
+ .collect();
+ if values.is_empty() {
+ None
+ } else {
+ Some(values)
+ }
+ }
+ Some(other) => Some(vec![other.to_string().to_lowercase()]),
+ None => None,
}
}
@@ -33767,31 +33777,22 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
} else {
Vec::new()
};
- let field_texts: Vec<String> = fields
+ // Per-field values, one entry per array element (#332). The
+ // non-phrase arms below join them back with a space — byte-
+ // identical to the extraction they had before — while the phrase
+ // arm evaluates each element on its own, matching the
+ // `POSITION_INCREMENT_GAP` the segment postings now carry.
+ let field_value_lists: Vec<Vec<String>> = fields
.iter()
.filter_map(|field_spec| {
let (field, _) = parse_field_boost(field_spec);
- match get_field_value(source, field) {
- Some(Value::String(s)) => Some(s.to_lowercase()),
- Some(Value::Array(arr)) => {
- let joined: Vec<String> = arr
- .iter()
- .map(|v| match v {
- Value::String(s) => s.to_lowercase(),
- other => other.to_string(),
- })
- .collect();
- if joined.is_empty() {
- None
- } else {
- Some(joined.join(" "))
- }
- }
- Some(other) => Some(other.to_string().to_lowercase()),
- None => None,
- }
+ multi_match_field_values_lc(source, field)
})
.collect();
+ let field_texts: Vec<String> = field_value_lists
+ .iter()
+ .map(|values| values.join(" "))
+ .collect();
if is_phrase && !phrase_q_tokens.is_empty() {
// phrase / phrase_prefix: POSITIONAL, per field (issue
// #230). ES lowers these types to a dis_max over per-field
@@ -33818,9 +33819,9 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
// consults it), and pre-fix `{"type":"phrase","operator":
// "and"}` fell into the token-AND branch below and silently
// stopped being a phrase at all.
- field_texts
- .iter()
- .any(|ft| multi_match_phrase_hit(ft, &phrase_q_tokens, is_phrase_prefix, *slop))
+ field_value_lists.iter().any(|values| {
+ multi_match_phrase_hit(values, &phrase_q_tokens, is_phrase_prefix, *slop)
+ })
} else if is_cross && is_and && !tokens.is_empty() {
// cross_fields + operator AND: every token must appear in
// at least one listed field (combined perspective).
@@ -35530,15 +35531,16 @@ fn score_query_against_doc(q: &QueryNode, source: &Value) -> f32 {
} else {
Vec::new()
};
- let field_hit = |text_lc: &str| -> bool {
+ let field_hit = |values_lc: &[String]| -> bool {
if is_phrase && !phrase_q_tokens.is_empty() {
return multi_match_phrase_hit(
- text_lc,
+ values_lc,
&phrase_q_tokens,
is_phrase_prefix,
*slop,
);
}
+ let text_lc = &values_lc.join(" ");
if q_tokens.is_empty() {
return text_lc.contains(&q_lower);
}
@@ -35557,30 +35559,13 @@ fn score_query_against_doc(q: &QueryNode, source: &Value) -> f32 {
let mut matched = false;
for field_spec in fields {
let (field, field_boost) = parse_field_boost(field_spec);
- // Same per-field text extraction as `doc_matches_query`:
- // strings, arrays (joined), and other scalars — an array-
- // valued field admitted by membership must not score 0.0.
- let text_lc: Option<String> = match get_field_value(source, field) {
- Some(Value::String(s)) => Some(s.to_lowercase()),
- Some(Value::Array(arr)) => {
- let joined: Vec<String> = arr
- .iter()
- .map(|v| match v {
- Value::String(s) => s.to_lowercase(),
- other => other.to_string(),
- })
- .collect();
- if joined.is_empty() {
- None
- } else {
- Some(joined.join(" "))
- }
- }
- Some(other) => Some(other.to_string().to_lowercase()),
- None => None,
- };
- if let Some(text) = text_lc {
- if field_hit(&text) {
+ // Same per-field value extraction as `doc_matches_query`:
+ // strings, arrays (one entry per element, #332), and other
+ // scalars — an array-valued field admitted by membership must
+ // not score 0.0.
+ let values_lc: Option<Vec<String>> = multi_match_field_values_lc(source, field);
+ if let Some(values) = values_lc {
+ if field_hit(&values) {
sum_score += field_boost;
if field_boost > max_score {
max_score = field_boost;
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 201eab085..22c96c03b 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -183,8 +183,9 @@ struct MemEntry {
/// Reconstruct the (field_name → flattened text) map that pre-M4.9
/// `MemEntry` used to cache eagerly at ingest time. Called only by
-/// flush (`drain_with_sources`, `drain`) and by the rare
-/// `get_source` query path — neither is on the hot ingest loop.
+/// the legacy `drain_with_sources` / `drain` / `get_source` paths, which
+/// are NOT the segment-FTS input path — use
+/// [`extract_field_values_excluding`] for anything that will be indexed.
pub fn extract_text_fields_from(source: &Value) -> HashMap<String, String> {
extract_text_fields_from_excluding(source, &std::collections::HashSet::new())
}
@@ -193,21 +194,114 @@ pub fn extract_text_fields_from_excluding(
source: &Value,
excluded: &std::collections::HashSet<String>,
) -> HashMap<String, String> {
+ extract_field_values_excluding(source, excluded)
+ .into_iter()
+ .map(|(k, v)| (k, v.iter().collect::<Vec<_>>().join(" ")))
+ .collect()
+}
+
+/// Extract the top-level source fields as the segment-FTS writer wants them:
+/// one [`FieldValues`] per field, with a JSON **array preserved as N separate
+/// values** rather than joined into one string.
+///
+/// This is the single walker feeding both segment-build paths — the flush path
+/// (`do_flush_shard` → `add_documents_parallel`) and the merge path
+/// (`run_merge_once`). Before #332 each path had its own copy
+/// (`memtable::extract_text_value` and `index::extract_field_text`), and both
+/// copies did `arr.join(" ")`, so `{"tags":["red","blue"]}` was indexed by the
+/// keyword analyzer as the single term `"red blue"` — matching neither `red`
+/// nor `blue`.
+///
+/// It also carries #328's nested-exclusion pruning, which the merge path's own
+/// copy used to do inline: a field whose value is an OBJECT with an excluded
+/// descendant (a `dense_vector` mapped at `passages.vec`) is flattened from a
+/// PRUNED copy of that object, so the segment builder never resurrects the
+/// vector's decimal components as a term under the parent's name. Unifying the
+/// two paths onto this walker would otherwise have dropped that pruning on the
+/// merge path (`.passages.fst` back to ~398 KB); doing it here gives it to the
+/// flush path too, matching the live-insert path (`collect_text_fields`). It
+/// only fires for objects, so #332's array handling is untouched.
+pub fn extract_field_values_excluding(
+ source: &Value,
+ excluded: &std::collections::HashSet<String>,
+) -> HashMap<String, xerj_fts::index::FieldValues> {
+ use xerj_fts::index::FieldValues;
let mut out = HashMap::new();
if let Some(obj) = source.as_object() {
for (key, val) in obj {
if excluded.contains(key) {
continue;
}
- let text = extract_text_value(val);
- if !text.is_empty() {
- out.insert(key.clone(), text);
+ // #328 — object field with an excluded descendant: flatten a pruned
+ // copy so a nested `dense_vector` cannot leak into the parent
+ // object's term dictionary. Arrays never enter this arm, so the
+ // #332 split below still owns every multi-valued keyword field.
+ if val.is_object() && has_excluded_descendant(key, excluded) {
+ let text = extract_text_value_excluding(val, key, excluded);
+ if !text.is_empty() {
+ out.insert(key.clone(), FieldValues::One(text));
+ }
+ continue;
+ }
+ if let Some(values) = extract_field_values(val) {
+ out.insert(key.clone(), values);
}
}
}
out
}
+/// Convert one JSON value into the values the FTS writer should index for it.
+///
+/// * scalars → `One`
+/// * arrays → one entry per element, recursively flattened (a nested array is
+/// itself multi-valued in ES; an object element is JSON-encoded as before)
+/// * objects → `One(json)`, unchanged: the root-level JSON blob that
+/// flattened-style whole-object queries rely on
+/// * `null` / empty → `None`, i.e. the field is not indexed for this doc
+///
+/// Empty strings are dropped INSIDE an array too — the pre-#332 code produced
+/// them only as a by-product of `join(" ")` on `[null, "x"]`, and an empty
+/// keyword token would otherwise become a real, matchable term.
+fn extract_field_values(val: &Value) -> Option<xerj_fts::index::FieldValues> {
+ use xerj_fts::index::FieldValues;
+ match val {
+ Value::Array(arr) => {
+ let mut values: Vec<String> = Vec::with_capacity(arr.len());
+ collect_array_values(arr, &mut values);
+ if values.is_empty() {
+ None
+ } else {
+ Some(FieldValues::from_values(values))
+ }
+ }
+ _ => {
+ let text = extract_text_value(val);
+ if text.is_empty() {
+ None
+ } else {
+ Some(FieldValues::One(text))
+ }
+ }
+ }
+}
+
+fn collect_array_values(arr: &[Value], out: &mut Vec<String>) {
+ for element in arr {
+ match element {
+ // A nested array contributes its own elements as separate values —
+ // ES flattens arrays of arrays for indexing purposes.
+ Value::Array(inner) => collect_array_values(inner, out),
+ _ => {
+ let text = extract_text_value(element);
+ if !text.is_empty() {
+ out.push(text);
+ }
+ }
+ }
+ }
+}
+
/// Interned document identifier.
///
/// One `Arc<str>` is allocated per document at insert time and its
@@ -1422,7 +1516,12 @@ impl ShardedFtsMemtable {
/// path. Newer writes must be filtered by the caller before restoration.
pub(crate) fn restore_failed_flush(
&self,
- entries: Vec<(u64, String, HashMap<String, String>, Arc<Value>)>,
+ entries: Vec<(
+ u64,
+ String,
+ HashMap<String, xerj_fts::index::FieldValues>,
+ Arc<Value>,
+ )>,
version_map: &xerj_storage::version_map::VersionMap,
) {
for (seq_no, doc_id, fields, source) in entries {
@@ -1446,9 +1545,20 @@ impl ShardedFtsMemtable {
.get_analyzer("default")
.or_else(|| shard.registry.get_analyzer("standard"))
.expect("standard analyzer always present");
+ // Each value of a multi-valued field is analyzed on its own and the
+ // token streams concatenated (#332). The memtable's postings carry
+ // term frequencies only — no positions — so no position gap is
+ // needed here; the gap lives in the segment writer, which is where
+ // positions exist.
let analyzed: Vec<(String, Vec<Token>)> = fields
.into_iter()
- .map(|(field, text)| (field, analyzer.analyze(&text)))
+ .map(|(field, values)| {
+ let tokens = values
+ .iter()
+ .flat_map(|value| analyzer.analyze(value))
+ .collect();
+ (field, tokens)
+ })
.collect();
let size = (source.to_string().len() + doc_id.len()) * 3 + 64;
shard.insert_analyzed(seq_no, doc_id, source, &analyzed, size);
diff --git a/engine/crates/xerj-engine/tests/multi_valued_keyword_terms.rs b/engine/crates/xerj-engine/tests/multi_valued_keyword_terms.rs
new file mode 100644
index 000000000..99b8d605a
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/multi_valued_keyword_terms.rs
@@ -0,0 +1,276 @@
+//! Regression tests for issue #332: a multi-valued keyword field was
+//! flattened into ONE FTS token.
+//!
+//! `{"tags": ["red", "blue"]}` on a `keyword` field used to be joined into the
+//! single string `"red blue"` before the FTS layer saw it
+//! (`memtable::extract_text_value` / `index::extract_field_text`, both doing
+//! `arr.join(" ")`). The keyword analyzer emits its whole input as one token,
+//! so the segment carried the term `"red blue"` and neither `"red"` nor
+//! `"blue"` existed as a posting. Every clause that projects to a whole-value
+//! `FtsQuery::Term` therefore missed the document once it was flushed —
+//! visible in the live engine as `multi_match: {"query": "red"}` returning 0
+//! hits while `multi_match: {"query": "red blue"}` returned 1, the exact
+//! inverse of Elasticsearch.
+//!
+//! Every test asserts the hit set is the same BEFORE and AFTER `flush()`:
+//! this class of defect is flush-dependent, and a post-flush-only assertion
+//! would pass on a build that had simply stopped indexing the field.
+
+use std::collections::BTreeSet;
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn req(q: Value) -> xerj_query::ast::SearchRequest {
+ parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request")
+}
+
+async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {
+ idx.search(&req(q.clone()))
+ .await
+ .unwrap()
+ .hits
+ .iter()
+ .map(|h| h.id.clone())
+ .collect()
+}
+
+fn expect(ids: &[&str]) -> BTreeSet<String> {
+ ids.iter().map(|s| s.to_string()).collect()
+}
+
+/// Two docs, one index:
+/// * `1` — multi-valued keyword `tags: ["red","blue"]`, multi-valued text
+/// `notes: ["alpha bravo","charlie delta"]`
+/// * `2` — the single-valued control, `tags: "red"`, `notes: "alpha bravo"`
+async fn seed(engine: &Engine, name: &str) -> std::sync::Arc<Index> {
+ let mut schema = Schema::empty();
+ schema.fields.push(FieldConfig::new("body", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("notes", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("tags", FieldType::Keyword));
+ engine.create_index(name, schema).unwrap();
+ let idx = engine.get_index(name).unwrap();
+ idx.index_document(
+ Some("1".into()),
+ json!({
+ "body": "hello",
+ "tags": ["red", "blue"],
+ "notes": ["alpha bravo", "charlie delta"]
+ }),
+ )
+ .await
+ .unwrap();
+ idx.index_document(
+ Some("2".into()),
+ json!({"body": "hello", "tags": "red", "notes": "alpha bravo"}),
+ )
+ .await
+ .unwrap();
+ idx
+}
+
+/// Run every case pre-flush and post-flush and require the same hit set.
+async fn assert_flush_parity(idx: &std::sync::Arc<Index>, cases: &[(Value, &[&str], &str)]) {
+ for (q, exp, label) in cases {
+ assert_eq!(
+ ids(idx, q).await,
+ expect(exp),
+ "{label}: PRE-flush (memtable) hit set wrong for {q}"
+ );
+ }
+ idx.flush().await.unwrap();
+ for (q, exp, label) in cases {
+ assert_eq!(
+ ids(idx, q).await,
+ expect(exp),
+ "{label}: POST-flush (segment) hit set wrong for {q}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn term_match_multi_match_see_every_keyword_array_element() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mvk_terms").await;
+
+ assert_flush_parity(
+ &idx,
+ &[
+ // The headline: an exact `term` for one element of the array.
+ (json!({"term": {"tags": "red"}}), &["1", "2"], "term red"),
+ (json!({"term": {"tags": "blue"}}), &["1"], "term blue"),
+ // `match` on a keyword field lowers to the same whole-value term.
+ (json!({"match": {"tags": "red"}}), &["1", "2"], "match red"),
+ (json!({"match": {"tags": "blue"}}), &["1"], "match blue"),
+ // `multi_match` — the clause that visibly missed doc 1 on `main`.
+ (
+ json!({"multi_match": {"query": "red", "fields": ["tags", "body"]}}),
+ &["1", "2"],
+ "multi_match red",
+ ),
+ (
+ json!({"multi_match": {"query": "blue", "fields": ["tags", "body"]}}),
+ &["1"],
+ "multi_match blue",
+ ),
+ // `terms` takes the stored-scan route and always agreed with ES;
+ // keep it as the in-binary control that the two routes now match.
+ (
+ json!({"terms": {"tags": ["blue"]}}),
+ &["1"],
+ "terms control",
+ ),
+ (
+ json!({"bool": {"must": [{"match": {"body": "hello"}}],
+ "filter": [{"terms": {"tags": ["red"]}}]}}),
+ &["1", "2"],
+ "bool must+filter control",
+ ),
+ ],
+ )
+ .await;
+}
+
+/// The joined string must NOT survive as a term. Pre-fix `"red blue"` was
+/// the ONLY term the segment held for doc 1, so this query matched it —
+/// exactly backwards from Elasticsearch, where no keyword value equals
+/// `"red blue"`.
+#[tokio::test]
+async fn joined_array_is_not_a_term() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mvk_joined").await;
+
+ assert_flush_parity(
+ &idx,
+ &[
+ (json!({"term": {"tags": "red blue"}}), &[], "term joined"),
+ (
+ json!({"multi_match": {"query": "red blue", "fields": ["tags"]}}),
+ &[],
+ "multi_match joined",
+ ),
+ ],
+ )
+ .await;
+}
+
+/// A phrase must not span two elements of an array — Lucene separates them by
+/// `position_increment_gap` (100), and so does the segment writer now.
+/// Pre-fix the memtable said 0 hits and the flushed segment said 1: the
+/// joined string put `bravo` and `charlie` at adjacent positions.
+#[tokio::test]
+async fn phrase_does_not_span_array_elements() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mvk_phrase").await;
+
+ assert_flush_parity(
+ &idx,
+ &[
+ (
+ json!({"match_phrase": {"notes": "bravo charlie"}}),
+ &[],
+ "match_phrase across the boundary",
+ ),
+ (
+ json!({"multi_match": {"query": "bravo charlie",
+ "fields": ["notes"], "type": "phrase"}}),
+ &[],
+ "multi_match phrase across the boundary",
+ ),
+ // ... but a phrase WITHIN one element still matches, in both docs
+ // for the first element and only doc 1 for the second.
+ (
+ json!({"match_phrase": {"notes": "alpha bravo"}}),
+ &["1", "2"],
+ "match_phrase inside element 0",
+ ),
+ (
+ json!({"multi_match": {"query": "charlie delta",
+ "fields": ["notes"], "type": "phrase"}}),
+ &["1"],
+ "multi_match phrase inside element 1",
+ ),
+ ],
+ )
+ .await;
+}
+
+/// A `terms` aggregation over a multi-valued keyword field buckets EVERY
+/// value, and the answer does not change at flush.
+#[tokio::test]
+async fn terms_agg_buckets_every_array_element() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mvk_agg").await;
+
+ let body = json!({
+ "query": {"match_all": {}},
+ "size": 0,
+ "aggs": {"t": {"terms": {"field": "tags"}}}
+ });
+ let expected = json!([
+ {"key": "red", "doc_count": 2},
+ {"key": "blue", "doc_count": 1}
+ ]);
+
+ for state in ["PRE-flush", "POST-flush"] {
+ if state == "POST-flush" {
+ idx.flush().await.unwrap();
+ }
+ let res = idx.search(&parse_request(&body).unwrap()).await.unwrap();
+ let buckets = res.aggs.as_ref().and_then(|a| a["t"]["buckets"].as_array())
+ .cloned()
+ .unwrap_or_default();
+ let trimmed: Vec<Value> = buckets
+ .iter()
+ .map(|b| json!({"key": b["key"], "doc_count": b["doc_count"]}))
+ .collect();
+ assert_eq!(
+ Value::Array(trimmed),
+ expected,
+ "{state}: terms agg over a multi-valued keyword field"
+ );
+ }
+}
+
+/// `_source` is stored verbatim and must come back as the array it went in
+/// as — the fix touches the inverted index, never the stored document.
+#[tokio::test]
+async fn source_round_trips_unchanged() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = seed(&engine, "mvk_source").await;
+
+ for state in ["PRE-flush", "POST-flush"] {
+ if state == "POST-flush" {
+ idx.flush().await.unwrap();
+ }
+ let res = idx
+ .search(&req(json!({"term": {"tags": "blue"}})))
+ .await
+ .unwrap();
+ assert_eq!(res.hits.len(), 1, "{state}: expected exactly doc 1");
+ assert_eq!(
+ res.hits[0].source["tags"],
+ json!(["red", "blue"]),
+ "{state}: _source must keep the original array"
+ );
+ }
+}
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 02a0c083d..1a2046afe 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -540,6 +540,143 @@ fn decode_field_meta_binary(bytes: &[u8]) -> Result<FieldMeta> {
})
}
+// ── Multi-valued field input ──────────────────────────────────────────────────
+
+/// Position gap inserted between two values of the same multi-valued field.
+///
+/// Mirrors Lucene/Elasticsearch's default `position_increment_gap` of 100.
+/// Without it, `["alpha bravo", "charlie"]` would put `bravo` at position 1 and
+/// `charlie` at position 2, so `match_phrase: "bravo charlie"` would match a
+/// phrase that exists in NEITHER value. 100 is large enough that no realistic
+/// `slop` bridges the boundary, and small enough to stay in the VByte fast path.
+pub const POSITION_INCREMENT_GAP: u32 = 100;
+
+/// The values a single document supplies for a single field.
+///
+/// A JSON document field is either a scalar or an array, and Elasticsearch
+/// treats the array as N independent values of the field — not as one
+/// concatenated value. That distinction is invisible for an analyzed `text`
+/// field only by accident (the standard tokenizer re-splits on the joining
+/// space); for a `keyword` field, which is indexed with the whole input as one
+/// token, joining `["red","blue"]` into `"red blue"` produces a term that
+/// matches neither `red` nor `blue`. See issue #332.
+///
+/// `One` exists so the overwhelmingly common single-valued case costs no extra
+/// allocation on the flush/merge path — the `String` is moved in as-is.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum FieldValues {
+ /// Exactly one value (a JSON scalar, or a one-element array).
+ One(String),
+ /// Two or more values (a JSON array). Never constructed with < 2 elements
+ /// by [`FieldValues::from_values`], but tolerated if built directly.
+ Many(Vec<String>),
+}
+
+impl FieldValues {
+ /// Build from an iterator, collapsing the single-value case to `One`.
+ pub fn from_values<I: IntoIterator<Item = String>>(values: I) -> Self {
+ let mut v: Vec<String> = values.into_iter().collect();
+ if v.len() == 1 {
+ FieldValues::One(v.pop().unwrap())
+ } else {
+ FieldValues::Many(v)
+ }
+ }
+
+ /// Iterate every value in document order.
+ pub fn iter(&self) -> impl Iterator<Item = &str> {
+ match self {
+ FieldValues::One(s) => std::slice::from_ref(s).iter(),
+ FieldValues::Many(v) => v.iter(),
+ }
+ .map(|s| s.as_str())
+ }
+
+ /// Number of values.
+ pub fn len(&self) -> usize {
+ match self {
+ FieldValues::One(_) => 1,
+ FieldValues::Many(v) => v.len(),
+ }
+ }
+
+ /// True when the field carries no value at all (only reachable via a
+ /// directly-constructed `Many(vec![])`).
+ pub fn is_empty(&self) -> bool {
+ match self {
+ FieldValues::One(s) => s.is_empty(),
+ FieldValues::Many(v) => v.iter().all(|s| s.is_empty()),
+ }
+ }
+}
+
+impl From<String> for FieldValues {
+ fn from(s: String) -> Self {
+ FieldValues::One(s)
+ }
+}
+
+impl From<&str> for FieldValues {
+ fn from(s: &str) -> Self {
+ FieldValues::One(s.to_owned())
+ }
+}
+
+impl From<Vec<String>> for FieldValues {
+ fn from(v: Vec<String>) -> Self {
+ FieldValues::from_values(v)
+ }
+}
+
+/// Analyze every value of one field for one document and fold the tokens into
+/// `field_data`.
+///
+/// Shared by [`FtsIndexWriter::add_document`] and
+/// [`FtsIndexWriter::add_documents_parallel`] so the serial and parallel paths
+/// cannot drift — the two used to hold independent copies of this loop.
+///
+/// Semantics per value:
+/// * positions restart from `base`, which advances past the previous value's
+/// last position plus [`POSITION_INCREMENT_GAP`];
+/// * the norm (field length) is the SUM of every value's token count, matching
+/// Lucene, so BM25 length-normalisation sees the whole field;
+/// * exactly one norm entry and one `total_docs` increment per (doc, field),
+/// however many values the field has.
+fn index_field_values<'a>(
+ field_data: &mut FieldData,
+ analyzer: &crate::analyzer::AnalyzerPipeline,
+ doc_ord: u32,
+ values: impl Iterator<Item = &'a str>,
+) {
+ let mut field_len: u64 = 0;
+ let mut base: u32 = 0;
+ let mut first = true;
+ for value in values {
+ if !first {
+ base = base.saturating_add(POSITION_INCREMENT_GAP);
+ }
+ first = false;
+ let tokens = analyzer.analyze(value);
+ let mut max_position = 0u32;
+ for token in &tokens {
+ field_data
+ .postings
+ .add_occurrence(&token.text, doc_ord, base.saturating_add(token.position));
+ max_position = max_position.max(token.position);
+ }
+ if !tokens.is_empty() {
+ base = base.saturating_add(max_position).saturating_add(1);
+ }
+ field_len += tokens.len() as u64;
+ }
+
+ field_data
+ .norms
+ .push((doc_ord, field_len.min(u16::MAX as u64) as u16));
+ field_data.stats.total_docs += 1;
+ field_data.stats.total_field_length += field_len;
+}
+
// ── FtsIndexWriter ────────────────────────────────────────────────────────────
/// Builds the FTS inverted index for one segment.
@@ -686,11 +823,15 @@ impl FtsIndexWriter {
/// Index all text fields of one document.
///
- /// `fields` is a map of field name → field text value.
+ /// `fields` is a map of field name → that field's [`FieldValues`]. Each
+ /// value is analyzed independently and separated from the next by
+ /// [`POSITION_INCREMENT_GAP`] — a JSON array is N values of the field, not
+ /// one joined value (issue #332).
+ ///
/// Fields not previously registered via `configure_field` are indexed
/// with the default configuration (standard analyzer, positions on).
- pub fn add_document(&mut self, doc_id: u32, fields: &HashMap<String, String>) {
- for (field_name, text) in fields {
+ pub fn add_document(&mut self, doc_id: u32, fields: &HashMap<String, FieldValues>) {
+ for (field_name, values) in fields {
let registry = Arc::clone(&self.registry);
// Ensure field entry exists
@@ -715,30 +856,13 @@ impl FtsIndexWriter {
.or_else(|| registry.get_analyzer("standard"))
.unwrap();
- let tokens = analyzer.analyze(text);
- let field_len = tokens.len() as u64;
-
- // Record norms (capped at u16::MAX)
- field_data
- .norms
- .push((doc_id, field_len.min(u16::MAX as u64) as u16));
-
- // Update field stats
- field_data.stats.total_docs += 1;
- field_data.stats.total_field_length += field_len;
-
- // Accumulate postings
- for token in &tokens {
- field_data
- .postings
- .add_occurrence(&token.text, doc_id, token.position);
- }
+ index_field_values(field_data, &analyzer, doc_id, values.iter());
}
}
/// V4 M4 — **parallel batch** add for flush time.
///
- /// Reshapes `(doc_id, field, text)` from row-major (per-doc) into
+ /// Reshapes `(doc_id, field, values)` from row-major (per-doc) into
/// column-major (per-field) then tokenises + builds per-field
/// postings in parallel via rayon. The underlying PostingsWriter
/// state is still single-threaded per field, but fields run in
@@ -758,24 +882,24 @@ impl FtsIndexWriter {
/// `Arc<serde_json::Value>`, …): this method never reads it.
pub fn add_documents_parallel<S: Sync>(
&mut self,
- docs: &[(String, HashMap<String, String>, S)],
+ docs: &[(String, HashMap<String, FieldValues>, S)],
) {
use rayon::prelude::*;
use std::collections::HashMap as StdHashMap;
- // Column-major reshape: field_name → Vec<(doc_ordinal, text)>.
+ // Column-major reshape: field_name → Vec<(doc_ordinal, values)>.
// Lookup-first so the common case (field already seen) skips the
// per-doc-field `field_name.clone()` the `entry()` API forced.
- let mut per_field: StdHashMap<String, Vec<(u32, &str)>> = StdHashMap::new();
+ let mut per_field: StdHashMap<String, Vec<(u32, &FieldValues)>> = StdHashMap::new();
for (ord, (_id, fields, _src)) in docs.iter().enumerate() {
- for (field_name, text) in fields {
+ for (field_name, values) in fields {
if let Some(v) = per_field.get_mut(field_name) {
- v.push((ord as u32, text.as_str()));
+ v.push((ord as u32, values));
} else {
per_field
.entry(field_name.clone())
.or_default()
- .push((ord as u32, text.as_str()));
+ .push((ord as u32, values));
}
}
}
@@ -806,7 +930,7 @@ impl FtsIndexWriter {
.map(|(k, v)| (k.clone(), v.config.clone()))
.collect();
- let per_field_vec: Vec<(String, Vec<(u32, &str)>)> = per_field.into_iter().collect();
+ let per_field_vec: Vec<(String, Vec<(u32, &FieldValues)>)> = per_field.into_iter().collect();
let built: Vec<(String, FieldData)> = per_field_vec
.into_par_iter()
@@ -829,17 +953,8 @@ impl FtsIndexWriter {
norms: Vec::with_capacity(entries.len()),
};
- for (doc_ord, text) in entries {
- let tokens = analyzer.analyze(text);
- let field_len = tokens.len() as u64;
- fd.norms
- .push((doc_ord, field_len.min(u16::MAX as u64) as u16));
- fd.stats.total_docs += 1;
- fd.stats.total_field_length += field_len;
- for token in &tokens {
- fd.postings
- .add_occurrence(&token.text, doc_ord, token.position);
- }
+ for (doc_ord, values) in entries {
+ index_field_values(&mut fd, &analyzer, doc_ord, values.iter());
}
(field_name, fd)
})
@@ -1741,14 +1856,14 @@ mod tests {
let mut writer = FtsIndexWriter::new(dir.path(), "seg0", registry);
- let docs: Vec<HashMap<String, String>> = vec![
- [("body".to_owned(), "the quick brown fox".to_owned())]
+ let docs: Vec<HashMap<String, FieldValues>> = vec![
+ [("body".to_owned(), FieldValues::from("the quick brown fox"))]
.into_iter()
.collect(),
- [("body".to_owned(), "the lazy dog".to_owned())]
+ [("body".to_owned(), FieldValues::from("the lazy dog"))]
.into_iter()
.collect(),
- [("body".to_owned(), "quick fox lazy dog".to_owned())]
+ [("body".to_owned(), FieldValues::from("quick fox lazy dog"))]
.into_iter()
.collect(),
];
@@ -1804,10 +1919,10 @@ mod tests {
writer.configure_field("title", cfg);
let docs = [
- [("title".to_owned(), "hello world".to_owned())]
+ [("title".to_owned(), FieldValues::from("hello world"))]
.into_iter()
.collect(),
- [("title".to_owned(), "hello rust".to_owned())]
+ [("title".to_owned(), FieldValues::from("hello rust"))]
.into_iter()
.collect(),
];
@@ -1842,10 +1957,10 @@ mod tests {
let dir = TempDir::new().unwrap();
let mut writer = FtsIndexWriter::new(dir.path(), "seg-safe", make_registry());
let fields = ["title.body-2", "@timestamp", "space field", "配置.名字"];
- let document: HashMap<String, String> = fields
+ let document: HashMap<String, FieldValues> = fields
.into_iter()
.enumerate()
- .map(|(index, field)| (field.to_owned(), format!("needle{index}")))
+ .map(|(index, field)| (field.to_owned(), FieldValues::One(format!("needle{index}"))))
.collect();
writer.add_document(0, &document);
writer.finish().unwrap();
@@ -1864,7 +1979,7 @@ mod tests {
fn write_legacy_raw_field_fixture(segment_dir: &Path, segment_id: &str, legacy_field: &str) {
let source_field = "legacy_fixture_source";
let mut writer = FtsIndexWriter::new(segment_dir, segment_id, make_registry());
- let document = [(source_field.to_owned(), "legacyneedle".to_owned())]
+ let document = [(source_field.to_owned(), FieldValues::from("legacyneedle"))]
.into_iter()
.collect();
writer.add_document(0, &document);
@@ -1926,9 +2041,9 @@ mod tests {
let (unsafe_name, legacy_alias) = collision_fields();
let source_fields = ["source_unsafe", "source_alias"];
let mut writer = FtsIndexWriter::new(segment_dir, segment_id, make_registry());
- let document = [
- (source_fields[0].to_owned(), "unsafeneedle".to_owned()),
- (source_fields[1].to_owned(), "aliasneedle".to_owned()),
+ let document: HashMap<String, FieldValues> = [
+ (source_fields[0].to_owned(), FieldValues::from("unsafeneedle")),
+ (source_fields[1].to_owned(), FieldValues::from("aliasneedle")),
]
.into_iter()
.collect();
@@ -1994,9 +2109,9 @@ mod tests {
let segment_id = "v2-collision";
let (unsafe_name, legacy_alias) = collision_fields();
let mut writer = FtsIndexWriter::new(&segment_dir, segment_id, make_registry());
- let document = [
- (unsafe_name.clone(), "unsafeneedle".to_owned()),
- (legacy_alias.clone(), "aliasneedle".to_owned()),
+ let document: HashMap<String, FieldValues> = [
+ (unsafe_name.clone(), FieldValues::from("unsafeneedle")),
+ (legacy_alias.clone(), FieldValues::from("aliasneedle")),
]
.into_iter()
.collect();
@@ -2066,7 +2181,7 @@ mod tests {
let mut writer = FtsIndexWriter::new(&segment_dir, segment_id, make_registry());
writer.add_document(
0,
- &[(field.to_owned(), "encodedneedle".to_owned())]
+ &[(field.to_owned(), FieldValues::from("encodedneedle"))]
.into_iter()
.collect(),
);
@@ -2204,10 +2319,12 @@ mod tests {
assert!(legacy_field_sidecar_path(&segment_dir, "seg-unsafe", field, "fst").is_none());
}
let mut writer = FtsIndexWriter::new(&segment_dir, "seg-unsafe", make_registry());
- let document: HashMap<String, String> = fields
+ let document: HashMap<String, FieldValues> = fields
.iter()
.enumerate()
- .map(|(index, field)| ((*field).to_owned(), format!("needle{index}")))
+ .map(|(index, field)| {
+ ((*field).to_owned(), FieldValues::One(format!("needle{index}")))
+ })
.collect();
writer.add_document(0, &document);
writer.publish_encoded_filename_layout().unwrap();
diff --git a/engine/crates/xerj-fts/src/lib.rs b/engine/crates/xerj-fts/src/lib.rs
index d37239bb3..f3d4cd533 100644
--- a/engine/crates/xerj-fts/src/lib.rs
+++ b/engine/crates/xerj-fts/src/lib.rs
@@ -32,7 +32,7 @@
//! use std::sync::Arc;
//! use xerj_fts::{
//! analyzer::AnalyzerRegistry,
-//! index::{FtsIndexWriter, FtsIndexReader},
+//! index::{FieldValues, FtsIndexWriter, FtsIndexReader},
//! search::{FtsSearcher, Query, TermQuery},
//! };
//!
@@ -42,7 +42,7 @@
//! let mut writer = FtsIndexWriter::new(dir.path(), "seg0", Arc::clone(&registry));
//!
//! let mut doc = HashMap::new();
-//! doc.insert("body".to_owned(), "the quick brown fox".to_owned());
+//! doc.insert("body".to_owned(), FieldValues::from("the quick brown fox"));
//! writer.add_document(0, &doc);
//! writer.finish().unwrap();
//!
@@ -75,7 +75,9 @@ pub use bm25::{
DEFAULT_K1,
};
-pub use index::{FieldIndexConfig, FtsIndexReader, FtsIndexWriter};
+pub use index::{
+ FieldIndexConfig, FieldValues, FtsIndexReader, FtsIndexWriter, POSITION_INCREMENT_GAP,
+};
pub use postings::{
DecodedPosting, PostingsReader, PostingsWriter, TermPostings, BLOCK_SIZE, SKIP_INTERVAL,
diff --git a/engine/crates/xerj-fts/src/search.rs b/engine/crates/xerj-fts/src/search.rs
index 15001d454..56abaa731 100644
--- a/engine/crates/xerj-fts/src/search.rs
+++ b/engine/crates/xerj-fts/src/search.rs
@@ -1555,7 +1555,7 @@ mod tests {
use super::*;
use crate::{
analyzer::AnalyzerRegistry,
- index::{FieldIndexConfig, FtsIndexReader, FtsIndexWriter},
+ index::{FieldIndexConfig, FieldValues, FtsIndexReader, FtsIndexWriter},
};
use std::collections::HashMap;
use tempfile::TempDir;
@@ -1581,9 +1581,10 @@ mod tests {
];
for (i, text) in docs.iter().enumerate() {
- let fields: HashMap<String, String> = [("body".to_owned(), text.to_string())]
- .into_iter()
- .collect();
+ let fields: HashMap<String, FieldValues> =
+ [("body".to_owned(), FieldValues::from(text.to_string()))]
+ .into_iter()
+ .collect();
writer.add_document(i as u32, &fields);
}
writer.finish().unwrap();
@@ -1749,9 +1750,9 @@ mod tests {
("morning news", "golf scores from sunday"),
];
for (i, (title, body)) in docs.iter().enumerate() {
- let fields: HashMap<String, String> = [
- ("title".to_owned(), title.to_string()),
- ("body".to_owned(), body.to_string()),
+ let fields: HashMap<String, FieldValues> = [
+ ("title".to_owned(), FieldValues::from(title.to_string())),
+ ("body".to_owned(), FieldValues::from(body.to_string())),
]
.into_iter()
.collect();
@@ -2004,7 +2005,7 @@ mod tests {
writer.configure_field("body", cfg);
writer.add_document(
0,
- &[("body".to_owned(), "Rust Systems".to_owned())]
+ &[("body".to_owned(), FieldValues::from("Rust Systems"))]
.into_iter()
.collect(),
);
@@ -2080,7 +2081,12 @@ mod tests {
.map(|i| format!("term{i:05}"))
.collect::<Vec<_>>()
.join(" ");
- writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());
+ writer.add_document(
+ 0,
+ &[("body".to_owned(), FieldValues::from(text))]
+ .into_iter()
+ .collect(),
+ );
writer.finish().unwrap();
let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());
@@ -2128,7 +2134,12 @@ mod tests {
.map(|i| format!("term{i:05}"))
.collect::<Vec<_>>()
.join(" ");
- writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());
+ writer.add_document(
+ 0,
+ &[("body".to_owned(), FieldValues::from(text))]
+ .into_iter()
+ .collect(),
+ );
writer.finish().unwrap();
let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());
@@ -2213,7 +2224,12 @@ mod tests {
for i in 0..DICTIONARY_TERMS {
text.push_str(&format!(" term{i:06}"));
}
- writer.add_document(0, &[("body".to_owned(), text)].into_iter().collect());
+ writer.add_document(
+ 0,
+ &[("body".to_owned(), FieldValues::from(text))]
+ .into_iter()
+ .collect(),
+ );
writer.finish().unwrap();
let reader = Arc::new(FtsIndexReader::open(dir.path(), "seg0", &["body"]).unwrap());