Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

300 KB
Raw Blame
//! In-memory inverted index for unflushed documents.
//!
//! The FTS memtable provides fast search over recently ingested documents
//! that have not yet been flushed to a segment on disk.  It uses a simple
//! HashMap-based inverted index with BM25 scoring, plus a columnar DocValues
//! store for O(N) term/range/agg queries without JSON parsing per document.

use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::sync::OnceLock;

use rustc_hash::{FxHashMap, FxHashSet};
use serde_json::Value;
use xerj_common::types::{FieldType, Schema};
use xerj_compress::field_codec::{FieldAnalyzer, FieldEncoding};
use xerj_fts::analyzer::{AnalyzerPipeline, AnalyzerRegistry, Token};
use xerj_fts::bm25::Bm25Scorer;

// ── Types ─────────────────────────────────────────────────────────────────────

/// A scored hit from the memtable.
#[derive(Debug, Clone)]
pub struct MemtableHit {
    pub doc_id: String,
    pub score: f32,
    /// WAL `seq_no` of the buffered document (`u64::MAX` when it can't be
    /// resolved).  #191 — the memtable's bounded FTS materialisation has to
    /// truncate under the SAME total order the final page sort uses
    /// (`score DESC, seq_no ASC, _id ASC`); score alone leaves the survivors
    /// of a tie decided by shard-walk order, which is not arrival order.
    pub seq_no: u64,
}

/// Order `hits` by the page key XERJ's final sort uses: `score DESC`, then
/// `seq_no ASC` (arrival order — the `_doc` analogue), then `_id ASC`
/// (`index.rs:16222-16228`).
///
/// #191 — any bounded materialisation must truncate under the SAME total order
/// the page is finally sorted by, otherwise the documents that survive a tie
/// are decided by whatever order the walk happened to visit them in.  Lucene
/// makes the same comparator total for the same reason
/// (`HitQueue.lessThan` breaks an equal score by the smaller doc id,
/// `lucene/core/src/java/org/apache/lucene/search/HitQueue.java:76-82`);
/// approach only, no code taken.
fn sort_hits_by_page_key(hits: &mut [MemtableHit]) {
    hits.sort_unstable_by(|a, b| {
        b.score
            .partial_cmp(&a.score)
            .unwrap_or(std::cmp::Ordering::Equal)
            .then_with(|| a.seq_no.cmp(&b.seq_no))
            .then_with(|| a.doc_id.cmp(&b.doc_id))
    });
}

#[cfg(test)]
mod ingest_memory_drain_tests {
    use super::*;
    use crate::ingest_memory::{Category, Ledger, Measurement};
    use std::sync::{Arc, Barrier};

    #[test]
    fn authoritative_active_moves_to_drained_through_blocked_finalize_success_and_error() {
        for inject_error in [false, true] {
            let mem = Arc::new(ShardedFtsMemtable::new());
            mem.insert(
                "doc-1".to_string(),
                &serde_json::json!({"body": "retained"}),
                &Schema::default(),
                1,
            );
            let shard = mem
                .shard_loads()
                .into_iter()
                .find(|(_, docs, _)| *docs != 0)
                .unwrap()
                .0;
            let ledger = Ledger::new();
            let charged_at_freeze = mem.size_bytes() as u64;
            ledger.observe(Category::MemtableActive, charged_at_freeze as usize);
            assert!(charged_at_freeze > 0);

            let ledger: &'static Ledger = Box::leak(Box::new(ledger));
            let (_entries, claim) = mem.freeze_shard_accounted_for_test(shard, false, ledger);
            // #1015 — the freeze claims the generations for the flush but
            // moves NO bytes: the docs stay reader-visible (and charged to
            // the memtable aggregate) for the whole segment build.  The
            // pre-#1015 drain zeroed this gauge immediately and parked the
            // bytes in FlushDrained for the build's duration instead.
            ledger.observe(Category::MemtableActive, mem.size_bytes());
            assert_eq!(
                ledger
                    .gauge(Category::MemtableActive, Measurement::Estimated)
                    .current,
                charged_at_freeze,
                "frozen generations stay charged through the build"
            );
            assert_eq!(
                ledger
                    .gauge(Category::FlushDrained, Measurement::Estimated)
                    .current,
                0,
                "no flush-pipeline charge until retire"
            );
            let entered = Arc::new(Barrier::new(2));
            let release = Arc::new(Barrier::new(2));
            std::thread::scope(|scope| {
                let entered_worker = Arc::clone(&entered);
                let release_worker = Arc::clone(&release);
                let handle = scope.spawn(move || -> Result<(), &'static str> {
                    // Mirror do_flush_shard's publish-side disposition: retire
                    // (bytes leave the aggregate; the Retained guard parks
                    // them in FlushDrained) on success, release (bytes never
                    // move) on failure.
                    let retained = if inject_error {
                        claim.release();
                        None
                    } else {
                        claim.retire()
                    };
                    entered_worker.wait();
                    release_worker.wait();
                    drop(retained);
                    if inject_error {
                        Err("injected finalizer error")
                    } else {
                        Ok(())
                    }
                });
                entered.wait();
                ledger.observe(Category::MemtableActive, mem.size_bytes());
                if inject_error {
                    // Released: the failed flush keeps its bytes charged to
                    // the memtable — the docs are still reader-visible and
                    // the next flush recycles the generations.
                    assert_eq!(
                        ledger
                            .gauge(Category::MemtableActive, Measurement::Estimated)
                            .current,
                        charged_at_freeze,
                        "released claim keeps its bytes charged"
                    );
                    assert_eq!(
                        ledger
                            .gauge(Category::FlushDrained, Measurement::Estimated)
                            .current,
                        0
                    );
                } else {
                    // Retired: the bytes left the aggregate and sit in
                    // FlushDrained exactly as long as the Retained guard
                    // lives.
                    assert_eq!(
                        ledger
                            .gauge(Category::MemtableActive, Measurement::Estimated)
                            .current,
                        0
                    );
                    assert!(
                        ledger
                            .gauge(Category::FlushDrained, Measurement::Estimated)
                            .current
                            > 0
                    );
                }
                release.wait();
                assert_eq!(handle.join().unwrap().is_err(), inject_error);
            });
            ledger.observe(Category::MemtableActive, mem.size_bytes());
            assert_eq!(
                ledger
                    .gauge(Category::FlushDrained, Measurement::Estimated)
                    .current,
                0,
                "Retained dropped — the flush pipeline no longer holds the bytes"
            );
            if inject_error {
                assert_eq!(
                    ledger
                        .gauge(Category::MemtableActive, Measurement::Estimated)
                        .current,
                    charged_at_freeze,
                    "a released claim's docs stay charged until a later flush retires them"
                );
            }
        }
    }
}

/// #1015 acceptance coverage for the frozen-generation contract.  The freeze
/// moves the active generation behind a claimed `FrozenGen` and returns the
/// flush snapshot; the docs NEVER leave the reader-visible set until
/// `retire()`.  These tests pin the three properties the design claims:
/// visibility through the (multi-second) segment build, deletes applied to a
/// mid-build doc, and recycle-not-stack on a failed flush's retry.
#[cfg(test)]
mod frozen_generation_tests {
    use super::*;
    use crate::ingest_memory::Ledger;
    use serde_json::json;

    fn test_ledger() -> &'static Ledger {
        Box::leak(Box::new(Ledger::new()))
    }

    fn insert(mem: &ShardedFtsMemtable, id: &str, body: &str, seq: u64) {
        mem.insert(
            id.to_string(),
            &json!({ "body": body }),
            &Schema::default(),
            seq,
        );
    }

    fn shard_of(mem: &ShardedFtsMemtable, id: &str) -> usize {
        let s = mem.shard_for_dynamic(id);
        assert!(
            mem.shards[s].read().active.contains(id),
            "test setup: {id} must sit in shard {s}'s active generation"
        );
        s
    }

    fn hit_ids(mem: &ShardedFtsMemtable, term: &str) -> Vec<String> {
        let mut ids: Vec<String> = mem
            .search_text(term, &["body"], 32)
            .into_iter()
            .map(|h| h.doc_id)
            .collect();
        ids.sort();
        ids
    }

    fn entry_ids(entries: &[(u64, String, Arc<Value>, Arc<[u8]>)]) -> Vec<String> {
        let mut ids: Vec<String> = entries.iter().map(|(_, id, _, _)| id.clone()).collect();
        ids.sort();
        ids
    }

    /// The core #1015 claim: between freeze and retire — the window the
    /// segment build runs in — the frozen docs stay queryable, and ingest into
    /// the same shard continues.  Pre-#1015 the drain removed them from the
    /// view for the whole build, which is what forced the publication guard to
    /// span it.
    #[test]
    fn frozen_generation_stays_visible_through_the_claim() {
        let mem = Arc::new(ShardedFtsMemtable::new());
        insert(&mem, "doc-frozen", "quokka frozen", 1);
        let shard = shard_of(&mem, "doc-frozen");
        let bytes_at_freeze = mem.size_bytes();
        assert!(bytes_at_freeze > 0);

        let (entries, claim) = mem.freeze_shard_accounted_for_test(shard, false, test_ledger());
        assert_eq!(entry_ids(&entries), vec!["doc-frozen"]);
        // The claim moved NO bytes: the aggregate still carries the frozen
        // generation until retire.
        assert_eq!(mem.size_bytes(), bytes_at_freeze);

        // Ingest continues into the shard while the (hypothetical) build runs.
        insert(&mem, "doc-active", "wallaby active", 2);

        assert!(
            mem.contains("doc-frozen"),
            "frozen doc left the view at freeze"
        );
        assert!(mem.contains("doc-active"));
        assert_eq!(hit_ids(&mem, "quokka"), vec!["doc-frozen"]);
        assert_eq!(hit_ids(&mem, "wallaby"), vec!["doc-active"]);
        assert_eq!(
            mem.all_doc_ids().len(),
            2,
            "the view is frozen + active, not active alone"
        );

        drop(claim.retire());
        assert!(
            !mem.contains("doc-frozen"),
            "retire is what removes the frozen docs"
        );
        assert!(!hit_ids(&mem, "quokka").contains(&"doc-frozen".to_string()));
        assert!(
            mem.contains("doc-active"),
            "retire must not touch the active generation"
        );
    }

    /// A delete landing on a doc mid-build is applied to the frozen generation
    /// it lives in, so the view drops the doc immediately.  The flush SNAPSHOT
    /// was taken at freeze and still carries it — pinned here explicitly: the
    /// publish-time version-map repoint owns that tombstone, exactly as it did
    /// for a delete racing the pre-#1015 drain (the race window is unchanged,
    /// only its length shrank).
    #[test]
    fn delete_during_the_claim_removes_the_doc_from_the_view() {
        let mem = Arc::new(ShardedFtsMemtable::new());
        insert(&mem, "doc-doomed", "numbat delete", 1);
        let shard = shard_of(&mem, "doc-doomed");

        let (entries, claim) = mem.freeze_shard_accounted_for_test(shard, false, test_ledger());
        mem.remove("doc-doomed");

        assert!(!mem.contains("doc-doomed"));
        assert!(
            hit_ids(&mem, "numbat").is_empty(),
            "the delete reaches the frozen generation"
        );
        assert_eq!(
            entry_ids(&entries),
            vec!["doc-doomed"],
            "the snapshot predates the delete by construction — the version-map \
             repoint at publish resolves it (same window as the pre-#1015 drain)"
        );

        drop(claim.retire());
        assert!(!mem.contains("doc-doomed"));
    }

    /// A failed flush releases its claim; the generation stays in the shard
    /// unclaimed, reader-visible, byte-accounted.  The NEXT freeze must recycle
    /// that generation rather than stack a second one beside it — otherwise a
    /// flaky object store turns every retry into another frozen generation and
    /// the query fan-out grows without bound.
    #[test]
    fn released_claim_is_recycled_not_stacked() {
        let mem = Arc::new(ShardedFtsMemtable::new());
        insert(&mem, "doc-retry", "potoroo retry", 1);
        let shard = shard_of(&mem, "doc-retry");
        let bytes_at_freeze = mem.size_bytes();

        let (_first, claim) = mem.freeze_shard_accounted_for_test(shard, false, test_ledger());
        claim.release();
        assert!(
            mem.contains("doc-retry"),
            "release keeps the docs in the view"
        );
        assert_eq!(mem.size_bytes(), bytes_at_freeze);

        let (entries, claim) = mem.freeze_shard_accounted_for_test(shard, false, test_ledger());
        assert_eq!(entry_ids(&entries), vec!["doc-retry"]);
        assert_eq!(
            mem.shards[shard].read().frozen.len(),
            1,
            "the retry recycled the released generation"
        );

        drop(claim.retire());
        assert!(!mem.contains("doc-retry"));
        assert!(mem.shards[shard].read().frozen.is_empty());
        assert_eq!(mem.size_bytes(), 0, "retire returns the generation's bytes");
    }
}

/// Entry stored in the memtable.
///
/// Post-M4.9 the `fields: HashMap<String, String>` was removed — it was
/// a per-doc allocation built at ingest time that duplicated data
/// already present in `source: Arc<Value>`.  Flush-time consumers
/// reconstruct it lazily from `source` via `extract_text_fields_from`,
/// so the hot ingest path does nothing more than a `Vec::push` of this
/// struct plus one `Arc::clone` of the pre-parsed source.
///
/// Post-M5.0 the struct carries `seq_no` so that `drain_with_sources`
/// can SORT the drained docs by global sequence number — this lets us
/// lift the WAL append OUT of the engine memtable write lock.  Pre-M5.0
/// the lock spanned WAL append + memtable push so both memtables saw
/// docs in identical order; now the WAL and engine memtable can be
/// advanced independently under different locks, and flush drains
/// recover the global order by seq_no sort.
#[derive(Debug, Clone)]
struct MemEntry {
    /// Global WAL sequence number assigned by the storage layer.
    /// Zero when unknown (e.g. test shims and single-doc `insert`).
    seq_no: u64,
    doc_id: String,
    /// Original source document as ingested (preserves arrays, nested objects, etc.).
    /// Stored as `Arc<Value>` so that turbo-ingest batches can share the same
    /// allocation across WAL serialisation and this memtable entry — no deep copy.
    ///
    /// M5.11 — in the ultra-turbo hot path this can be `Value::Null` as a
    /// placeholder.  When the caller supplies `source_bytes` (non-empty) we
    /// defer the JSON parse all the way until drain-for-flush time so the
    /// ingest path doesn't burn ~5 µs/doc on `serde_json::from_str`.
    source: Arc<Value>,
    /// M5.11 — raw NDJSON bytes for the document, kept alongside `source` so
    /// that drain can lazily parse them into `Value` if `source` is still
    /// `Value::Null`.  This lets the bulk hot path SKIP the doc-body parse
    /// entirely and push work to the background flush thread pool.
    source_bytes: Arc<[u8]>,
    /// M5.18 — read-side memo for the M5.11 deferred parse.  When `source`
    /// is `Null` and `source_bytes` is non-empty, the FIRST read that needs
    /// the parsed `Value` (`resolve_source*`, `get_doc_source_arc`) parses
    /// the bytes once and caches the resulting `Arc<Value>` here; every
    /// subsequent read/scan is an `Arc::clone` instead of a per-doc
    /// `serde_json::from_slice`.  Under the mixed read-under-write cluster a
    /// range/bool scan walks the whole buffered memtable, so without this
    /// memo each query re-parsed ~10⁵–10⁶ raw-bytes docs (live WARN:
    /// mem_admit=2151 ms spike on one range read).  Inline `OnceLock` →
    /// `OnceLock::new()` is a const, so the ingest hot path pays NO extra
    /// heap allocation; the cell only allocates a `Value` when actually
    /// read.  Empty (never initialised) for eager entries where `source`
    /// already holds the parsed tree.
    parsed_memo: OnceLock<Arc<Value>>,
    /// Approximate byte size for flush threshold tracking.
    size_bytes: usize,
}

/// Charge for one raw-bytes (turbo `_bulk`) memtable entry — issue #948.
///
/// The entry retains the full NDJSON payload plus the doc-id string twice
/// (`doc_id_index` key + `MemEntry.doc_id`) and the entry/index nodes; the
/// parsed `Value` and postings are deferred to the flush thread, so the
/// payload length IS the honest hot-path footprint. The historical flat
/// 800-byte estimate stays as the floor so compressed log docs (~115 B)
/// keep their exact M5.6/M5.17 charge and flush cadence — only docs larger
/// than the floor charge more, which is precisely the case where the flat
/// number let a ~10 KB mailbox doc count as 800 bytes while the
/// process-wide memtable budget believed it had 12× more headroom than it
/// did.
const RAW_BYTES_CHARGE_FLOOR: usize = 800;
/// `MemEntry` + Arc header + `doc_id_index` node allowance for one raw-bytes
/// entry. Sized from the struct fields (2×u64, 2×String, Arc<Value>,
/// Arc<[u8]>, OnceLock, usize ≈ 100 B) plus a hash-map node (~48 B).
const RAW_BYTES_ENTRY_OVERHEAD: usize = 160;

fn raw_bytes_charge(doc_id: &str, source_bytes: &[u8]) -> usize {
    RAW_BYTES_CHARGE_FLOOR.max(
        source_bytes
            .len()
            .saturating_add(doc_id.len())
            .saturating_add(RAW_BYTES_ENTRY_OVERHEAD),
    )
}

/// Reconstruct the (field_name → flattened text) map that pre-M4.9
/// `MemEntry` used to cache eagerly at ingest time.  Called only by
/// the legacy `drain_with_sources` / `drain` / `get_source` paths, which
/// are NOT the segment-FTS input path — use
/// [`extract_field_values_excluding`] for anything that will be indexed.
pub fn extract_text_fields_from(source: &Value) -> HashMap<String, String> {
    extract_text_fields_from_excluding(source, &std::collections::HashSet::new())
}

pub fn extract_text_fields_from_excluding(
    source: &Value,
    excluded: &std::collections::HashSet<String>,
) -> HashMap<String, String> {
    extract_field_values_excluding(source, excluded)
        .into_iter()
        .map(|(k, v)| (k, v.iter().collect::<Vec<_>>().join(" ")))
        .collect()
}

/// Extract the top-level source fields as the segment-FTS writer wants them:
/// one [`FieldValues`] per field, with a JSON **array preserved as N separate
/// values** rather than joined into one string.
///
/// This is the single walker feeding both segment-build paths — the flush path
/// (`do_flush_shard` → `add_documents_parallel`) and the merge path
/// (`run_merge_once`).  Before #332 each path had its own copy
/// (`memtable::extract_text_value` and `index::extract_field_text`), and both
/// copies did `arr.join(" ")`, so `{"tags":["red","blue"]}` was indexed by the
/// keyword analyzer as the single term `"red blue"` — matching neither `red`
/// nor `blue`.
///
/// It also carries #328's nested-exclusion pruning, which the merge path's own
/// copy used to do inline: a field whose value is an OBJECT with an excluded
/// descendant (a `dense_vector` mapped at `passages.vec`) is flattened from a
/// PRUNED copy of that object, so the segment builder never resurrects the
/// vector's decimal components as a term under the parent's name. Unifying the
/// two paths onto this walker would otherwise have dropped that pruning on the
/// merge path (`.passages.fst` back to ~398 KB); doing it here gives it to the
/// flush path too, matching the live-insert path (`collect_text_fields`). It
/// only fires for objects, so #332's array handling is untouched.
pub fn extract_field_values_excluding(
    source: &Value,
    excluded: &std::collections::HashSet<String>,
) -> HashMap<String, xerj_fts::index::FieldValues> {
    use xerj_fts::index::FieldValues;
    let mut out = HashMap::new();
    if let Some(obj) = source.as_object() {
        for (key, val) in obj {
            if excluded.contains(key) {
                continue;
            }
            // #328 — object field with an excluded descendant: flatten a pruned
            // copy so a nested `dense_vector` cannot leak into the parent
            // object's term dictionary. Arrays never enter this arm, so the
            // #332 split below still owns every multi-valued keyword field.
            if val.is_object() && has_excluded_descendant(key, excluded) {
                let text = extract_text_value_excluding(val, key, excluded);
                if !text.is_empty() {
                    out.insert(key.clone(), FieldValues::One(text));
                }
                continue;
            }
            if let Some(values) = extract_field_values(val) {
                out.insert(key.clone(), values);
            }
        }
    }
    out
}

/// Convert one JSON value into the values the FTS writer should index for it.
///
/// * scalars → `One`
/// * arrays → one entry per element, recursively flattened (a nested array is
///   itself multi-valued in ES; an object element is JSON-encoded as before)
/// * objects → `One(json)`, unchanged: the root-level JSON blob that
///   flattened-style whole-object queries rely on
/// * `null` / empty → `None`, i.e. the field is not indexed for this doc
///
/// Empty strings are dropped INSIDE an array too — the pre-#332 code produced
/// them only as a by-product of `join(" ")` on `[null, "x"]`, and an empty
/// keyword token would otherwise become a real, matchable term.
fn extract_field_values(val: &Value) -> Option<xerj_fts::index::FieldValues> {
    use xerj_fts::index::FieldValues;
    match val {
        Value::Array(arr) => {
            let mut values: Vec<String> = Vec::with_capacity(arr.len());
            collect_array_values(arr, &mut values);
            if values.is_empty() {
                None
            } else {
                Some(FieldValues::from_values(values))
            }
        }
        _ => {
            let text = extract_text_value(val);
            if text.is_empty() {
                None
            } else {
                Some(FieldValues::One(text))
            }
        }
    }
}

fn collect_array_values(arr: &[Value], out: &mut Vec<String>) {
    for element in arr {
        match element {
            // A nested array contributes its own elements as separate values —
            // ES flattens arrays of arrays for indexing purposes.
            Value::Array(inner) => collect_array_values(inner, out),
            _ => {
                let text = extract_text_value(element);
                if !text.is_empty() {
                    out.push(text);
                }
            }
        }
    }
}

/// Interned document identifier.
///
/// One `Arc<str>` is allocated per document at insert time and its
/// pointer is cloned (a relaxed refcount bump, no `malloc`) into every
/// doc-id-keyed map — the postings lists, `field_lengths`, and
/// `doc_id_index`.  Pre-interning, `insert_analyzed` paid a fresh
/// `String` heap allocation of the doc id for EVERY (doc × token) it
/// pushed into a posting list plus one per (doc × field) length entry —
/// profiled as the dominant remaining under-write-lock cost on the
/// 1M×c8 bulk ceiling (~40 ms / 10k-doc batch).  Interning collapses
/// that to a single allocation per doc; the per-token work becomes a
/// pointer bump inside the shard critical section.
///
/// `Arc<str>: Borrow<str>` and its `Hash`/`Eq` forward to the pointed-to
/// `str`, so every existing lookup by `&str` (`remove`, `contains_key`,
/// BM25 `field_lengths` probe) is unchanged and byte-for-byte
/// identical to the `String`-keyed maps it replaces.
type DocId = Arc<str>;

/// Posting list entry: doc_id → term frequency.
///
/// FxHashMap (rustc-hash) instead of the SipHash-keyed std HashMap:
/// perf profiling of the c8 bulk-ingest ceiling showed >21% of on-CPU
/// time inside `core::hash::sip` + `hash_one` for the nested posting
/// merges — all executed under the memtable shard write locks.  These
/// maps are never exposed to untrusted-key HashDoS (doc ids and field
/// names of a single tenant's own documents), so the DoS-resistant
/// hasher buys nothing here.
type PostingList = FxHashMap<DocId, u32>; // doc_id → tf

// ── DocValues ─────────────────────────────────────────────────────────────────

/// Number of raw string samples to collect per field before running
/// smart encoding analysis.
// Intentional scaffolding for the deferred field-encoding-analysis subsystem
// (`collect_sample` + the `pub` samples/analyzed/analyzed_encodings columns); the
// bounded-delta read-side fold superseded its ingest-time call site, but the writer
// half and its public columns are retained, so silence dead_code.
#[allow(dead_code)]
const ANALYSIS_THRESHOLD: usize = 1000;

/// Columnar doc-values store, one column per field, one row per document
/// (in insertion order, parallel to `FtsMemtable::docs`).
///
/// This enables O(N * simple_compare) term/range queries and O(N) aggregations
/// instead of O(N * JSON_parse) full-document scans.
///
/// After `ANALYSIS_THRESHOLD` values are collected per field, `FieldAnalyzer`
/// is run to determine the optimal encoding.  Subsequent writes use the
/// chosen encoding.
#[derive(Default)]
pub struct DocValues {
    /// field → per-doc numeric value (for Long, Double, Date fields and any
    /// field whose value can be parsed as f64).
    pub numeric: FxHashMap<String, Vec<Option<f64>>>,
    /// field → per-doc keyword value (for Keyword, IP, and any string field).
    pub keyword: FxHashMap<String, Vec<Option<String>>>,
    /// Bounded-delta maintained per-value counts, distinct-value sets, and
    /// sorted-numeric range indexes for the columns above.  Kept behind a
    /// `Mutex` (interior mutability) so the query path folds only the
    /// column positions appended since the last query — O(delta), not
    /// O(memtable) — under the shard READ lock, instead of the old
    /// `counts_dirty` full-rebuild under an all-shard WRITE lock.  That
    /// full-rebuild + write-lock serialisation WAS the residual
    /// terms/cardinality/range/bool read-under-write p99 term.  See
    /// [`CountState`] and [`DocValues::with_keyword_field`].
    ///
    /// The ingest hot path (`push_field`) does NOT touch this — it only
    /// appends to the raw columns, so per-doc ingest cost is unchanged;
    /// the fold happens lazily on the read side over just the delta.
    pub counts: parking_lot::Mutex<CountState>,
    /// Analyzed field encodings — built lazily after sufficient samples.
    pub analyzed_encodings: HashMap<String, FieldEncoding>,
    /// Raw string samples per field for deferred analysis (cleared after analysis).
    pub samples: HashMap<String, Vec<String>>,
    /// Whether analysis has been performed for each field.
    pub analyzed: HashMap<String, bool>,
    /// Fields that have EVER carried an array value in this memtable.
    /// The keyword column stores only an array's FIRST scalar, so any
    /// columnar fast path whose brute twin fans out over every element
    /// (terms/cardinality aggs) must bail for these fields.  Never
    /// removed on delete — conservative (a stale flag only costs the
    /// fast path, not correctness).
    pub array_fields: FxHashSet<String>,
    /// Fields whose keyword column has EVER contained a whitespace-bearing
    /// value.  Such a column stores an analyzed-text field's full source
    /// string verbatim; the `term` / `terms` / fused-`bool` doc-values fast
    /// paths compare against a single token, which can never equal a
    /// multi-token value, so they MUST bail to the scan path for these
    /// fields.  Computed ONCE at insert time (see `push_field`) instead of
    /// the old O(N) `col.iter().any(is_whitespace)` prescan every query ran.
    /// Never cleared on delete — conservative (a stale flag only sends the
    /// query down the always-correct scan path, it can't change results),
    /// mirroring `array_fields`.
    pub keyword_has_whitespace: FxHashSet<String>,
    /// Incremental per-(field, order) sorted-candidate cache for the
    /// field-sorted match_all memtable arm — see
    /// `FtsMemtable::sort_candidates_cached`.  Keyed
    /// `"{field}\u{1}{asc|desc}"`.  Reset with the rest of `DocValues`
    /// at drain; cleared by `remove_at` (positions shift).
    ///
    /// Step 4: behind a `Mutex` so the sorted-`match_all` fast path can
    /// extend the cache under the shard's READ lock instead of forcing a
    /// write lock across all 16 shards (which serialised every sorted read
    /// against concurrent ingest — the residual `match_all` read-under-write
    /// p99 term).  The shard write lock is exclusive, so `push_field` /
    /// `remove_at` mutate the columns with no concurrent reader; two
    /// concurrent readers under the shard read lock serialise briefly on
    /// this leaf mutex while one folds in the new docs.
    pub sort_cand_cache: parking_lot::Mutex<FxHashMap<String, SortCandCache>>,
}

/// State of one incremental per-shard sorted-candidate extraction: the
/// shard's top-`cap` (+boundary ties) by sort key plus up to `cap`
/// missing-key positions, extended by O(new docs) per query instead of
/// re-walking the whole shard (the O(memtable) epoch-memo walk was
/// 15-30 ms/query at a drain-lagged 300-500 k-doc memtable — the
/// residual match_all read-under-write p99 term).
pub struct SortCandCache {
    /// Docs positions `[0, seen_docs)` are already folded in.
    pub seen_docs: usize,
    /// The cap this cache was built for; a larger request rebuilds.
    pub cap: usize,
    /// Candidate pool: (key, docs index).  Kept cut to ~2×cap between
    /// queries (page-order sort + boundary-tie truncate on overflow).
    pub top: Vec<(f64, usize)>,
    /// Up to `cap` positions missing the sort key.
    pub missing: Vec<usize>,
    /// A value failed to normalise (or the field carried arrays) — the
    /// shard is ineligible until the next drain resets the memtable.
    pub poisoned: bool,
}

/// Amortised sorted numeric index for O(log n + tail) memtable range
/// COUNTs.  A sorted run plus a bounded unsorted `tail`; the tail is
/// merged into the run once it grows past `TAIL_MERGE`, so a range count
/// is `partition_point` bisects over the run plus a linear scan of the
/// (≤ `TAIL_MERGE`) tail — never an O(memtable) column walk.  The merge
/// is O(run+tail) but amortised across `TAIL_MERGE` appends, and it runs
/// on the READ side (bounded-delta fold), never on the ingest hot path.
/// Bounded-delta maintained aggregation state for a [`DocValues`] store:
/// per-value keyword counts, distinct-value sets (exact cardinality), and
/// numeric-value counts.  Folded incrementally + **PER FIELD** on the read
/// side — `kw_built`/`num_built` record how many leading positions of each
/// field's column are already reflected, so a query folds only the newly-
/// appended tail of the ONE field it touches (O(delta)), never the whole
/// memtable and never other fields' columns.
///
/// Per-field folding matters: a `terms`/`cardinality` query on a low-
/// cardinality field (`model`, `status`) must NOT pay to fold a high-
/// cardinality sibling column (`cost_usd` doubles stringified into the
/// keyword store) — folding everything on every query regressed the very
/// cells this exists to serve.
#[derive(Default)]
pub struct CountState {
    /// per keyword-field: column positions already folded into the maps.
    kw_built: FxHashMap<String, usize>,
    /// per numeric-field: column positions already folded into numeric_counts.
    num_built: FxHashMap<String, usize>,
    /// field → distinct keyword value → live doc count.
    pub keyword_counts: FxHashMap<String, FxHashMap<String, u32>>,
    /// field → numeric-bits → live doc count.
    pub numeric_counts: FxHashMap<String, FxHashMap<u64, u32>>,
    /// field → set of distinct keyword values (exact cardinality).
    pub keyword_set: FxHashMap<String, FxHashSet<String>>,
}

impl DocValues {
    /// Fold the keyword column for ONE `field` up to date (counts + set).
    /// O(positions appended to that field's column since the last fold).
    fn fold_keyword_field(&self, cs: &mut CountState, field: &str) {
        let Some(col) = self.keyword.get(field) else {
            return;
        };
        let built = cs.kw_built.get(field).copied().unwrap_or(0);
        if built >= col.len() {
            return;
        }
        let counts = cs.keyword_counts.entry(field.to_string()).or_default();
        let set = cs.keyword_set.entry(field.to_string()).or_default();
        for s in col[built..].iter().flatten() {
            *counts.entry(s.clone()).or_insert(0) += 1;
            if !set.contains(s) {
                set.insert(s.clone());
            }
        }
        cs.kw_built.insert(field.to_string(), col.len());
    }

    /// Fold the numeric column for ONE `field` up to date (numeric_counts).
    fn fold_numeric_field(&self, cs: &mut CountState, field: &str) {
        let Some(col) = self.numeric.get(field) else {
            return;
        };
        let built = cs.num_built.get(field).copied().unwrap_or(0);
        if built >= col.len() {
            return;
        }
        let counts = cs.numeric_counts.entry(field.to_string()).or_default();
        for f in col[built..].iter().flatten() {
            *counts.entry(f.to_bits()).or_insert(0) += 1;
        }
        cs.num_built.insert(field.to_string(), col.len());
    }

    /// Fold ONE keyword field's maps up to date, then run `f` with a
    /// read-only borrow of the shared `CountState`.  Takes only `&self`
    /// (interior mutability via the `counts` Mutex), so the query path
    /// holds the shard READ lock — never the all-shard WRITE lock the old
    /// full-rebuild `&mut self` count path forced.
    pub fn with_keyword_field<R>(&self, field: &str, f: impl FnOnce(&CountState) -> R) -> R {
        let mut cs = self.counts.lock();
        self.fold_keyword_field(&mut cs, field);
        f(&cs)
    }

    /// Fold ONE numeric field's counts up to date, then run `f`.
    pub fn with_numeric_field<R>(&self, field: &str, f: impl FnOnce(&CountState) -> R) -> R {
        let mut cs = self.counts.lock();
        self.fold_numeric_field(&mut cs, field);
        f(&cs)
    }

    /// Reset the maintained maps.  Called on delete (`remove_at`) because a
    /// positional shift invalidates the append-only watermarks; the next
    /// query re-folds the (now smaller) columns from scratch.  Deletes are
    /// rare (never on the append-only bulk path), so this is trivially
    /// correct at negligible amortised cost.
    fn reset_counts(&mut self) {
        *self.counts.get_mut() = CountState::default();
    }
}

impl DocValues {
    /// Append values for all fields in `source` to the columnar store.
    ///
    /// `doc_index` is the index that this document will occupy in `docs`.
    /// All existing columns are padded to length `doc_index` with `None`
    /// before appending so that every column stays aligned with `docs`.
    fn push(&mut self, source: &Value, doc_index: usize) {
        if let Some(obj) = source.as_object() {
            for (key, val) in obj {
                self.push_field(key, val, doc_index);
            }
        }
    }

    /// Collect a string sample for a field and trigger analysis when threshold is reached.
    // Not on any current call path: the bounded-delta read-side fold superseded the
    // ingest-time sampling, but this is the writer for the `pub` samples/analyzed/
    // analyzed_encodings scaffolding (and the sole user of `FieldAnalyzer`), so it is
    // retained intentionally rather than deleted.
    #[allow(dead_code)]
    fn collect_sample(&mut self, field: &str, value: &str) {
        if self.analyzed.get(field).copied().unwrap_or(false) {
            // Already analyzed — no more samples needed.
            return;
        }
        let samples = self.samples.entry(field.to_string()).or_default();
        samples.push(value.to_string());

        if samples.len() >= ANALYSIS_THRESHOLD {
            // Run the smart analyzer on all collected samples.
            let analyzer = FieldAnalyzer::default();
            let refs: Vec<&str> = samples.iter().map(|s| s.as_str()).collect();
            let encoding = analyzer.analyze(field, &refs);
            self.analyzed_encodings.insert(field.to_string(), encoding);
            self.analyzed.insert(field.to_string(), true);
            // Free samples memory now that we have an encoding.
            self.samples.remove(field);
        }
    }

    fn push_field(&mut self, field: &str, val: &Value, doc_index: usize) {
        // V4 M4 / bounded-delta: ingest-path push_field is reduced to the
        // minimum — raw column storage only.  The `keyword_counts`,
        // `numeric_counts`, `keyword_set`, and `samples` maps are all
        // **populated lazily** on the read side via `with_keyword_field` /
        // `with_numeric_field` (per-field bounded-delta fold over just the
        // appended tail of the touched field).  This keeps the
        // hot ingest path at ~1 Vec push per field — no per-doc HashMap
        // churn — so ingest throughput is unchanged.
        // Column keys (`field`) recur across every doc, so `entry_no_clone`
        // skips the per-doc `field.to_string()` allocation on the hot path.
        match val {
            Value::Number(n) => {
                let col = entry_no_clone(&mut self.numeric, field, Default::default);
                pad_to(col, doc_index);
                col.push(n.as_f64());
                // Keep the keyword column populated too for mixed-type
                // access but defer the set/count maps.
                let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
                pad_to(kcol, doc_index);
                kcol.push(Some(n.to_string()));
            }
            Value::String(s) => {
                // Step 2: cache the analyzed-text eligibility flag at insert
                // time so the term/terms/bool fast paths skip the per-query
                // whole-column whitespace prescan.
                if s.contains(char::is_whitespace) && !self.keyword_has_whitespace.contains(field) {
                    self.keyword_has_whitespace.insert(field.to_string());
                }
                let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
                pad_to(kcol, doc_index);
                kcol.push(Some(s.clone()));
                // If the string looks numeric, also index it in the numeric column.
                if let Ok(f) = s.parse::<f64>() {
                    let ncol = entry_no_clone(&mut self.numeric, field, Default::default);
                    pad_to(ncol, doc_index);
                    ncol.push(Some(f));
                }
            }
            Value::Bool(b) => {
                // Numeric column FIRST, matching the on-disk segment
                // encoding (`build_doc_value_columns` in index.rs: a
                // boolean is always the f64 bit-pattern of 1.0/0.0, never
                // a keyword-only value) — the disk-segment builder reads
                // straight from the same `Value` tree and has always
                // gotten this right, but the memtable's own in-memory
                // representation never populated `numeric` for booleans at
                // all, only `keyword`. Any query still hitting a
                // memtable-resident boolean doc through a numeric-column
                // path (`ScoreEval::Bool`, `doc_values_numeric_count`,
                // range/terms-agg fast paths) saw the field as absent —
                // found empirically: a real OpenSearch Dashboards filter
                // pill on a boolean field returned 0 hits against
                // freshly-bulk-imported data (matches every
                // real-world/OSD import shape — natural background flush,
                // no explicit `_flush` — while the aggregation reading
                // the SAME field correctly bucketed true/false, since
                // aggregations use a different, disk-segment-only read
                // path that was never affected).
                let ncol = entry_no_clone(&mut self.numeric, field, Default::default);
                pad_to(ncol, doc_index);
                ncol.push(Some(if *b { 1.0 } else { 0.0 }));
                let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
                pad_to(kcol, doc_index);
                kcol.push(Some(b.to_string()));
            }
            Value::Array(arr) => {
                // Flatten: store the first element (or None for empty arrays).
                // Each element is treated as a separate value for aggregations,
                // but for term/range queries we store the first scalar found.
                if !self.array_fields.contains(field) {
                    self.array_fields.insert(field.to_string());
                }
                let first_num = arr.iter().find_map(|v| v.as_f64());
                let first_str: Option<String> = arr.iter().find_map(|v| match v {
                    Value::String(s) => Some(s.clone()),
                    Value::Number(n) => Some(n.to_string()),
                    Value::Bool(b) => Some(b.to_string()),
                    _ => None,
                });

                let ncol = entry_no_clone(&mut self.numeric, field, Default::default);
                pad_to(ncol, doc_index);
                ncol.push(first_num);

                if let Some(ref s) = first_str {
                    if s.contains(char::is_whitespace)
                        && !self.keyword_has_whitespace.contains(field)
                    {
                        self.keyword_has_whitespace.insert(field.to_string());
                    }
                    // keyword_set is maintained lazily on the read side
                    // (bounded-delta fold of the keyword column, which
                    // already receives `first_str` below).
                }
                let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
                pad_to(kcol, doc_index);
                kcol.push(first_str);
            }
            Value::Null | Value::Object(_) => {
                // Push None for null/object fields so columns stay aligned.
                let ncol = entry_no_clone(&mut self.numeric, field, Default::default);
                pad_to(ncol, doc_index);
                ncol.push(None);

                let kcol = entry_no_clone(&mut self.keyword, field, Default::default);
                pad_to(kcol, doc_index);
                kcol.push(None);
            }
        }
    }

    /// Remove the entry at `doc_index` from all columns (called on delete).
    fn remove_at(&mut self, doc_index: usize) {
        // Positions shift left — every cached sorted-candidate pool's
        // `(key, docs-index)` pairs go stale.  Deletes are rare on the
        // hot ingest path; a wholesale clear is simplest-correct.
        self.sort_cand_cache.lock().clear();
        // The bounded-delta count/set/range maps track leading column
        // positions by count; a positional shift invalidates the
        // watermarks, so reset and let the next read re-fold the (now
        // smaller) columns.  Keeps counts/sets/range EXACTLY equal to a
        // full recount after a delete (not an estimate).
        self.reset_counts();
        for col in self.numeric.values_mut() {
            if doc_index < col.len() {
                col.remove(doc_index);
            }
        }
        for col in self.keyword.values_mut() {
            if doc_index < col.len() {
                col.remove(doc_index);
            }
        }
        // analyzed_encodings, samples, analyzed are not updated on remove —
        // they are statistical summaries, not per-document state.
    }

    /// Return a reference to the analyzed field encodings map.
    pub fn field_encodings(&self) -> &HashMap<String, FieldEncoding> {
        &self.analyzed_encodings
    }
}

// ── ShardedFtsMemtable ───────────────────────────────────────────────────────
//
// A 16-shard wrapper over FtsMemtable.  The ingest hot path takes only
// one shard's mutex for the entire batch it's pushing, so N concurrent
// bulk clients pushing to N different shards run truly in parallel on
// the write side.  Query path acquires every shard sequentially and
// aggregates — the 16 lock acquisitions × ~100 ns each are negligible
// compared to actual query work and keep the public API point-in-time
// consistent.

/// Default number of shards in the engine memtable when callers don't
/// supply one. Must be a power of two so `hash & (N-1)` picks the shard.
/// 16 overlaps cleanly with up to 16 concurrent bulk clients without
/// significant collision (doc_id hash distribution is uniform).
///
/// Production callers do NOT use this default — they pass the operator-
/// tuned value via `with_registry_and_shards(registry, n)` from
/// `Config.engine.ingest_shards`. This constant survives only to keep
/// `ShardedFtsMemtable::new()` and `with_registry()` ergonomic for
/// tests and tools that don't care.
const DEFAULT_ENGINE_MEMTABLE_SHARDS: usize = 16;

/// Sharded engine FTS memtable.
///
/// Internally holds `num_shards` (set at construction) independent
/// `parking_lot::RwLock<FtsMemtable>` instances.  Batches in the turbo
/// ingest path route to exactly one shard (by hash of the first
/// doc_id) and take the **write** lock; query paths iterate all shards
/// under **read** locks, so 32 concurrent queries scale with core
/// count instead of serialising through one exclusive mutex per shard.
///
/// M5.19 — pre-M5.19 this was `Mutex<FtsMemtable>`, which made the
/// concurrent QPS bench collapse from ~5 k QPS (ES) / ~5 k QPS
/// (single-threaded xerj) down to 1 QPS (xerj under load) because
/// every query serialised through the same mutex sweep across 16
/// shards.  With RwLock reader-writer separation, 32 clients can
/// run search_inner concurrently.
///
/// Drain methods re-order the per-shard outputs by global WAL seq_no
/// so the downstream segment writer observes docs in canonical order
/// even though different shards saw pushes at different wall-clock
/// instants.
pub struct ShardedFtsMemtable {
    shards: Vec<parking_lot::RwLock<ShardGenerations>>,
    shard_mask: usize,
    /// Shared aggregate of all shards' `total_bytes` for this index — every
    /// shard mirrors its deltas here so the sampler reads one atomic.
    aggregate_bytes: Arc<AtomicUsize>,
}

/// One shard's reader-visible generations (#1015).
///
/// `active` receives every new write.  A flush FREEZES the active generation
/// (moves it into `frozen`, O(1) pointer move) and installs a fresh `active`,
/// so ingest continues on an empty memtable while the segment is being built —
/// but the frozen docs stay reader-visible until the flush PUBLISHES, when the
/// frozen generation is retired inside the same publication bracket that
/// swaps the segment into the snapshot.  This removes the drain→install
/// invisibility window that previously forced the flush to hold the
/// collection-publication guard across its whole finalize.
///
/// Readers hold the shard's outer read lock across ALL generations of that
/// shard, so a freeze or retire can never split their view.  A freeze does
/// not change the visible doc set (the docs move, the reader still sees
/// them), so it needs no publication bracket; a retire does, and the
/// capture-boundary validation from #1014 makes a straddled reader retry.
///
/// Frozen generations are individually locked (not immutable) so that
/// `remove` of a doc sitting in one behaves exactly as it does on the active
/// generation — tombstone the version map, strip the live structures, keep
/// the ghost accumulators.  A flush that fails leaves its frozen generations
/// in place; the shard's next flush drains them again (see
/// [`ShardedFtsMemtable::freeze_shard_accounted`]).
pub(crate) struct ShardGenerations {
    /// The generation new writes land in.  Replaced (O(1)) by each freeze.
    pub(crate) active: FtsMemtable,
    /// Former actives being flushed, oldest first.  Retired at publish.
    pub(crate) frozen: Vec<FrozenGen>,
    /// Ingredients for installing a fresh active generation at each freeze
    /// (every generation of a shard shares the same registry and byte
    /// aggregate — see [`FtsMemtable::with_registry_and_aggregate`]).
    registry: Arc<AnalyzerRegistry>,
    aggregate: Arc<AtomicUsize>,
}

/// One frozen generation plus its ownership state.
pub(crate) struct FrozenGen {
    pub(crate) gen: Arc<parking_lot::RwLock<FtsMemtable>>,
    /// `true` while the flush that drained this generation is in flight.
    /// Unclaimed generations are leftovers of a failed flush and join the
    /// next freeze's drain set.
    pub(crate) claimed: bool,
}

/// The flush's handle on the generations it drained (#1015).
///
/// Produced by [`ShardedFtsMemtable::freeze_shard_accounted`].  The flush
/// holds the claim across the segment build; `retire` (inside the
/// publication bracket, alongside the snapshot swap) drops the generations
/// from the reader view, and `release` (failure) leaves them visible and
/// recyclable.  A claim that is simply dropped releases — the safe failure
/// mode keeps the docs reader-visible.
pub(crate) struct FrozenClaim {
    memtable: Arc<ShardedFtsMemtable>,
    shard_idx: usize,
    gens: Vec<Arc<parking_lot::RwLock<FtsMemtable>>>,
    finished: bool,
    /// Where `retire` charges the freed bytes.  Defaults to the thread's
    /// active ingest ledger; tests override it to assert attribution.
    ledger: Option<&'static crate::ingest_memory::Ledger>,
}

impl Default for ShardedFtsMemtable {
    fn default() -> Self {
        Self::new()
    }
}

impl ShardedFtsMemtable {
    pub fn new() -> Self {
        let registry = Arc::new(AnalyzerRegistry::default());
        Self::with_registry(registry)
    }

    pub fn with_registry(registry: Arc<AnalyzerRegistry>) -> Self {
        Self::with_registry_and_shards(registry, DEFAULT_ENGINE_MEMTABLE_SHARDS)
    }

    pub fn with_registry_and_shards(registry: Arc<AnalyzerRegistry>, num_shards: usize) -> Self {
        let n = num_shards.max(1).next_power_of_two();
        let aggregate_bytes = Arc::new(AtomicUsize::new(0));
        let shards = (0..n)
            .map(|_| {
                parking_lot::RwLock::new(ShardGenerations {
                    active: FtsMemtable::with_registry_and_aggregate(
                        Arc::clone(&registry),
                        Arc::clone(&aggregate_bytes),
                    ),
                    frozen: Vec::new(),
                    registry: Arc::clone(&registry),
                    aggregate: Arc::clone(&aggregate_bytes),
                })
            })
            .collect();
        Self {
            shards,
            shard_mask: n - 1,
            aggregate_bytes,
        }
    }

    #[inline]
    pub fn shard_for_dynamic(&self, doc_id: &str) -> usize {
        let h = xxhash_rust::xxh3::xxh3_64(doc_id.as_bytes());
        (h as usize) & self.shard_mask
    }

    pub fn shard_count(&self) -> usize {
        self.shards.len()
    }

    /// Run `f` with exclusive access to a specific shard's ACTIVE
    /// `FtsMemtable`.  Used by the turbo ingest hot path to hold one
    /// shard's lock for the entire bulk batch so all docs of a batch
    /// see each other's state consistently.  New writes always target
    /// the active generation — frozen ones belong to in-flight flushes.
    pub fn with_shard_mut<R>(&self, shard: usize, f: impl FnOnce(&mut FtsMemtable) -> R) -> R {
        let mut g = self.shards[shard].write();
        f(&mut g.active)
    }

    /// Run `f` with shared (read-only) access to a specific shard's ACTIVE
    /// generation only.  Reads that must see EVERY buffered doc of the shard
    /// (the search path) fold across generations inside the per-shard read
    /// accessors instead — see [`ShardGenerations`].
    pub fn with_shard<R>(&self, shard: usize, f: impl FnOnce(&FtsMemtable) -> R) -> R {
        let g = self.shards[shard].read();
        f(&g.active)
    }

    /// The analyzer `FtsMemtable::insert` uses for text fields: the
    /// registry's "default" pipeline when one was configured via index
    /// settings, else "standard".  Every shard shares the same registry
    /// (see `with_registry_and_shards`), so shard 0 is representative.
    /// Used by the bulk turbo path to pre-analyze docs OUTSIDE the
    /// shard write locks via [`analyze_doc`].
    pub fn default_analyzer(&self) -> Option<Arc<AnalyzerPipeline>> {
        let g = self.shards[0].read();
        g.active
            .registry
            .get_analyzer("default")
            .or_else(|| g.active.registry.get_analyzer("standard"))
    }

    /// Total document count across all shards and all generations — the
    /// reader-visible buffered count, frozen generations included (their
    /// docs stay visible until the flush publishes).
    pub fn doc_count(&self) -> usize {
        let mut n = 0;
        for s in &self.shards {
            let g = s.read();
            n += g.active.doc_count();
            for fz in &g.frozen {
                n += fz.gen.read().doc_count();
            }
        }
        n
    }

    /// Number of frozen generations across all shards — the test-observable
    /// signal that a flush has claimed buffered docs without retiring them
    /// yet (#1015: the drain is a freeze; `doc_count()` cannot signal it
    /// because frozen docs stay counted until the publish retires them).
    #[cfg(test)]
    pub(crate) fn frozen_generation_count(&self) -> usize {
        self.shards.iter().map(|s| s.read().frozen.len()).sum()
    }

    /// Frozen generations still HELD by an in-flight flush claim — the
    /// test-observable counterpart of [`Self::frozen_generation_count`]:
    /// a released claim flips `claimed` back without changing either the
    /// generation count or `doc_count()`, so this is the only way to wait
    /// out a detached finalizer's failure path.
    #[cfg(test)]
    pub(crate) fn claimed_frozen_generation_count(&self) -> usize {
        self.shards
            .iter()
            .map(|s| s.read().frozen.iter().filter(|fg| fg.claimed).count())
            .sum()
    }

    /// Total approximate byte size across all shards.  Lock-free: a single
    /// relaxed load of the shared aggregate the shards maintain
    /// incrementally at every `total_bytes` update — no shard locks taken.
    pub fn size_bytes(&self) -> usize {
        self.aggregate_bytes.load(Ordering::Relaxed)
    }

    /// Drop a doc from whichever shard owns it.  A doc can sit in a frozen
    /// generation (its flush is mid-build); removing it there keeps today's
    /// live-shard semantics — strip live structures, keep ghost stats, and
    /// the guarded version-map repoint at publish resolves the rest.
    pub fn remove(&self, doc_id: &str) {
        let s = self.shard_for_dynamic(doc_id);
        let mut g = self.shards[s].write();
        if g.active.contains(doc_id) {
            g.active.remove(doc_id);
            return;
        }
        for fz in &g.frozen {
            let mut m = fz.gen.write();
            if m.contains(doc_id) {
                m.remove(doc_id);
                return;
            }
        }
    }

    /// Return `true` if any shard holds the doc.  Picks the
    /// correct shard directly — no need to scan others.
    pub fn contains(&self, doc_id: &str) -> bool {
        let s = self.shard_for_dynamic(doc_id);
        let g = self.shards[s].read();
        if g.active.contains(doc_id) {
            return true;
        }
        g.frozen.iter().any(|fz| fz.gen.read().contains(doc_id))
    }

    pub fn get_doc_source_as_value(&self, doc_id: &str) -> Option<Value> {
        let s = self.shard_for_dynamic(doc_id);
        let g = self.shards[s].read();
        g.active.get_doc_source_as_value(doc_id).or_else(|| {
            g.frozen
                .iter()
                .find_map(|fz| fz.gen.read().get_doc_source_as_value(doc_id))
        })
    }

    pub fn get_doc_source_arc(&self, doc_id: &str) -> Option<Arc<Value>> {
        let s = self.shard_for_dynamic(doc_id);
        let g = self.shards[s].read();
        g.active.get_doc_source_arc(doc_id).or_else(|| {
            g.frozen
                .iter()
                .find_map(|fz| fz.gen.read().get_doc_source_arc(doc_id))
        })
    }

    /// Return every doc id in every shard (unordered).
    pub fn all_doc_ids(&self) -> Vec<String> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            out.extend(g.active.all_doc_ids());
            for fz in &g.frozen {
                out.extend(fz.gen.read().all_doc_ids());
            }
        }
        out
    }

    /// Bounded sorted-candidate extraction for a single-field sort
    /// (the implicit `@timestamp desc` index sort in particular).
    ///
    /// Returns `(candidate_ids, total_buffered_docs)` where the candidates
    /// are every buffered doc that could possibly reach the global
    /// top-`cap` page: the per-shard top-`cap` by sort key (boundary
    /// ties included) plus up to `cap` docs missing the field (the top-N
    /// heap's full-key comparison places them per the sort's `missing`
    /// policy).  Served INCREMENTALLY per shard via `SortCandCache` —
    /// each query folds in only the docs inserted since the last one —
    /// under the shard write lock (brief: O(new docs)).
    ///
    /// Per-doc key resolution (mirrors how the heap ranks the hit later):
    /// numeric column value when present, else the keyword column value
    /// through `normalize` (the caller passes the memoised date→epoch
    /// normaliser `compute_sort_values` uses per hit — this is what lets
    /// date-STRING `@timestamp` columns take the bounded path), else
    /// missing.
    ///
    /// `None` when any value is un-normalisable or the field has carried
    /// array values (per-element `mode` semantics need the full walk) —
    /// caller falls back to the exact full walk.
    pub fn sort_candidates_numeric(
        &self,
        field: &str,
        desc: bool,
        cap: usize,
        normalize: &dyn Fn(&str) -> Option<f64>,
    ) -> Option<(Vec<String>, u64)> {
        let cap = cap.max(1);
        let mut total: u64 = 0;
        let mut cands: Vec<(f64, String)> = Vec::new();
        let mut missing: Vec<String> = Vec::new();
        for s in &self.shards {
            // Step 4: READ lock — sort_candidates_cached now folds new docs
            // into the interior-mutable cache without an exclusive shard lock.
            // Frozen generations join the fold under the SAME shard read
            // hold: their caches are per-generation and their doc sets are
            // frozen, so the incremental fold stays exact for each.
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                let n = gen.doc_count();
                total += n as u64;
                if n == 0 {
                    continue;
                }
                let (mut top, miss, _) = gen.sort_candidates_cached(field, desc, cap, normalize)?;
                cands.append(&mut top);
                for m in miss {
                    if missing.len() < cap {
                        missing.push(m);
                    } else {
                        break;
                    }
                }
            }
        }
        // Global cut to cap (+boundary ties).
        if desc {
            cands.sort_unstable_by(|a, b| {
                b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)
            });
        } else {
            cands.sort_unstable_by(|a, b| {
                a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)
            });
        }
        let mut keep = cands.len().min(cap);
        while keep < cands.len() && keep > 0 && cands[keep].0 == cands[keep - 1].0 {
            keep += 1;
        }
        let mut ids: Vec<String> = cands.into_iter().take(keep).map(|(_, id)| id).collect();
        ids.extend(missing);
        Some((ids, total))
    }

    /// Bounded variant: clone at most `limit` doc ids and return the TOTAL
    /// buffered doc count alongside.  The unsorted match_all page only
    /// needs `from+size+ε` ids, but `all_doc_ids` cloned every buffered id
    /// on every request — ~1 s per query at a 300 k-doc memtable under
    /// sustained bulk load (the `mem_admit` phase of the read-under-write
    /// breakdown).
    /// #191 — reduce per-shard bounded candidates to the globally best `limit`
    /// under the order the final page sort uses.
    ///
    /// Every hit these paths produce carries the SAME score (`1.0` for the
    /// id-only and doc-values collectors; a top-level `constant_score` rewrites
    /// them all to the wrapper's boost), so the page order reduces to
    /// `(seq_no ASC, _id ASC)` — the tail of `(score DESC, seq_no ASC, _id ASC)`
    /// at `index.rs:16222-16228`.
    ///
    /// Returns the surviving cap-th `seq_no` as the next shard's cutoff, so the
    /// walk can stop cloning ids that can no longer reach the page.
    fn narrow_to_page<T>(cands: &mut Vec<(u64, String, T)>, limit: usize) -> u64 {
        if cands.len() <= limit {
            return u64::MAX;
        }
        cands.sort_unstable_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
        cands.truncate(limit);
        cands.last().map_or(u64::MAX, |c| c.0)
    }

    pub fn doc_ids_bounded(&self, limit: usize) -> (Vec<String>, u64) {
        // #191 — this used to fill greedily from shard 0 (`limit - out.len()`
        // of room for each later shard), so an index whose memtable held more
        // than `limit` documents returned "shard 0's first `limit` ids", not
        // "the `limit` earliest-arriving ids".  Shard assignment is by doc-id
        // hash (and by BATCH for bulk), so that set is neither arrival order
        // nor stable, and `size:5` stopped being a prefix of `size:1000` on an
        // all-tied corpus.  Bound each shard independently and keep the global
        // best `limit` by `(seq_no, _id)` instead.
        let mut cands: Vec<(u64, String, ())> = Vec::with_capacity(limit.min(4096));
        let mut total: u64 = 0;
        let mut cutoff = u64::MAX;
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                total += gen.doc_count() as u64;
                cands.extend(
                    gen.ranked_ids_up_to(limit, cutoff)
                        .into_iter()
                        .map(|(q, id)| (q, id, ())),
                );
                cutoff = Self::narrow_to_page(&mut cands, limit);
            }
        }
        (cands.into_iter().map(|(_, id, ())| id).collect(), total)
    }

    /// Return every (doc_id, source) pair.  Aggregates across all
    /// shards; insertion order is preserved within a shard but not
    /// across shards.  Callers that need global order should sort
    /// downstream.
    pub fn all_docs_with_sources(&self) -> Vec<(String, Value)> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            out.extend(g.active.all_docs_with_sources());
            for fz in &g.frozen {
                out.extend(fz.gen.read().all_docs_with_sources());
            }
        }
        out
    }

    /// Arc-sharing twin of `all_docs_with_sources` — no deep clones.
    pub fn all_docs_with_sources_arc(&self) -> Vec<(String, Arc<Value>)> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            out.extend(g.active.all_docs_with_sources_arc());
            for fz in &g.frozen {
                out.extend(fz.gen.read().all_docs_with_sources_arc());
            }
        }
        out
    }

    /// `(seq_no, doc_id, source_arc)` for every buffered document.
    ///
    /// #191 — the brute per-doc scan's bounded collector has to decide whether
    /// a document it reaches AFTER filling the cap still belongs on the page.
    /// Shards are concatenated in index order, which is not arrival order, so
    /// that decision needs the document's arrival `seq_no`.  Carrying it in the
    /// snapshot costs 8 bytes per entry and saves a `VersionMap` hash lookup
    /// per match on the hottest memtable read path.
    pub fn all_docs_with_seq_arc(&self) -> Vec<(u64, String, Arc<Value>)> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            out.extend(g.active.all_docs_with_seq_arc());
            for fz in &g.frozen {
                out.extend(fz.gen.read().all_docs_with_seq_arc());
            }
        }
        out
    }

    /// Columnar-filtered twin of `all_docs_with_sources_arc`: materialise ONLY
    /// the `(doc_id, source_arc)` pairs matching `preds` (a pure conjunction of
    /// Term/Range predicates), so a filtered `size:0` aggregation folds
    /// O(matching) docs instead of hydrating the whole memtable under the shard
    /// read lock.  Returns `None` (caller must fall back to
    /// `all_docs_with_sources_arc`) when any shard can't prove column↔source
    /// equivalence for a predicate field — see `filtered_docs_arc_into`.
    ///
    /// Each shard's enumeration + source materialisation happens under a single
    /// `s.read()` held only for that shard's walk (the same brief per-shard hold
    /// `all_docs_with_sources_arc` takes), so this both shrinks the work and the
    /// lock hold that the mixed read-under-write contention is bound on.
    pub fn filtered_docs_arc(&self, preds: &[MemBoolPred]) -> Option<Vec<(String, Arc<Value>)>> {
        if preds.is_empty() {
            return None;
        }
        let mut out: Vec<(String, Arc<Value>)> = Vec::new();
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                if gen.doc_count() == 0 {
                    continue;
                }
                gen.filtered_docs_arc_into(preds, &mut out)?;
            }
        }
        Some(out)
    }

    /// Whole-memtable drain, merge-sort by seq_no.  Test-facing since #1015:
    /// the flush path uses `freeze_shard_accounted` (generations stay
    /// reader-visible); this legacy shape survives for tests that need a
    /// fully-empty memtable, and drains every generation of every shard.
    pub fn drain_with_sources(&self) -> Vec<(String, HashMap<String, String>, Value)> {
        let mut all: Vec<(u64, (String, HashMap<String, String>, Value))> = Vec::new();
        for s in &self.shards {
            let mut g = s.write();
            all.extend(g.active.drain_with_sources_raw());
            for fz in &mut g.frozen {
                all.extend(fz.gen.write().drain_with_sources_raw());
            }
        }
        all.sort_by_key(|(seq, _)| *seq);
        all.into_iter().map(|(_, t)| t).collect()
    }

    /// Drain every shard (deletes tombstones only), merge by seq_no.
    /// Test-facing, like `drain_with_sources`.
    pub fn drain(&self) -> Vec<(String, HashMap<String, String>)> {
        let mut all: Vec<(u64, (String, HashMap<String, String>))> = Vec::new();
        for s in &self.shards {
            let mut g = s.write();
            all.extend(g.active.drain_raw());
            for fz in &mut g.frozen {
                all.extend(fz.gen.write().drain_raw());
            }
        }
        all.sort_by_key(|(seq, _)| *seq);
        all.into_iter().map(|(_, t)| t).collect()
    }

    /// Per-doc field-length getter used by memtable BM25 scoring.
    pub fn get_source(&self, doc_id: &str) -> Option<HashMap<String, String>> {
        let s = self.shard_for_dynamic(doc_id);
        let g = self.shards[s].read();
        g.active.get_source(doc_id).or_else(|| {
            g.frozen
                .iter()
                .find_map(|fz| fz.gen.read().get_source(doc_id))
        })
    }

    /// Combined numeric doc-values column — concatenates per-shard
    /// columns.  Note: column indexes are per-shard; callers that use
    /// absolute doc indices must account for shard offsets.  For
    /// aggregations this is sufficient because the values are what
    /// matter, not the indices.
    pub fn doc_values_numeric_values(&self, field: &str) -> Vec<Option<f64>> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            if let Some(col) = g.active.doc_values_numeric_column(field) {
                out.extend_from_slice(col);
            }
            for fz in &g.frozen {
                if let Some(col) = fz.gen.read().doc_values_numeric_column(field) {
                    out.extend_from_slice(col);
                }
            }
        }
        out
    }

    pub fn doc_values_keyword_values(&self, field: &str) -> Vec<Option<String>> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            if let Some(col) = g.active.doc_values_keyword_column(field) {
                out.extend(col.iter().cloned());
            }
            for fz in &g.frozen {
                if let Some(col) = fz.gen.read().doc_values_keyword_column(field) {
                    out.extend(col.iter().cloned());
                }
            }
        }
        out
    }

    /// Fan out a term count query across shards and sum.
    ///
    /// Bounded-delta: `FtsMemtable::doc_values_keyword_count` now takes
    /// `&self` (the maintained maps are folded under interior mutability),
    /// so this holds only the shard **READ** lock — concurrent term-count
    /// queries no longer serialise against each other or the writer.
    pub fn doc_values_keyword_count(&self, field: &str, value: &str) -> u32 {
        let mut n = 0;
        for s in &self.shards {
            let g = s.read();
            n += g.active.doc_values_keyword_count(field, value).unwrap_or(0);
            for fz in &g.frozen {
                n += fz
                    .gen
                    .read()
                    .doc_values_keyword_count(field, value)
                    .unwrap_or(0);
            }
        }
        n
    }

    pub fn doc_values_numeric_count(&self, field: &str, value: f64) -> u32 {
        let mut n = 0;
        for s in &self.shards {
            let g = s.read();
            n += g.active.doc_values_numeric_count(field, value).unwrap_or(0);
            for fz in &g.frozen {
                n += fz
                    .gen
                    .read()
                    .doc_values_numeric_count(field, value)
                    .unwrap_or(0);
            }
        }
        n
    }

    /// Aggregate all terms + counts for a field across shards.
    pub fn all_terms_for_field(&self, field: &str) -> Vec<(String, usize)> {
        let mut acc: HashMap<String, usize> = HashMap::new();
        for s in &self.shards {
            let g = s.read();
            for (term, count) in g.active.all_terms_for_field(field) {
                *acc.entry(term).or_insert(0) += count;
            }
            for fz in &g.frozen {
                for (term, count) in fz.gen.read().all_terms_for_field(field) {
                    *acc.entry(term).or_insert(0) += count;
                }
            }
        }
        acc.into_iter().collect()
    }

    pub fn all_keyword_values_for_field(&self, field: &str) -> Vec<(String, usize)> {
        let mut acc: HashMap<String, usize> = HashMap::new();
        for s in &self.shards {
            let g = s.read();
            for (v, c) in g.active.all_keyword_values_for_field(field) {
                *acc.entry(v).or_insert(0) += c;
            }
            for fz in &g.frozen {
                for (v, c) in fz.gen.read().all_keyword_values_for_field(field) {
                    *acc.entry(v).or_insert(0) += c;
                }
            }
        }
        acc.into_iter().collect()
    }

    /// Full-text search across all shards, top-`limit` hits.
    ///
    /// BM25 is computed with GLOBAL doc_count + global per-term doc_freq +
    /// global avg_field_length so the score is segment-invariant. Without
    /// this, sharded memtable runs would give each shard its own (smaller)
    /// statistics — flat IDFs and dropped length normalisation, since
    /// many shards would have N=1, doc_freq=1.
    pub fn search_text(&self, query: &str, fields: &[&str], limit: usize) -> Vec<MemtableHit> {
        static NO_BOOSTS: std::sync::OnceLock<std::collections::HashMap<String, f32>> =
            std::sync::OnceLock::new();
        self.search_text_boosted(
            query,
            fields,
            limit,
            NO_BOOSTS.get_or_init(std::collections::HashMap::new),
        )
    }

    /// `search_text` with per-field boost multipliers from the query tree
    /// (ES `boost` on match clauses / `field^N` on multi_match). Fields
    /// absent from the map score with boost 1.0.
    pub fn search_text_boosted(
        &self,
        query: &str,
        fields: &[&str],
        limit: usize,
        field_boosts: &std::collections::HashMap<String, f32>,
    ) -> Vec<MemtableHit> {
        self.search_text_boosted_inner(query, fields, limit, limit, field_boosts, None)
            .0
    }

    /// `search_text_boosted` with counting DECOUPLED from materialisation:
    /// returns the top-`limit` hits plus the EXACT number of matching docs
    /// across all shards, independent of `limit`.  Shards are searched
    /// uncapped — the per-match score-map entry is paid either way; only
    /// the `MemtableHit` materialisation grows — which is exactly the cost
    /// the `count_only` path already paid via `usize::MAX`.  This is the
    /// memtable twin of the segment `search_bounded` (hits, seg_total)
    /// contract: pre-fix, size>0 match totals capped the memtable
    /// contribution at the ~256 fetch limit (b7 DEFECT 1b).
    pub fn search_text_boosted_with_total(
        &self,
        query: &str,
        fields: &[&str],
        limit: usize,
        field_boosts: &std::collections::HashMap<String, f32>,
    ) -> (Vec<MemtableHit>, u64) {
        self.search_text_boosted_inner(query, fields, limit, usize::MAX, field_boosts, None)
    }

    /// `search_text_boosted_with_total` scored against INDEX-WIDE BM25
    /// statistics (#188).
    ///
    /// `stats` is the union over every live arm (all segments + this
    /// memtable), computed once per search by the engine.  Without it the
    /// memtable's "global" statistics are global only across its own shards —
    /// so a document whose live copy has just been moved here by an overwrite
    /// scores against `N = 1`, `df = 1`, `dl/avgdl = 1`, and outranks every
    /// correctly-normalised segment hit.
    ///
    /// `None` reproduces `search_text_boosted_with_total` bit-for-bit; the
    /// engine passes `None` when the index has at most one live scoring arm.
    pub fn search_text_boosted_with_total_using(
        &self,
        query: &str,
        fields: &[&str],
        limit: usize,
        field_boosts: &std::collections::HashMap<String, f32>,
        stats: Option<&xerj_fts::CollectionStats>,
    ) -> (Vec<MemtableHit>, u64) {
        self.search_text_boosted_inner(query, fields, limit, usize::MAX, field_boosts, stats)
    }

    /// `dis_max` over single-field text subs, each scored through the SAME
    /// per-field BM25 path a standalone `match` takes (#834): every
    /// `(field, query, boost)` sub runs UNCAPPED (a doc outside one sub's
    /// page can still win the max via another sub, and a capped sub list
    /// would silently drop its contribution), the per-doc scores combine as
    /// `max + tie_breaker × (Σ − max)` — mirroring the segment's
    /// `FtsSearcher::execute_dis_max` — and only the COMBINED result is
    /// page-key sorted and truncated to `limit`.  A doc lives in exactly
    /// one shard, so its `seq_no` is identical across subs and the first
    /// contributing sub's value is authoritative.
    pub fn search_dis_max_boosted_with_total_using(
        &self,
        subs: &[(&str, &str, f32)],
        tie_breaker: f32,
        limit: usize,
        stats: Option<&xerj_fts::CollectionStats>,
    ) -> (Vec<MemtableHit>, u64) {
        // Per-doc (max, sum, seq_no) accumulator over sub scores — the same
        // f32 (max, sum) pair `execute_dis_max` keeps.
        let acc: std::collections::HashMap<String, (f32, f32, u64)> = subs.iter().fold(
            std::collections::HashMap::new(),
            |acc, &(field, query, boost)| {
                let boosts: std::collections::HashMap<String, f32> =
                    std::iter::once((field.to_string(), boost)).collect();
                let (hits, _) = self.search_text_boosted_inner(
                    query,
                    &[field],
                    usize::MAX,
                    usize::MAX,
                    &boosts,
                    stats,
                );
                hits.into_iter().fold(acc, |mut acc, h| {
                    let e = acc
                        .entry(h.doc_id)
                        .or_insert((f32::NEG_INFINITY, 0.0, h.seq_no));
                    if h.score > e.0 {
                        e.0 = h.score;
                    }
                    e.1 += h.score;
                    acc
                })
            },
        );
        let mut all: Vec<MemtableHit> = acc
            .into_iter()
            .map(|(doc_id, (max, sum, seq_no))| MemtableHit {
                doc_id,
                score: max + tie_breaker * (sum - max),
                seq_no,
            })
            .collect();
        sort_hits_by_page_key(&mut all);
        let total = all.len() as u64;
        all.truncate(limit);
        (all, total)
    }

    /// This memtable's contribution to the index-wide BM25 collection
    /// statistics (#188): per-field `FieldStats` and per-(field, term)
    /// doc_freq for the analysed `query` tokens, restricted to `fields`
    /// (empty ⇒ every indexed field).
    ///
    /// Ghost-inclusive, exactly like the in-search aggregation it was hoisted
    /// out of: tombstoned and superseded versions count until a flush/merge
    /// purges them, which is what Lucene does and what keeps a delete from
    /// silently shifting every score.
    ///
    /// Returns `None` when the memtable can't analyse the query (no analyzer,
    /// or the query analyses to zero tokens) — the caller must then not build
    /// index-wide stats at all rather than build partial ones.
    pub fn collection_stats(
        &self,
        query: &str,
        fields: &[&str],
    ) -> Option<xerj_fts::CollectionStats> {
        let analyzer = self.shards.iter().find_map(|s| {
            let g = s.read();
            g.active
                .registry
                .get_analyzer("default")
                .or_else(|| g.active.registry.get_analyzer("standard"))
        })?;
        let q_tokens = analyzer.analyze(query);
        if q_tokens.is_empty() {
            return None;
        }
        let (field_total_len, term_df) = self.aggregate_shard_stats(&q_tokens, fields);
        let mut out = xerj_fts::CollectionStats::new();
        for (fname, (sum, n)) in field_total_len {
            out.add_field(
                &fname,
                &xerj_fts::FieldStats {
                    total_docs: n,
                    total_field_length: sum.round() as u64,
                },
            );
        }
        for ((fname, term), df) in term_df {
            out.add_doc_freq(&fname, &term, df);
        }
        Some(out)
    }

    /// Shared cross-shard fold behind both [`Self::collection_stats`] and the
    /// in-search aggregation: `(per-field (Σ field_len, docs-with-field),
    /// per-(field, term) doc_freq)`, ghosts included.
    fn aggregate_shard_stats(
        &self,
        q_tokens: &[xerj_fts::analyzer::Token],
        fields: &[&str],
    ) -> (
        std::collections::HashMap<String, (f64, u64)>,
        std::collections::HashMap<(String, String), u64>,
    ) {
        let mut field_total_len: std::collections::HashMap<String, (f64, u64)> =
            std::collections::HashMap::new();
        let mut term_df: std::collections::HashMap<(String, String), u64> =
            std::collections::HashMap::new();
        for shard in &self.shards {
            let g = shard.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                // Field length sums (live).
                for (fname, (sum, n)) in &gen.avg_field_lengths {
                    let entry = field_total_len.entry(fname.clone()).or_insert((0.0, 0));
                    entry.0 += sum;
                    entry.1 += n;
                }
                // Field length sums (tombstoned versions retained for avgdl).
                for (fname, (sum, n)) in &gen.ghost_field_len {
                    let entry = field_total_len.entry(fname.clone()).or_insert((0.0, 0));
                    entry.0 += sum;
                    entry.1 += n;
                }
                // Per-term doc_freq across shards (live postings).
                for (fname, postings) in &gen.index {
                    if !fields.is_empty() && !fields.iter().any(|f| f == fname) {
                        continue;
                    }
                    for token in q_tokens {
                        if let Some(pl) = postings.get(&token.text) {
                            *term_df
                                .entry((fname.clone(), token.text.clone()))
                                .or_insert(0) += pl.len() as u64;
                        }
                    }
                }
                // Per-term doc_freq from tombstoned versions (delete-aware df).
                for (fname, terms) in &gen.ghost_doc_freq {
                    if !fields.is_empty() && !fields.iter().any(|f| f == fname) {
                        continue;
                    }
                    for token in q_tokens {
                        if let Some(df) = terms.get(&token.text) {
                            *term_df
                                .entry((fname.clone(), token.text.clone()))
                                .or_insert(0) += *df;
                        }
                    }
                }
            }
        }
        (field_total_len, term_df)
    }

    /// Shared body: `shard_limit` caps each shard's hit materialisation
    /// (`usize::MAX` ⇒ the returned total is the exact global match count);
    /// `limit` caps the merged, score-sorted result.
    fn search_text_boosted_inner(
        &self,
        query: &str,
        fields: &[&str],
        limit: usize,
        shard_limit: usize,
        field_boosts: &std::collections::HashMap<String, f32>,
        external: Option<&xerj_fts::CollectionStats>,
    ) -> (Vec<MemtableHit>, u64) {
        // Pre-pass: tokenise the query (use any shard's analyzer — they're
        // all the same registry-provided one) and aggregate per-term
        // global doc_freq + per-field global stats.
        let analyzer = self.shards.iter().find_map(|s| {
            let g = s.read();
            g.active
                .registry
                .get_analyzer("default")
                .or_else(|| g.active.registry.get_analyzer("standard"))
        });
        let analyzer = match analyzer {
            Some(a) => a,
            None => return (Vec::new(), 0),
        };
        let q_tokens = analyzer.analyze(query);
        if q_tokens.is_empty() {
            return (Vec::new(), 0);
        }

        // ── Statistics fed to the per-shard scorers ──────────────────────
        //
        // TWO modes, and the difference is deliberate:
        //
        //  * `external = Some(cs)` (#188) — the engine computed the INDEX-WIDE
        //    union (all segments + this memtable) and every arm scores against
        //    it, so a document's score no longer depends on which arm holds it.
        //    `N` is then per-field docs-with-field, matching Lucene's pairing
        //    of `FieldStats.docCount()` with `TermStats.docFreq()`.
        //  * `external = None` — the historical memtable-only aggregation,
        //    global across SHARDS with a single scalar `N`.  Kept byte-exact
        //    because the engine takes this path whenever the index has at most
        //    one live scoring arm, where the union is the identity, and any
        //    drift there would move every memtable-only score.
        //
        // Delete-aware in both modes (Lucene/ES parity): the scoring `N`
        // counts live docs AND tombstoned/superseded versions not yet merged
        // away — tantivy's scalar N is likewise physical, Σ `max_doc()` over
        // segments (`Bm25StatisticsProvider for Searcher::total_num_docs`,
        // tantivy/src/query/bm25.rs:38-45, MIT).  This is *only* the BM25
        // IDF `N` — hits.total and pagination still use the live
        // `doc_count()`.
        //
        // The ghost-inclusive scalar is computed ONCE here because BOTH
        // modes need the same value (#193): the external mode falls back to
        // it for any scored field the supplied union does not carry (or
        // carries with `total_docs == 0`), and using the live-only count
        // there silently raised IDF on indices holding tombstoned or
        // superseded versions — an unintended divergence from the `None`
        // path that the previous comment ("scalar N is unused in this
        // mode") wrongly waved off.
        let ghost_inclusive_n: u64 = {
            let mut n: u64 = self.doc_count() as u64;
            for shard in &self.shards {
                let g = shard.read();
                n += g.active.ghost_docs;
                for fz in &g.frozen {
                    n += fz.gen.read().ghost_docs;
                }
            }
            n
        };
        let mut global_field_doc_count: std::collections::HashMap<String, u64> =
            std::collections::HashMap::new();
        let global_doc_count: u64;
        let global_avg_field_len: std::collections::HashMap<String, f32>;
        let term_global_df: std::collections::HashMap<(String, String), u64>;
        match external {
            Some(cs) => {
                // Scalar N backs the per-field fallback (field absent from
                // the union) — keep it ghost-inclusive, exactly like the
                // `None` branch, so the two modes agree on IDF wherever
                // the fallback engages.
                global_doc_count = ghost_inclusive_n;
                let mut avg = std::collections::HashMap::new();
                for (fname, fs) in cs.field_iter() {
                    avg.insert(fname.clone(), fs.avg_field_length());
                    global_field_doc_count.insert(fname.clone(), fs.total_docs);
                }
                global_avg_field_len = avg;
                let mut df = std::collections::HashMap::new();
                for token in &q_tokens {
                    for fname in global_avg_field_len.keys() {
                        if !fields.is_empty() && !fields.iter().any(|f| f == fname) {
                            continue;
                        }
                        if let Some(v) = cs.df(fname, &token.text) {
                            df.insert((fname.clone(), token.text.clone()), v);
                        }
                    }
                }
                term_global_df = df;
            }
            None => {
                global_doc_count = ghost_inclusive_n;
                let (field_total_len, df) = self.aggregate_shard_stats(&q_tokens, fields);
                global_avg_field_len = field_total_len
                    .into_iter()
                    .map(|(k, (sum, n))| (k, if n == 0 { 0.0 } else { (sum / n as f64) as f32 }))
                    .collect();
                term_global_df = df;
            }
        }

        let mut all: Vec<MemtableHit> = Vec::new();
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                all.extend(gen.search_text_with_global_stats(
                    query,
                    fields,
                    shard_limit,
                    global_doc_count,
                    &global_field_doc_count,
                    &global_avg_field_len,
                    &term_global_df,
                    field_boosts,
                ));
            }
        }
        // #191 — the cross-shard merge sorts by the FULL page key, not by score
        // alone.  Score alone leaves ties in shard-concatenation order, so
        // `truncate(limit)` kept "the tied documents that happen to live in the
        // lowest-numbered shards" rather than the earliest-arriving ones, and a
        // `size:5` page stopped being a prefix of a `size:1000` one.
        sort_hits_by_page_key(&mut all);
        // A doc_id lives in exactly ONE shard, so `all` has no duplicates:
        // its pre-truncation length IS the exact global match count when
        // `shard_limit == usize::MAX` (a lower bound otherwise).
        let total = all.len() as u64;
        all.truncate(limit);
        (all, total)
    }

    /// Drop-in replacement for the old single-memtable `insert` used by
    /// the `IndexStore::index()` single-doc path.
    pub fn insert(&self, doc_id: String, source: &Value, schema: &Schema, seq_no: u64) {
        self.insert_shared(doc_id, Arc::new(source.clone()), schema, seq_no);
    }

    /// [`Self::insert`] for a source the caller already holds in an `Arc`.
    /// The memtable keeps that `Arc` rather than a deep copy, so a document
    /// the storage memtable also holds is in memory once (#2427).
    pub fn insert_shared(&self, doc_id: String, source: Arc<Value>, schema: &Schema, seq_no: u64) {
        let s = self.shard_for_dynamic(&doc_id);
        // New writes always land in the ACTIVE generation — frozen ones are
        // immutable snapshots held for an in-flight flush (#1015).
        self.shards[s]
            .write()
            .active
            .insert_shared(doc_id, source, schema, seq_no);
    }

    /// Drop-in for `insert_pretokenized_with_seq` — picks a shard by
    /// doc_id so delete / re-insert on the same doc collide under
    /// the same shard lock.
    pub fn insert_pretokenized_with_seq(
        &self,
        seq_no: u64,
        doc_id: String,
        source: Arc<Value>,
        tokens: &[String],
    ) {
        let s = self.shard_for_dynamic(&doc_id);
        self.shards[s]
            .write()
            .active
            .insert_pretokenized_with_seq(seq_no, doc_id, source, tokens);
    }

    /// M5.11 — raw-bytes ultra-turbo insert (shard-routed).
    pub fn insert_raw_bytes_with_seq(&self, seq_no: u64, doc_id: String, source_bytes: Arc<[u8]>) {
        let s = self.shard_for_dynamic(&doc_id);
        self.shards[s]
            .write()
            .active
            .insert_raw_bytes_with_seq(seq_no, doc_id, source_bytes);
    }

    /// Iterate every document in every shard as `(doc_id, Value)`.
    /// Clones each `Arc<Value>`'s inner so callers that expect an
    /// owned `Value` keep working.
    pub fn all_docs_cloned(&self) -> Vec<(String, Value)> {
        let mut out = Vec::new();
        for s in &self.shards {
            let g = s.read();
            out.extend(g.active.all_docs());
            for fz in &g.frozen {
                out.extend(fz.gen.read().all_docs());
            }
        }
        out
    }

    /// Fold `f` over every document's `(doc_id, Value)` pair under
    /// the owning shard's lock.  Used by query paths that want a
    /// streaming view without materialising a full Vec.
    pub fn for_each_doc<F: FnMut(&str, &Value)>(&self, mut f: F) {
        for s in &self.shards {
            let g = s.read();
            for (doc_id, val) in g.active.all_docs() {
                f(&doc_id, &val);
            }
            for fz in &g.frozen {
                for (doc_id, val) in fz.gen.read().all_docs() {
                    f(&doc_id, &val);
                }
            }
        }
    }

    /// Fold `f` over every numeric value (from doc-values) for
    /// `field` across all shards.  Used by stats/sum/avg/min/max
    /// agg runners — replaces the `mem.doc_values.numeric.get(field)`
    /// single-shard borrow that the old code did.  The closure may
    /// return `false` to stop iteration early.
    pub fn for_each_numeric_value<F: FnMut(f64)>(&self, field: &str, mut f: F) {
        for s in &self.shards {
            let g = s.read();
            if let Some(col) = g.active.doc_values_numeric_column(field) {
                for v in col.iter().flatten() {
                    f(*v);
                }
            }
            for fz in &g.frozen {
                if let Some(col) = fz.gen.read().doc_values_numeric_column(field) {
                    for v in col.iter().flatten() {
                        f(*v);
                    }
                }
            }
        }
    }

    /// Fold `f` over every (keyword_value, count) pair for a field
    /// across shards.
    pub fn for_each_keyword_value<F: FnMut(&str)>(&self, field: &str, mut f: F) {
        for s in &self.shards {
            let g = s.read();
            if let Some(col) = g.active.doc_values_keyword_column(field) {
                for v in col.iter().flatten() {
                    f(v);
                }
            }
            for fz in &g.frozen {
                if let Some(col) = fz.gen.read().doc_values_keyword_column(field) {
                    for v in col.iter().flatten() {
                        f(v);
                    }
                }
            }
        }
    }

    /// Total live numeric entries for `field` (sum of per-shard
    /// non-null counts).
    pub fn numeric_count(&self, field: &str) -> u64 {
        let mut n: u64 = 0;
        for s in &self.shards {
            let g = s.read();
            if let Some(col) = g.active.doc_values_numeric_column(field) {
                n += col.iter().filter(|v| v.is_some()).count() as u64;
            }
            for fz in &g.frozen {
                if let Some(col) = fz.gen.read().doc_values_numeric_column(field) {
                    n += col.iter().filter(|v| v.is_some()).count() as u64;
                }
            }
        }
        n
    }

    /// Total live keyword entries for `field`.
    pub fn keyword_count(&self, field: &str) -> u64 {
        let mut n: u64 = 0;
        for s in &self.shards {
            let g = s.read();
            if let Some(col) = g.active.doc_values_keyword_column(field) {
                n += col.iter().filter(|v| v.is_some()).count() as u64;
            }
            for fz in &g.frozen {
                if let Some(col) = fz.gen.read().doc_values_keyword_column(field) {
                    n += col.iter().filter(|v| v.is_some()).count() as u64;
                }
            }
        }
        n
    }

    /// Fused columnar bool query across all shards — one position walk per
    /// shard applying every predicate, all under a single read lock per
    /// shard (so positions can't shift between per-child walks under a
    /// concurrent flush drain).
    ///
    /// Returns `(hits, total)` where `hits` holds at most `limit` cloned
    /// `(doc_id, local_idx)` pairs (page materialisation) and `total` is
    /// the EXACT matching-doc count across the whole memtable.  `None`
    /// when any predicate's column is missing in a non-empty shard or a
    /// keyword column looks analyzed-text (same conservative bailouts as
    /// `doc_values_term_query` / `doc_values_range_query`) — caller falls
    /// back to the stored-source scan.
    ///
    /// This replaces the per-query `DocsForScan` walk for composite bool
    /// filters, which paid a `doc_matches_query` JSON descent per buffered
    /// doc per query (~1-6 s at a 40-280 k-doc memtable under a sustained
    /// bulk writer — the read-under-write bool collapse).
    pub fn doc_values_bool_query(
        &self,
        preds: &[MemBoolPred],
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        if preds.is_empty() {
            return None;
        }
        // #191 — bound each shard independently and keep the global best
        // `limit` by `(seq_no, _id)`; `limit - out.len()` starved every shard
        // after the first, so the page was "shard 0's first `limit` matches".
        let mut cands: Vec<(u64, String, usize)> = Vec::new();
        let mut total: u64 = 0;
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                if gen.doc_count() == 0 {
                    continue;
                }
                let (hits, t) = gen.doc_values_bool_hits(preds, limit)?;
                cands.extend(
                    hits.into_iter()
                        .map(|(id, pos)| (gen.seq_no_at(pos), id, pos)),
                );
                total += t;
                Self::narrow_to_page(&mut cands, limit);
            }
        }
        Some((
            cands.into_iter().map(|(_, id, pos)| (id, pos)).collect(),
            total,
        ))
    }

    /// Columnar per-value counts for one field across all shards, for the
    /// terms/cardinality agg fast paths.  Returns `(value → live doc
    /// count, missing_docs)` — value RENDERING matches the brute per-doc
    /// extractor exactly for the shapes this serves (the keyword column
    /// stores strings verbatim, numbers via `Number::to_string`, bools as
    /// "true"/"false" — the same strings `flatten_to_strings` emits).
    ///
    /// `None` (caller falls back to the per-doc JSON extraction walk,
    /// which cost 100-300 ms/query against a drain-lagged 300 k-doc
    /// memtable — the terms-agg read-under-write tail) when equivalence
    /// isn't provable:
    /// - the field has EVER carried an array value in a shard (column
    ///   keeps only the first element; brute fans out over all);
    /// - any doc carries `_doc_count` (brute weights buckets by it);
    /// - the caller must additionally gate on plain field names (no
    ///   dotted paths / `.keyword` fallbacks — those resolve through
    ///   `get_nested_field`, not the column).
    pub fn terms_counts_columnar(
        &self,
        field: &str,
    ) -> Option<(std::collections::HashMap<String, u64>, u64)> {
        use rayon::prelude::*;
        // Bounded-delta: instead of the old FULL O(memtable) keyword-column
        // walk on EVERY terms/cardinality agg request (25-32 ms/query at a
        // drain-lagged 100-270 k-doc memtable — the dominant `fast_aggs`
        // read-under-write p95/p99 term), each shard reads its maintained
        // `keyword_counts` map (folded on the read side over just the docs
        // appended since the last query).  Result is O(distinct values) per
        // shard, not O(docs).  Value RENDERING is identical: `keyword_counts`
        // is folded from the same keyword column the old walk read.  `present`
        // (sum of the per-value counts) == the old "non-None slots" count, so
        // `missing = n - present` is unchanged.  Any shard that hits a bail
        // condition (array-valued field, `_doc_count` weights) yields `None`,
        // and `collect::<Option<_>>` short-circuits the whole call to `None`.
        let per_shard: Option<Vec<(std::collections::HashMap<String, u64>, u64)>> = self
            .shards
            .par_iter()
            .map(|s| {
                let g = s.read();
                let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                    g.frozen.iter().map(|fz| fz.gen.read()).collect();
                let mut counts: std::collections::HashMap<String, u64> =
                    std::collections::HashMap::new();
                let mut missing: u64 = 0;
                for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                    let gn = gen.doc_count();
                    if gn == 0 {
                        continue;
                    }
                    if gen.doc_values.array_fields.contains(field)
                        || gen.doc_values.numeric.contains_key("_doc_count")
                        || gen.doc_values.keyword.contains_key("_doc_count")
                    {
                        return None;
                    }
                    let gcounts = gen.doc_values.with_keyword_field(field, |c| {
                        match c.keyword_counts.get(field) {
                            None => {
                                // No doc in this generation carries the field
                                // as a scalar — all of them are "missing".
                                (std::collections::HashMap::new(), gn as u64)
                            }
                            Some(m) => {
                                let mut gc: std::collections::HashMap<String, u64> =
                                    std::collections::HashMap::with_capacity(m.len());
                                let mut present: u64 = 0;
                                for (k, &cnt) in m.iter() {
                                    gc.insert(k.clone(), cnt as u64);
                                    present += cnt as u64;
                                }
                                (gc, gn as u64 - present)
                            }
                        }
                    });
                    missing += gcounts.1;
                    for (term, cnt) in gcounts.0 {
                        *counts.entry(term).or_insert(0) += cnt;
                    }
                }
                Some((counts, missing))
            })
            .collect();
        let per_shard = per_shard?;
        // Merge the per-shard partials (commutative — identical to the serial
        // single-map accumulation this replaces).
        let mut counts: std::collections::HashMap<String, u64> = std::collections::HashMap::new();
        let mut missing: u64 = 0;
        for (shard_counts, shard_missing) in per_shard {
            missing += shard_missing;
            if counts.is_empty() {
                // First non-empty partial becomes the base map — avoids
                // re-hashing its keys on the common single-populated-shard
                // path.
                counts = shard_counts;
            } else {
                for (term, cnt) in shard_counts {
                    *counts.entry(term).or_insert(0) += cnt;
                }
            }
        }
        Some((counts, missing))
    }

    /// True when ANY shard's single-valued doc-values column for `field` is
    /// lossy, so no shard's column may be trusted for whole-value matching.
    ///
    /// #332 — the per-shard `doc_values_term_query` / `doc_values_terms_query`
    /// return `None` for two very different reasons: "this shard holds no
    /// column for the field" (a true zero-match answer) and "this shard's
    /// column cannot answer" (array-valued → first element only;
    /// analyzed-text → whitespace values). The cross-shard wrappers below
    /// could not tell them apart and folded a REFUSAL into "zero hits here",
    /// so a two-shard memtable with `{"tags":["red","blue"]}` in shard A and
    /// `{"tags":"red"}` in shard B answered `term tags:red` with shard B's
    /// doc only. `doc_values_bool_query` already propagates a per-shard
    /// refusal globally (`?` inside its loop); this makes the single-leaf
    /// wrappers behave the same way.
    fn dv_column_unusable(&self, field: &str) -> bool {
        // #677: a dotted path (an object sub-field like `meta.owner`, or a
        // `.keyword` multi-field) resolves through `get_field_value` /
        // `get_nested_field` against the stored `_source`, NOT a faithful
        // single-valued DV column — the same rule `terms_counts_columnar`
        // states. Counting such a field via the column undercounts (0 while
        // the buffered source matches → the #677 size:0-vs-size:5 split), so
        // force the authoritative source scan. Always correct (the scan is the
        // authority); only a bounded perf cost on dotted-field term counts.
        if field.contains('.') {
            return true;
        }
        self.shards.iter().any(|s| {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            std::iter::once(&g.active)
                .chain(frozen.iter().map(|rd| &**rd))
                .any(|gen| {
                    gen.doc_values.array_fields.contains(field)
                        || gen.doc_values.keyword_has_whitespace.contains(field)
                })
        })
    }

    /// True when the memtable's single-valued doc-values column for `field`
    /// cannot authoritatively answer a `term`/count query: the field has
    /// carried an ARRAY (only the first element is stored — see `push_field`)
    /// or a whitespace keyword (the column holds analyzed-text source). Callers
    /// that count via the column MUST fall back to the stored-source scan,
    /// which matches array membership. This is the same guard `doc_values_term_query`
    /// and `doc_values_bool_hits` already apply; exposed so the `try_shortcut_count`
    /// bare-`term` count path can bail identically (#423/#408: a `term` on a
    /// keyword array counted 0 while buffered but N after flush).
    pub fn term_count_needs_source_scan(&self, field: &str) -> bool {
        self.dv_column_unusable(field)
    }

    /// True when ANY generation's single-valued numeric column for `field`
    /// cannot answer a `range`: the field has carried an array, and
    /// `push_field` keeps only the array's first element, so a document
    /// whose later element is in range would be missed (#1750). Callers
    /// must fall back to the stored-source scan, whose range matcher checks
    /// every element. Narrower than [`Self::dv_column_unusable`]: whitespace
    /// keyword values do not affect the numeric column.
    pub fn range_column_unusable(&self, field: &str) -> bool {
        self.shards.iter().any(|s| {
            let g = s.read();
            g.active.doc_values.array_fields.contains(field)
                || g.frozen
                    .iter()
                    .any(|fz| fz.gen.read().doc_values.array_fields.contains(field))
        })
    }

    /// DocValues term query — aggregates hits across all shards.
    /// Returns `Some(Vec<(doc_id, local_idx)>)` if any shard matched.
    /// The `local_idx` is shard-local; callers use the doc_id to
    /// resolve the source via `get_doc_source_*`.
    pub fn doc_values_term_query(
        &self,
        field: &str,
        value: &str,
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        if self.dv_column_unusable(field) {
            return None;
        }
        let mut cands: Vec<(u64, String, usize)> = Vec::new();
        let mut total: u64 = 0;
        let mut any_hit = false;
        for s in &self.shards {
            let g = s.read();
            // Step 1: bound the id clone per shard; the total is still exact
            // per shard.  Mirrors `doc_values_bool_query`.  #191 — the bound
            // used to be the REMAINING global window, which handed the whole
            // page to shard 0.
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                if let Some((hits, t)) = gen.doc_values_term_query(field, value, limit) {
                    any_hit = true;
                    cands.extend(
                        hits.into_iter()
                            .map(|(id, pos)| (gen.seq_no_at(pos), id, pos)),
                    );
                    total += t;
                    Self::narrow_to_page(&mut cands, limit);
                }
            }
        }
        if any_hit {
            Some((
                cands.into_iter().map(|(_, id, pos)| (id, pos)).collect(),
                total,
            ))
        } else {
            None
        }
    }

    pub fn doc_values_terms_query(
        &self,
        field: &str,
        values: &[String],
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        // #191 — per-shard bound + global `(seq_no, _id)` narrowing, as in
        // `doc_values_term_query`.
        // #332 — one lossy shard poisons the whole field; see
        // `dv_column_unusable`.
        if self.dv_column_unusable(field) {
            return None;
        }
        let mut cands: Vec<(u64, String, usize)> = Vec::new();
        let mut total: u64 = 0;
        let mut any_hit = false;
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                if let Some((hits, t)) = gen.doc_values_terms_query(field, values, limit) {
                    any_hit = true;
                    cands.extend(
                        hits.into_iter()
                            .map(|(id, pos)| (gen.seq_no_at(pos), id, pos)),
                    );
                    total += t;
                    Self::narrow_to_page(&mut cands, limit);
                }
            }
        }
        if any_hit {
            Some((
                cands.into_iter().map(|(_, id, pos)| (id, pos)).collect(),
                total,
            ))
        } else {
            None
        }
    }

    /// Aggregated smart-field-encoding map across all shards.
    ///
    /// Computed lazily on the READ side from the doc-values `keyword`
    /// columns already maintained per shard — the same bounded-read pattern
    /// used by `all_keyword_values_for_field` / the `keyword_counts` fold.
    /// The ingest hot path only appends raw column values (it no longer runs
    /// `FieldAnalyzer` — the ingest-time `collect_sample` was dropped during
    /// the M4 perf pass, which is why `analyzed_encodings` is always empty),
    /// so this adds ZERO per-doc cost: the analysis happens here, once, when
    /// a caller (the `/v1/indices/:name/encodings` stats endpoint) asks for
    /// it.
    ///
    /// For each field we merge every shard's non-`None` keyword values into a
    /// single sample vector (capped at `ENCODING_SAMPLE_CAP` to bound work on
    /// a large memtable), then — for each field carrying at least
    /// `ENCODING_MIN_SAMPLES` values — run `FieldAnalyzer` to pick its optimal
    /// encoding.  Read-only: it never touches the columns or the ingest path.
    pub fn aggregated_field_encodings(&self) -> HashMap<String, FieldEncoding> {
        /// Upper bound on values analyzed per field — keeps this read bounded
        /// on a multi-hundred-k-doc memtable while staying statistically ample.
        const ENCODING_SAMPLE_CAP: usize = 4096;
        /// Minimum values a field needs before we report an encoding for it;
        /// below this the cardinality tiers aren't statistically meaningful.
        const ENCODING_MIN_SAMPLES: usize = 16;

        // Merge each shard's keyword columns into one capped sample vector
        // per field, under the shard READ lock.
        let mut samples: HashMap<String, Vec<String>> = HashMap::new();
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                for (field, col) in gen.doc_values.keyword.iter() {
                    let acc = samples.entry(field.clone()).or_default();
                    if acc.len() >= ENCODING_SAMPLE_CAP {
                        continue;
                    }
                    for v in col.iter().flatten() {
                        acc.push(v.clone());
                        if acc.len() >= ENCODING_SAMPLE_CAP {
                            break;
                        }
                    }
                }
            }
        }

        let analyzer = FieldAnalyzer::default();
        let mut out: HashMap<String, FieldEncoding> = HashMap::new();
        for (field, vals) in samples {
            if vals.len() < ENCODING_MIN_SAMPLES {
                continue;
            }
            let refs: Vec<&str> = vals.iter().map(|s| s.as_str()).collect();
            out.insert(field.clone(), analyzer.analyze(&field, &refs));
        }
        out
    }

    pub fn doc_values_range_query(
        &self,
        field: &str,
        gte: Option<f64>,
        gt: Option<f64>,
        lte: Option<f64>,
        lt: Option<f64>,
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        // #1750 — refuse globally, before any shard answers: a shard whose
        // column holds only an array's first element must not be folded into
        // a confident partial total (the #332 shape, for range).
        if self.range_column_unusable(field) {
            return None;
        }
        // #191 — per-shard bound + global `(seq_no, _id)` narrowing, as in
        // `doc_values_term_query`.
        let mut cands: Vec<(u64, String, usize)> = Vec::new();
        let mut total: u64 = 0;
        let mut any_hit = false;
        for s in &self.shards {
            let g = s.read();
            let frozen: Vec<parking_lot::RwLockReadGuard<'_, FtsMemtable>> =
                g.frozen.iter().map(|fz| fz.gen.read()).collect();
            for gen in std::iter::once(&g.active).chain(frozen.iter().map(|rd| &**rd)) {
                if let Some((hits, t)) = gen.doc_values_range_query(field, gte, gt, lte, lt, limit)
                {
                    any_hit = true;
                    cands.extend(
                        hits.into_iter()
                            .map(|(id, pos)| (gen.seq_no_at(pos), id, pos)),
                    );
                    total += t;
                    Self::narrow_to_page(&mut cands, limit);
                }
            }
        }
        if any_hit {
            Some((
                cands.into_iter().map(|(_, id, pos)| (id, pos)).collect(),
                total,
            ))
        } else {
            None
        }
    }

    /// Freeze ONE shard for flush (#1015): claim the flushable generations
    /// (the active one, plus any unclaimed frozen leftovers of a FAILED
    /// flush), install a fresh active generation for new writes, and return
    /// the drain tuples `(seq_no, doc_id, source_arc, raw_bytes)` in
    /// WAL-sequence order alongside the [`FrozenClaim`].
    ///
    /// Unlike the pre-#1015 drain, the claimed docs REMAIN READER-VISIBLE
    /// (the generations move to the shard's frozen list, they do not leave
    /// it), so the flush's segment build runs with NO collection-publication
    /// guard — only the publish (snapshot swap + generation retire) needs
    /// one, and it is millisecond-scale.  The drain tuples are a stable
    /// Arc-sharing snapshot for the segment writer; a doc deleted from a
    /// frozen generation mid-build keeps today's mid-flush-delete semantics
    /// (guarded version-map repoint at publish).
    ///
    /// The byte aggregate is NOT adjusted here: the frozen generations keep
    /// mirroring into it exactly as they did as actives, because readers
    /// still see their docs.  Bytes leave the aggregate at [`FrozenClaim::retire`],
    /// which is also where the ghost collection stats are purged (flush ==
    /// merge, at the same point the docs leave the reader view).
    ///
    /// M5.11 — entries with `source = Value::Null` but non-empty
    /// `source_bytes` are lazily parsed here on the flush thread,
    /// keeping the ingest hot path free of `serde_json::from_str`.
    pub(crate) fn freeze_shard_accounted(
        self: &Arc<Self>,
        shard_idx: usize,
        skip_parse: bool,
    ) -> (Vec<(u64, String, Arc<Value>, Arc<[u8]>)>, FrozenClaim) {
        // Claim phase — one hold of the shard write lock.  Pointer moves
        // only (O(1) per generation): the old drain's argument for moving
        // the heavy deallocation off the lock applies unchanged, and here
        // there is no deallocation at all yet — the maps stay live for
        // readers until retire.
        let claimed: Vec<Arc<parking_lot::RwLock<FtsMemtable>>> = {
            let mut g = self.shards[shard_idx].write();
            let mut claimed = Vec::new();
            // Recycle failed-flush leftovers first: their docs ride THIS
            // flush (the generation twin of the old `restore_failed_flush`,
            // which re-inserted drained docs on failure).
            for fg in &mut g.frozen {
                if !fg.claimed {
                    fg.claimed = true;
                    claimed.push(Arc::clone(&fg.gen));
                }
            }
            if g.active.doc_count() > 0 {
                let fresh = FtsMemtable::with_registry_and_aggregate(
                    Arc::clone(&g.registry),
                    Arc::clone(&g.aggregate),
                );
                let frozen_gen = Arc::new(parking_lot::RwLock::new(std::mem::replace(
                    &mut g.active,
                    fresh,
                )));
                g.frozen.push(FrozenGen {
                    gen: Arc::clone(&frozen_gen),
                    claimed: true,
                });
                claimed.push(frozen_gen);
            }
            claimed
        };
        // Snapshot phase — read each claimed generation under its own read
        // lock and build the drain tuples.  The generations stay mutable
        // (deletes land in them during the build), which is why this is a
        // snapshot by VALUE for the source Arcs, not a borrow of the maps.
        let mut out: Vec<(u64, String, Arc<Value>, Arc<[u8]>)> = Vec::new();
        for gen in &claimed {
            let m = gen.read();
            for e in &m.docs {
                let raw = e.source_bytes.clone();
                let source = if skip_parse {
                    Arc::clone(&e.source)
                } else if e.source.is_null() && !raw.is_empty() {
                    match serde_json::from_slice::<Value>(&raw) {
                        Ok(v) => Arc::new(v),
                        Err(_) => Arc::new(Value::Null),
                    }
                } else {
                    Arc::clone(&e.source)
                };
                out.push((e.seq_no, e.doc_id.clone(), source, raw));
            }
        }
        out.sort_by_key(|(seq, _, _, _)| *seq);
        (
            out,
            FrozenClaim {
                memtable: Arc::clone(self),
                shard_idx,
                gens: claimed,
                finished: false,
                ledger: crate::ingest_memory::active_ledger(),
            },
        )
    }

    /// Test override for the retire-time ledger (see `FrozenClaim::ledger`)
    /// — the engine's flush-publisher test hook needs the freed bytes
    /// attributed to a `Box::leak`ed ledger it can assert against.
    #[cfg(test)]
    pub(crate) fn freeze_shard_accounted_for_test(
        self: &Arc<Self>,
        shard_idx: usize,
        skip_parse: bool,
        ledger: &'static crate::ingest_memory::Ledger,
    ) -> (Vec<(u64, String, Arc<Value>, Arc<[u8]>)>, FrozenClaim) {
        let (rows, mut claim) = self.freeze_shard_accounted(shard_idx, skip_parse);
        claim.ledger = Some(ledger);
        (rows, claim)
    }

    /// Check if a shard's first entry was inserted via the raw-bytes
    /// path (`insert_raw_bytes_with_seq`).  Used by `do_flush_shard`
    /// to decide whether to build FTS sidecars at flush time.  A recycled
    /// frozen generation answers for itself too — the flush builds one
    /// segment for every claimed generation.
    pub fn peek_shard_has_raw_bytes(&self, shard_idx: usize) -> bool {
        let g = self.shards[shard_idx].read();
        let active = g
            .active
            .docs
            .first()
            .map(|e| !e.source_bytes.is_empty())
            .unwrap_or(false);
        if active {
            return true;
        }
        g.frozen.iter().any(|fg| {
            fg.gen
                .read()
                .docs
                .first()
                .map(|e| !e.source_bytes.is_empty())
                .unwrap_or(false)
        })
    }

    /// Return `(shard_idx, doc_count, size_bytes)` triples so the
    /// flush scheduler can pick which shard to drain next.  The
    /// tuple is sampled under each shard's own lock so it's
    /// point-in-time accurate.  Claimed frozen generations (a flush
    /// already in flight) do NOT count toward the load — their docs
    /// are being flushed; unclaimed ones (a failed flush's leftovers)
    /// do.
    pub fn shard_loads(&self) -> Vec<(usize, usize, usize)> {
        self.shards
            .iter()
            .enumerate()
            .map(|(i, s)| {
                let g = s.read();
                let mut docs = g.active.doc_count();
                let mut bytes = g.active.size_bytes();
                for fg in &g.frozen {
                    if !fg.claimed {
                        let m = fg.gen.read();
                        docs += m.doc_count();
                        bytes += m.size_bytes();
                    }
                }
                (i, docs, bytes)
            })
            .collect()
    }

    /// Return `(doc_count, size_bytes)` for a single shard.  Used by
    /// the sync ingest path to decide whether to raise the flush
    /// signal bit for the shard it just wrote to — cheaper than
    /// iterating all shards via `shard_loads()`.  Frozen generations
    /// count like `shard_loads` says: unclaimed yes, claimed no.
    pub fn shard_load(&self, shard_idx: usize) -> (usize, usize) {
        let g = self.shards[shard_idx].read();
        let mut docs = g.active.doc_count();
        let mut bytes = g.active.size_bytes();
        for fg in &g.frozen {
            if !fg.claimed {
                let m = fg.gen.read();
                docs += m.doc_count();
                bytes += m.size_bytes();
            }
        }
        (docs, bytes)
    }
}

impl FrozenClaim {
    /// Publish-side retire (#1015): remove the claimed generations from the
    /// shard's reader-visible list.  MUST run inside the collection
    /// publication bracket, immediately after the segment snapshot swap —
    /// the bracket is what makes "the segment is in the snapshot" and "the
    /// frozen docs left the memtable" atomic for a capturing reader (the
    /// #1014 evenness check rejects a capture that straddles the two).
    ///
    /// Bytes leave the shared aggregate HERE (the docs were reader-visible
    /// until this point), and the ghost collection stats are purged by the
    /// same drop — flush == merge, at the moment the docs leave the view.
    /// The heavy deallocation runs on a detached thread, off the publish
    /// path, exactly like the old drain's async-free.
    ///
    /// Returns the detached-free guard so the caller can hold it for the
    /// remaining lifetime of its flush task (the ledger keeps the freeing
    /// bytes attributed until then).
    pub(crate) fn retire(mut self) -> Option<crate::ingest_memory::Retained<'static>> {
        self.finished = true;
        let gens = std::mem::take(&mut self.gens);
        let mut dead: Vec<Arc<parking_lot::RwLock<FtsMemtable>>> = Vec::new();
        let mut removed_bytes: usize = 0;
        {
            let mut g = self.memtable.shards[self.shard_idx].write();
            g.frozen.retain(|fg| {
                if gens.iter().any(|c| Arc::ptr_eq(c, &fg.gen)) {
                    removed_bytes += fg.gen.read().total_bytes;
                    dead.push(Arc::clone(&fg.gen));
                    false
                } else {
                    true
                }
            });
        }
        if removed_bytes > 0 {
            self.memtable
                .aggregate_bytes
                .fetch_sub(removed_bytes, Ordering::Relaxed);
        }
        // Mirror the pre-#1015 drain accounting: the freeing bytes stay
        // attributed to the flush pipeline until the flush task finishes.
        let guard = self.ledger.map(|ledger| {
            crate::ingest_memory::Retained::for_ledger(
                ledger,
                crate::ingest_memory::Category::FlushDrained,
                removed_bytes,
            )
        });
        // Free the dead generations off the publish critical path. The
        // spawn ignores the watermark gate: at the watermark is exactly when
        // this memory-relief helper must run. If the thread cannot be
        // created — `pthread_create` fails, or (the #1122 failure) the new
        // thread's stack cannot be reserved — the Arcs drop inline: same
        // correctness, we only lose the async-free optimisation. The spawn
        // itself must not panic: std panics *inside* the child when its
        // alternate signal stack cannot be mapped, and the release profile
        // is panic=abort, so that one allocation used to kill the node.
        // `try_spawn` uses a non-panicking spawn and hands the job back;
        // dropping the job frees the Arcs on this thread. Logged once so a
        // flush storm under pressure does not emit a line per generation.
        static DRAIN_FREE_DEGRADED: AtomicBool = AtomicBool::new(false);
        if let Err(failure) =
            xerj_common::thread_spawn::try_spawn_memory_relief("xerj-drain-free", move || {
                drop(dead)
            })
        {
            xerj_common::thread_spawn::warn_best_effort_degraded(
                &DRAIN_FREE_DEGRADED,
                "xerj-drain-free",
                &failure.error,
            );
            drop(failure.job);
        }
        guard
    }

    /// Failure-side release (#1015): keep the claimed generations
    /// reader-visible and unclaim them, so the shard's next flush drains
    /// them again.  This is the generation twin of the old
    /// `restore_failed_flush` — nothing needs re-inserting because the
    /// docs never left the reader view, and the version-map guard that
    /// restore enforced (never resurrect an overwritten doc) is not
    /// needed for the same reason.
    pub(crate) fn release(mut self) {
        self.finished = true;
        let gens = std::mem::take(&mut self.gens);
        let mut g = self.memtable.shards[self.shard_idx].write();
        for fg in &mut g.frozen {
            if gens.iter().any(|c| Arc::ptr_eq(c, &fg.gen)) {
                fg.claimed = false;
            }
        }
    }
}

impl Drop for FrozenClaim {
    /// A claim dropped without an explicit retire/release releases — the
    /// safe failure mode keeps the docs reader-visible and flushable.
    fn drop(&mut self) {
        if !self.finished {
            self.finished = true;
            let gens = std::mem::take(&mut self.gens);
            let mut g = self.memtable.shards[self.shard_idx].write();
            for fg in &mut g.frozen {
                if gens.iter().any(|c| Arc::ptr_eq(c, &fg.gen)) {
                    fg.claimed = false;
                }
            }
        }
    }
}

// Expose raw drain helpers on FtsMemtable that include seq_no so the
// sharded wrapper can merge by global WAL order.
impl FtsMemtable {
    pub fn drain_with_sources_raw(
        &mut self,
    ) -> Vec<(u64, (String, HashMap<String, String>, Value))> {
        let mut drained: Vec<MemEntry> = std::mem::take(&mut self.docs);
        drained.sort_by_key(|e| e.seq_no);
        let result: Vec<_> = drained
            .into_iter()
            .map(|e| {
                let seq = e.seq_no;
                let val = Arc::try_unwrap(e.source).unwrap_or_else(|arc| (*arc).clone());
                let fields = extract_text_fields_from(&val);
                (seq, (e.doc_id, fields, val))
            })
            .collect();
        self.index = FxHashMap::default();
        self.doc_values = DocValues::default();
        self.aggregate_bytes
            .fetch_sub(self.total_bytes, Ordering::Relaxed);
        self.total_bytes = 0;
        self.field_lengths = FxHashMap::default();
        self.avg_field_lengths = FxHashMap::default();
        self.doc_id_index = FxHashMap::default();
        // Flush is the equivalent of a Lucene merge: tombstone contributions
        // are purged, so delete-aware ghost statistics reset to empty.
        self.ghost_docs = 0;
        self.ghost_field_len = FxHashMap::default();
        self.ghost_doc_freq = FxHashMap::default();
        self.docs.shrink_to_fit();
        result
    }

    pub fn drain_raw(&mut self) -> Vec<(u64, (String, HashMap<String, String>))> {
        let mut drained: Vec<MemEntry> = std::mem::take(&mut self.docs);
        drained.sort_by_key(|e| e.seq_no);
        let result: Vec<_> = drained
            .into_iter()
            .map(|e| {
                let seq = e.seq_no;
                let fields = extract_text_fields_from(e.source.as_ref());
                (seq, (e.doc_id, fields))
            })
            .collect();
        self.index = FxHashMap::default();
        self.doc_values = DocValues::default();
        self.aggregate_bytes
            .fetch_sub(self.total_bytes, Ordering::Relaxed);
        self.total_bytes = 0;
        self.field_lengths = FxHashMap::default();
        self.avg_field_lengths = FxHashMap::default();
        self.doc_id_index = FxHashMap::default();
        // Flush is the equivalent of a Lucene merge: tombstone contributions
        // are purged, so delete-aware ghost statistics reset to empty.
        self.ghost_docs = 0;
        self.ghost_field_len = FxHashMap::default();
        self.ghost_doc_freq = FxHashMap::default();
        self.docs.shrink_to_fit();
        result
    }
}

/// Pad a column with `None` up to (but not including) `target_len`.
fn pad_to<T>(col: &mut Vec<Option<T>>, target_len: usize) {
    while col.len() < target_len {
        col.push(None);
    }
}

/// Borrow the value for `key`, inserting `make()` when absent — but only
/// allocate a fresh OWNED key on the miss path.
///
/// `map.entry(key.to_owned()).or_insert_with(make)` clones `key` into a
/// heap `String` on EVERY call, even when the entry already exists.  On
/// the bulk-ingest write-lock critical path the field name and each term
/// recur across every doc of a shard's sub-batch, so that unconditional
/// clone was a `malloc` per (doc × field) and per (doc × term) — profiled
/// as the dominant under-lock cost (~40 ms / 10 k-doc batch).  Here the
/// common HIT path pays a second `&str` hash/probe instead (a few ns with
/// FxHash) and allocates NOTHING; only a genuinely new field/term pays the
/// key clone.
///
/// Crucially this shrinks the critical section WITHOUT moving any work off
/// the lock, so the write lock still serialises writer CPU against reader
/// CPU (measured: an off-lock rebuild instead regressed the CPU-heaviest
/// read aggs by removing that serialisation).  The resulting map is
/// byte-identical to the `entry` form.
#[inline]
fn entry_no_clone<'a, V>(
    map: &'a mut FxHashMap<String, V>,
    key: &str,
    make: impl FnOnce() -> V,
) -> &'a mut V {
    if !map.contains_key(key) {
        map.insert(key.to_string(), make());
    }
    // Present after the branch above; the second lookup is a hash+probe,
    // strictly cheaper than the key `String` allocation it replaces.
    map.get_mut(key).expect("just inserted / already present")
}

/// Whether a keyword-column value names the number `num`. The column holds
/// a JSON number as its source spelling (`10.0`, `1e1`), so a numeric needle
/// compares by value; `None` (a string needle) never matches here.
fn kw_equals_number(stored: &str, num: Option<f64>) -> bool {
    num.is_some_and(|n| stored.parse::<f64>().is_ok_and(|f| f == n))
}

/// One resolved predicate of a `bool { must/filter }` for the fused
/// columnar memtable walk (`doc_values_bool_query`).  Built by the engine
/// from Term / Range query nodes; semantics per predicate mirror
/// `doc_values_term_query` / `doc_values_range_query` exactly.
pub enum MemBoolPred {
    /// `num` is the needle's value when the query spelled it as a JSON
    /// number. A numeric needle matches numerically, as on the flushed
    /// columnar path and the `_source` comparator: the keyword column renders
    /// a number the way its document spelled it, so `10` must also match a
    /// stored `10.0` (and `1e1`). A string needle stays an exact match.
    Term {
        field: String,
        value: String,
        num: Option<f64>,
    },
    /// Disjunction of exact values on one field. The catalog finalize-verify
    /// query is `bool.filter [terms file_key, term run_id]`; without this arm
    /// that bool fell through to `DocsForScan` and cloned every buffered
    /// catalog row, including the other run's (#1761).
    /// `nums` holds the values the query spelled as JSON numbers; see
    /// `Term::num`.
    Terms {
        field: String,
        values: Vec<String>,
        nums: Vec<f64>,
    },
    Range {
        field: String,
        gte: Option<f64>,
        gt: Option<f64>,
        lte: Option<f64>,
        lt: Option<f64>,
    },
}

// ── FtsMemtable ──────────────────────────────────────────────────────────────

/// In-memory inverted index for documents awaiting flush.
pub struct FtsMemtable {
    /// Documents in insertion order.
    docs: Vec<MemEntry>,
    /// Inverted index: field → term → posting list (doc_id → tf).
    /// FxHashMap — see `PostingList` for the hasher rationale.
    index: FxHashMap<String, FxHashMap<String, PostingList>>,
    /// Columnar doc-values store for fast term/range/agg queries.
    pub doc_values: DocValues,
    /// Total accumulated byte size.
    total_bytes: usize,
    /// Shared aggregate across all shards of one index — mirrors every
    /// `total_bytes` delta so the sampler reads one atomic.
    aggregate_bytes: Arc<AtomicUsize>,
    /// Analyzer registry.
    registry: Arc<AnalyzerRegistry>,
    /// Precomputed field lengths for BM25 scoring: field → {doc_id → token_count}
    field_lengths: FxHashMap<String, FxHashMap<DocId, u32>>,
    /// Running average field length per field: field → (total_tokens, doc_count)
    avg_field_lengths: FxHashMap<String, (f64, u64)>,
    /// doc_id → position in self.docs for O(1) lookup
    doc_id_index: FxHashMap<DocId, usize>,
    /// Delete-aware BM25 collection statistics (Lucene/ES parity).
    ///
    /// When a document is superseded by an update (remove + re-insert) or
    /// explicitly deleted, Lucene keeps that document's contribution to the
    /// collection statistics (N, total field length, per-term doc_freq)
    /// until the segment is actually merged.  We mirror that: `remove()`
    /// strips the old version from the LIVE structures (so it no longer
    /// matches or counts toward hits.total) but folds its contribution into
    /// these "ghost" accumulators, which are added back ONLY for BM25
    /// scoring.  They reset to empty on every drain/flush — the moral
    /// equivalent of a Lucene merge purging tombstones.
    ///
    /// A document that was never removed contributes nothing here, so its
    /// BM25 score is bit-for-bit identical to before this feature existed.
    ghost_docs: u64,
    ghost_field_len: FxHashMap<String, (f64, u64)>,
    ghost_doc_freq: FxHashMap<String, FxHashMap<String, u64>>,
}

impl FtsMemtable {
    /// Create a new empty memtable with the default analyzer registry.
    pub fn new() -> Self {
        Self::with_registry_and_aggregate(
            Arc::new(AnalyzerRegistry::default()),
            Arc::new(AtomicUsize::new(0)),
        )
    }

    /// Create a memtable using a shared custom analyzer registry.
    ///
    /// Use this when custom analyzers (synonyms, edge-ngrams, etc.) have been
    /// configured in the index settings so that indexing and query expansion
    /// use the same pipeline.
    pub fn with_registry(registry: Arc<AnalyzerRegistry>) -> Self {
        Self::with_registry_and_aggregate(registry, Arc::new(AtomicUsize::new(0)))
    }

    /// Create a memtable mirroring its `total_bytes` deltas into
    /// `aggregate_bytes`, the counter shared by every shard of one index —
    /// this is what lets `ShardedFtsMemtable::size_bytes` read one atomic
    /// instead of read-locking every shard.  A standalone memtable
    /// (via `new` / `with_registry`) gets its own fresh counter.
    fn with_registry_and_aggregate(
        registry: Arc<AnalyzerRegistry>,
        aggregate_bytes: Arc<AtomicUsize>,
    ) -> Self {
        Self {
            docs: Vec::new(),
            index: FxHashMap::default(),
            doc_values: DocValues::default(),
            total_bytes: 0,
            aggregate_bytes,
            registry,
            field_lengths: FxHashMap::default(),
            avg_field_lengths: FxHashMap::default(),
            doc_id_index: FxHashMap::default(),
            ghost_docs: 0,
            ghost_field_len: FxHashMap::default(),
            ghost_doc_freq: FxHashMap::default(),
        }
    }

    /// Insert a document into the memtable, indexing all text fields.
    pub fn insert(&mut self, doc_id: String, source: &Value, schema: &Schema, seq_no: u64) {
        self.insert_shared(doc_id, Arc::new(source.clone()), schema, seq_no);
    }

    /// [`Self::insert`] for a source the caller already holds in an `Arc`:
    /// the entry stores that `Arc` instead of a deep copy (#2427).
    pub fn insert_shared(
        &mut self,
        doc_id: String,
        source: Arc<Value>,
        schema: &Schema,
        seq_no: u64,
    ) {
        // Prefer a "default" analyzer if one was registered via custom settings
        // (e.g. with synonym expansion), otherwise fall back to "standard".
        let analyzer = self
            .registry
            .get_analyzer("default")
            .or_else(|| self.registry.get_analyzer("standard"))
            .expect("standard analyzer always present");

        let analyzed = analyze_doc(&source, schema, &analyzer);

        // See `insert_pretokenized` for sizing rationale.
        let raw_size = source.to_string().len() + doc_id.len();
        let size = raw_size * 3 + 64;

        self.insert_analyzed(seq_no, doc_id, source, &analyzed, size);
    }

    /// Back half of [`insert`] — everything that MUST run under the
    /// shard write lock: field-length/avg updates, posting merges,
    /// size accounting, doc-values push and the docs/doc_id_index push.
    ///
    /// The front half (schema text-field extraction + dynamic-mapping
    /// tree walk + analyzer tokenisation — see [`analyze_doc`]) is pure
    /// and can run OUTSIDE the lock; the ES `_bulk` turbo path runs it
    /// on the rayon pool per batch and then calls this under the shard
    /// lock.  Splitting insert this way cuts the lock-held cost from
    /// ~39 µs/doc to ~9 µs/doc — the single biggest lever on the
    /// 8-client bulk-ingest scaling ceiling.
    ///
    /// (A deeper split that also pre-aggregated tf and pre-extracted
    /// doc-values cells outside the lock was tried and REVERTED: the
    /// extra per-doc map/Vec allocations it added on the analyze side
    /// cost more total CPU than the in-lock time they saved — c8
    /// throughput dropped ~10% because the 8-client workload is CPU-
    /// saturated, not lock-saturated, after this split.)
    ///
    /// `size` is the pre-computed `size_bytes` accounting value
    /// (`(serialized_len + doc_id_len) * 3 + 64`), passed in so the
    /// `source.to_string()` re-serialisation also stays off the lock.
    /// The `source` Arc is stored directly — no deep clone.
    pub fn insert_analyzed(
        &mut self,
        seq_no: u64,
        doc_id: String,
        source: Arc<Value>,
        analyzed: &[(String, Vec<Token>)],
        size: usize,
    ) {
        // Intern the doc id ONCE (one `malloc`).  Every posting / field-
        // length / doc-id-index entry below then stores a pointer-bumped
        // `Arc<str>` clone instead of a fresh `String` heap allocation —
        // the per-token clone under the shard write lock becomes a
        // relaxed refcount increment.  See `DocId` for the rationale.
        let doc_key: DocId = Arc::from(doc_id.as_str());

        for (field_name, tokens) in analyzed {
            let token_count = tokens.len() as u32;

            // Cache the field length for BM25 scoring.  `field_name`
            // recurs across every doc of the batch, so `entry_no_clone`
            // skips the per-doc field-key `malloc` on the hot path.
            entry_no_clone(&mut self.field_lengths, field_name, Default::default)
                .insert(doc_key.clone(), token_count);

            // Update running average.
            let entry = entry_no_clone(&mut self.avg_field_lengths, field_name, || (0.0, 0));
            entry.0 += token_count as f64;
            entry.1 += 1;

            let field_index = entry_no_clone(&mut self.index, field_name, Default::default);
            for token in tokens {
                // `token.text` likewise recurs (low-cardinality keyword-ish
                // fields), so only a genuinely new term pays the key clone.
                let posting = entry_no_clone(field_index, &token.text, Default::default);
                // Interned doc id — a pointer bump, not a `String` malloc.
                *posting.entry(doc_key.clone()).or_insert(0) += 1;
            }
        }

        self.total_bytes += size;
        self.aggregate_bytes.fetch_add(size, Ordering::Relaxed);

        // Populate the columnar DocValues store BEFORE pushing to docs so that
        // the doc_index equals the current length (i.e. the slot we're about to fill).
        let doc_index = self.docs.len();
        self.doc_values.push(&source, doc_index);

        // Track doc_id → index for O(1) lookup.
        self.doc_id_index.insert(doc_key, doc_index);

        self.docs.push(MemEntry {
            seq_no,
            doc_id,
            source,
            source_bytes: Arc::from(&[][..]),
            parsed_memo: OnceLock::new(),
            size_bytes: size,
        });
    }

    /// Insert with pre-tokenized terms — builds the inverted index using
    /// the standard analyzer pipeline (same as `insert`).
    /// Hot-path insert called by the turbo bulk ingest loop.  The
    /// `seq_no` comes from the WAL append that just completed — the
    /// engine stores it on the entry so `drain_with_sources` can
    /// restore global order by sort at flush time (this is what makes
    /// it safe to drop the engine memtable lock while the WAL append
    /// runs, so multiple concurrent bulk requests can pipeline).
    pub fn insert_pretokenized_with_seq(
        &mut self,
        seq_no: u64,
        doc_id: String,
        source: Arc<Value>,
        tokens: &[String],
    ) {
        self.insert_pretokenized_inner(seq_no, doc_id, source, tokens);
    }

    /// M5.11 — raw-bytes ultra-turbo insert.
    ///
    /// Skips the JSON parse on the ingest hot path.  Stores the raw
    /// NDJSON bytes; drain-for-flush reconstructs the `Value` lazily on
    /// the background flush thread.  Used by the `_bulk` turbo path to
    /// take the per-doc `serde_json::from_str` (~5 µs/doc) off the
    /// critical section.
    pub fn insert_raw_bytes_with_seq(
        &mut self,
        seq_no: u64,
        doc_id: String,
        source_bytes: Arc<[u8]>,
    ) {
        // Issue #948 — the charge must be an estimate of what this entry
        // actually retains. On this path that is the raw NDJSON payload
        // (`source_bytes`), the doc-id string (duplicated into
        // `doc_id_index` and `MemEntry`), and the entry/index nodes — the
        // parsed `Value` and postings are built later, on the flush thread.
        //
        // History, so nobody re-breaks M5.17: a bare `source_bytes.len()`
        // charge there regressed log-ingest throughput (400-640 k vs 870 k
        // docs/s) by disturbing the shard flush scheduler's cadence for
        // SMALL docs. The flat 800 bytes stays as the FLOOR — compressed
        // log docs (~115 B) keep their exact historical charge and cadence —
        // but a mailbox-shaped doc (~10 KB of raw JSON retained verbatim)
        // now charges its real footprint. At 800 B/doc the process-wide
        // memtable budget (governor item 1) was admitting ~12× more live
        // bytes than it was told about, which is exactly how the #948
        // mailbox ingest overshot the memory cap while "under budget".
        let estimated = raw_bytes_charge(&doc_id, &source_bytes);
        self.total_bytes += estimated;
        self.aggregate_bytes.fetch_add(estimated, Ordering::Relaxed);

        let doc_index = self.docs.len();
        self.doc_id_index
            .insert(Arc::from(doc_id.as_str()), doc_index);

        self.docs.push(MemEntry {
            seq_no,
            doc_id,
            source: Arc::new(Value::Null),
            source_bytes,
            parsed_memo: OnceLock::new(),
            size_bytes: estimated,
        });
    }

    /// Fresh-insert fast path: the caller guarantees `doc_id` does not
    /// already exist in this shard (bulk CLI ingest makes this promise —
    /// each scanner+seq pair is unique).  Avoids cloning `doc_id` twice
    /// (once for the HashMap key, once for `MemEntry`) by inserting
    /// the HashMap entry using a borrowed key lookup via `RawEntry`.
    /// Currently we still clone since stable Rust HashMap requires an
    /// owned key; the gain is skipping the prior `remove()` miss lookup.
    pub fn insert_raw_bytes_fresh(&mut self, seq_no: u64, doc_id: String, source_bytes: Arc<[u8]>) {
        let estimated = raw_bytes_charge(&doc_id, &source_bytes);
        self.total_bytes += estimated;
        self.aggregate_bytes.fetch_add(estimated, Ordering::Relaxed);

        let doc_index = self.docs.len();
        self.doc_id_index
            .insert(Arc::from(doc_id.as_str()), doc_index);
        self.docs.push(MemEntry {
            seq_no,
            doc_id,
            source: Arc::new(Value::Null),
            source_bytes,
            parsed_memo: OnceLock::new(),
            size_bytes: estimated,
        });
    }

    pub fn insert_pretokenized(&mut self, doc_id: String, source: Arc<Value>, tokens: &[String]) {
        // `seq_no = 0` means "unknown" — drain_with_sources falls back
        // to insertion order when all entries share seq_no 0.  Tests
        // and the legacy single-doc path (which never carries a WAL
        // seq_no through) land here.
        self.insert_pretokenized_inner(0, doc_id, source, tokens);
    }

    fn insert_pretokenized_inner(
        &mut self,
        seq_no: u64,
        doc_id: String,
        source: Arc<Value>,
        _tokens: &[String],
    ) {
        // M5.6 — ULTRA-TURBO hot path.
        //
        // Pre-M5.6 this function did, for each doc:
        //   - Full tree walk to extract text_fields (HashMap<String,String>)
        //   - Per-field token-split for length counting
        //   - Per-field `entry(field.to_string())` × 2 HashMap inserts
        //     (field_lengths + avg_field_lengths)  → 2-4 allocs/field
        //   - `doc_values.push(source, idx)` → another tree walk + per
        //     field HashMap entry + String clones for keyword columns
        //   - `doc_id_index.insert(doc_id.clone(), ...)`
        //   - docs.push(MemEntry { .. })
        //
        // Measured: ~7 µs per doc → 35 ms per 5000-doc batch = ~145 k
        // docs/s per lock-holder.  Lock contention across 32 workers
        // capped us at ~380 k/s.
        //
        // The only work that MUST happen on the hot path is:
        //   1. Push to docs Vec (for drainability at flush)
        //   2. Update doc_id_index (for remove() support in next batch)
        //   3. Rough size accounting (for back-pressure)
        //
        // field_lengths / avg_field_lengths / doc_values are all
        // **reconstructable** from `docs[].source` at flush time.  We
        // skip them on the hot path entirely and rebuild them lazily
        // from the Arc<Value> in `drain_for_flush` / search-time
        // fallbacks.
        //
        // Rough size estimate: 800 bytes/doc is a reasonable proxy
        // for log data and keeps back-pressure within 2× of truth.
        let estimated = 800usize;
        self.total_bytes += estimated;
        self.aggregate_bytes.fetch_add(estimated, Ordering::Relaxed);

        let doc_index = self.docs.len();
        self.doc_id_index
            .insert(Arc::from(doc_id.as_str()), doc_index);

        self.docs.push(MemEntry {
            seq_no,
            doc_id,
            source,
            source_bytes: Arc::from(&[][..]),
            parsed_memo: OnceLock::new(),
            size_bytes: estimated,
        });
    }

    /// Zero-clone turbo insert — takes an `Arc<Value>` already shared with the
    /// WAL serialisation step.  The Arc pointer is bumped (8 bytes), not the
    /// entire JSON tree.  Builds the inverted index so match queries work.
    pub fn insert_pretokenized_arc(
        &mut self,
        doc_id: String,
        source: Arc<Value>,
        _tokens: &[String],
    ) {
        // Build text_fields by reading through the Arc — read-only, no clone.
        let mut text_fields: HashMap<String, String> = HashMap::new();
        if let Some(obj) = source.as_object() {
            for (key, val) in obj {
                let text = extract_text_value(val);
                if !text.is_empty() {
                    text_fields.insert(key.clone(), text);
                }
            }
        }

        // Build the inverted index entries so match queries work correctly.
        let analyzer = self
            .registry
            .get_analyzer("default")
            .or_else(|| self.registry.get_analyzer("standard"))
            .expect("standard analyzer always present");

        let doc_key: DocId = Arc::from(doc_id.as_str());
        for (field_name, text) in &text_fields {
            let tokens = analyzer.analyze(text);
            let token_count = tokens.len() as u32;

            // Cache the field length for BM25 scoring.
            self.field_lengths
                .entry(field_name.clone())
                .or_default()
                .insert(doc_key.clone(), token_count);

            // Update running average.
            let entry = self
                .avg_field_lengths
                .entry(field_name.clone())
                .or_insert((0.0, 0));
            entry.0 += token_count as f64;
            entry.1 += 1;

            let field_index = self.index.entry(field_name.clone()).or_default();
            for token in &tokens {
                let posting = field_index.entry(token.text.clone()).or_default();
                *posting.entry(doc_key.clone()).or_insert(0) += 1;
            }
        }

        // See `insert_pretokenized` for the sizing rationale — 3× the raw
        // text-field byte count covers inverted index + Arc + map overhead.
        let raw_size = source.to_string().len() + doc_id.len();
        let size = raw_size * 3 + 64;
        self.total_bytes += size;
        self.aggregate_bytes.fetch_add(size, Ordering::Relaxed);

        let doc_index = self.docs.len();
        // Pass a reference through the Arc — DocValues reads without cloning source.
        self.doc_values.push(&source, doc_index);

        // Track doc_id → index for O(1) lookup (interned Arc<str> key).
        self.doc_id_index.insert(doc_key, doc_index);

        self.docs.push(MemEntry {
            seq_no: 0,
            doc_id,
            // Arc clone = atomic refcount increment (cheap pointer copy).
            source,
            source_bytes: Arc::from(&[][..]),
            parsed_memo: OnceLock::new(),
            size_bytes: size,
        });
        let _ = text_fields; // consumed above by inverted-index build
    }

    /// Remove a document from the memtable (for deletes/updates).
    ///
    /// Fast exit when the doc isn't present — this is critical on the
    /// turbo ingest hot path, which calls `remove()` before every
    /// `insert_pretokenized` to guard against the "reindex existing doc"
    /// case.  For the much-more-common "append new doc" case the doc
    /// isn't in `doc_id_index`, so we can skip the O(fields × terms)
    /// inverted-index sweep that used to run unconditionally.  That
    /// sweep was the dominant single cost in the ingest hot path at
    /// 30 k+-doc memtables, turning 5 k-doc bulks into 2 500 ms of work.
    pub fn remove(&mut self, doc_id: &str) {
        // Cheap check first — if the doc isn't tracked, none of the
        // parallel structures contain it either (they're all updated
        // together inside `insert_pretokenized`).
        if !self.doc_id_index.contains_key(doc_id) {
            return;
        }
        // Remove from inverted index.  Capture every (field, term) this doc
        // actually contributed to so we can preserve its delete-aware
        // doc_freq contribution (Lucene keeps tombstoned postings counted
        // until a merge).  Two-phase to avoid borrowing `self.index` and
        // `self.ghost_doc_freq` mutably at once.
        let mut ghosted_terms: Vec<(String, String)> = Vec::new();
        for (field_name, field_index) in self.index.iter_mut() {
            for (term, posting_list) in field_index.iter_mut() {
                if posting_list.remove(doc_id).is_some() {
                    ghosted_terms.push((field_name.clone(), term.clone()));
                }
            }
        }
        for (field_name, term) in ghosted_terms {
            *self
                .ghost_doc_freq
                .entry(field_name)
                .or_default()
                .entry(term)
                .or_insert(0) += 1;
        }
        // Remove from field length caches and update running averages.
        // Retain the removed doc's field-length contribution as a ghost so
        // avgdl stays delete-aware (total_field_length / N counts tombstones).
        let mut ghosted_lengths: Vec<(String, u32)> = Vec::new();
        for (field_name, lengths) in &mut self.field_lengths {
            if let Some(token_count) = lengths.remove(doc_id) {
                if let Some(entry) = self.avg_field_lengths.get_mut(field_name) {
                    entry.0 -= token_count as f64;
                    entry.1 = entry.1.saturating_sub(1);
                }
                ghosted_lengths.push((field_name.clone(), token_count));
            }
        }
        for (field_name, token_count) in ghosted_lengths {
            let g = self.ghost_field_len.entry(field_name).or_insert((0.0, 0));
            g.0 += token_count as f64;
            g.1 += 1;
        }
        // One more document is now a tombstone for collection-stats purposes.
        self.ghost_docs += 1;
        // Remove from docs list AND the parallel DocValues columns.
        if let Some(pos) = self.doc_id_index.remove(doc_id) {
            let entry = self.docs.remove(pos);
            let removed = self.total_bytes.min(entry.size_bytes);
            self.total_bytes -= removed;
            self.aggregate_bytes.fetch_sub(removed, Ordering::Relaxed);
            self.doc_values.remove_at(pos);
            // Shift all indices above pos down by 1.
            for idx in self.doc_id_index.values_mut() {
                if *idx > pos {
                    *idx -= 1;
                }
            }
        }
    }

    /// Search the memtable using a simple term-based BM25 query.
    ///
    /// Tokenizes the query string and scores documents using BM25.
    /// If a "default" analyzer is registered (e.g. with synonym expansion),
    /// it is used; otherwise falls back to "standard".
    pub fn search_text(&self, query: &str, fields: &[&str], limit: usize) -> Vec<MemtableHit> {
        // Local-stats variant — preserved for direct callers + tests
        // that don't care about cross-shard correctness. The
        // FtsMemtable orchestrator uses search_text_with_global_stats
        // instead so BM25 reflects the union.
        self.search_text_with_global_stats(
            query,
            fields,
            limit,
            0,
            &std::collections::HashMap::new(),
            &std::collections::HashMap::new(),
            &std::collections::HashMap::new(),
            &std::collections::HashMap::new(),
        )
    }

    /// search_text variant that uses caller-supplied GLOBAL doc_count,
    /// per-field avg lengths, and per-(field,term) doc frequencies.
    /// Falls back to local stats when the global maps are empty.
    ///
    /// `global_field_doc_count` is the per-field `N` (docs WITH the field).
    /// When it carries the field, it wins over the scalar `global_doc_count` —
    /// that is the Lucene pairing (`FieldStats.docCount()` with
    /// `TermStats.docFreq()`, `BM25Similarity.idfExplain`) and the shape the
    /// index-wide statistics of #188 arrive in.  Empty ⇒ the historical
    /// single-scalar behaviour, so memtable-only searches score unchanged.
    // The stats params mirror the cross-shard aggregation the orchestrator
    // computes once per query; bundling them into a struct would just move
    // the arity into a builder for a single internal call site.
    #[allow(clippy::too_many_arguments)]
    pub fn search_text_with_global_stats(
        &self,
        query: &str,
        fields: &[&str],
        limit: usize,
        global_doc_count: u64,
        global_field_doc_count: &std::collections::HashMap<String, u64>,
        global_avg_field_len: &std::collections::HashMap<String, f32>,
        global_term_df: &std::collections::HashMap<(String, String), u64>,
        field_boosts: &std::collections::HashMap<String, f32>,
    ) -> Vec<MemtableHit> {
        let analyzer = match self
            .registry
            .get_analyzer("default")
            .or_else(|| self.registry.get_analyzer("standard"))
        {
            Some(a) => a,
            None => return Vec::new(),
        };

        let tokens = analyzer.analyze(query);
        if tokens.is_empty() {
            return Vec::new();
        }

        let doc_count = if global_doc_count > 0 {
            global_doc_count
        } else {
            self.docs.len() as u64
        };
        // Keyed by the interned `Arc<str>` doc id from the postings lists,
        // so accumulating a term's contribution is a pointer-bump clone
        // rather than a per-hit `String` allocation.
        let mut scores: HashMap<DocId, f32> = HashMap::new();

        for token in &tokens {
            // Search across requested fields (or all indexed fields if none specified).
            let search_fields: Vec<&str> = if fields.is_empty() {
                self.index.keys().map(|s| s.as_str()).collect()
            } else {
                fields.to_vec()
            };

            for field_name in &search_fields {
                let field_index = match self.index.get(*field_name) {
                    Some(fi) => fi,
                    None => continue,
                };

                let posting_list = match field_index.get(&token.text) {
                    Some(pl) => pl,
                    None => continue,
                };

                // Prefer the caller-supplied global doc_freq; fall back
                // to local posting list size if absent.
                let doc_freq: u64 = global_term_df
                    .get(&(field_name.to_string(), token.text.clone()))
                    .copied()
                    .unwrap_or(posting_list.len() as u64);
                let avg_field_len = global_avg_field_len
                    .get(*field_name)
                    .copied()
                    .unwrap_or_else(|| self.avg_field_length(field_name));
                // Per-field N (docs WITH the field) when the caller supplied
                // index-wide statistics; the scalar otherwise.
                let field_doc_count = global_field_doc_count
                    .get(*field_name)
                    .copied()
                    .filter(|n| *n > 0)
                    .unwrap_or(doc_count);

                let scorer = Bm25Scorer::new(avg_field_len, field_doc_count);
                // Per-field boost from the query tree (ES `boost` on match /
                // `field^N` on multi_match). 1.0 when unboosted, so scores
                // stay bit-identical for boost-free queries.
                let field_boost = field_boosts.get(*field_name).copied().unwrap_or(1.0);

                for (doc_id, &tf) in posting_list {
                    let field_len = self
                        .field_lengths
                        .get(*field_name)
                        .and_then(|m| m.get(doc_id))
                        .copied()
                        .unwrap_or(1);

                    let score = scorer.score_term(doc_freq, tf, field_len) * field_boost;
                    *scores.entry(doc_id.clone()).or_insert(0.0) += score;
                }
            }
        }

        let mut hits: Vec<MemtableHit> = scores
            .into_iter()
            .map(|(doc_id, score)| {
                // #191 — carry the arrival order with the hit.  `scores` is a
                // HashMap, so without it the pre-truncation order of an
                // all-tied hit set is the hash iteration order: the surviving
                // documents changed from run to run.
                let seq_no = self
                    .doc_id_index
                    .get(&doc_id)
                    .map_or(u64::MAX, |&pos| self.seq_no_at(pos));
                MemtableHit {
                    doc_id: doc_id.to_string(),
                    score,
                    seq_no,
                }
            })
            .collect();

        // Sort by the FULL page key (score DESC, seq_no ASC, `_id` ASC) — the
        // comparator `index.rs:16222-16228` applies to the final page — so a
        // truncation at `limit` keeps exactly the documents that page would.
        sort_hits_by_page_key(&mut hits);
        hits.truncate(limit);
        hits
    }

    /// Look up a document's indexed text fields by ID (used for BM25 scoring).
    ///
    /// Post-M4.9 this is computed lazily from `e.source` — we no
    /// longer eagerly cache a per-doc `HashMap<String, String>` at
    /// ingest time.  Query-time cost is one tree walk + ~10
    /// allocations, which is fine for the sub-second SLAs xerj
    /// targets on memtable-FTS fallback queries.
    pub fn get_source(&self, doc_id: &str) -> Option<HashMap<String, String>> {
        self.docs
            .iter()
            .find(|e| e.doc_id == doc_id)
            .map(|e| extract_text_fields_from(e.source.as_ref()))
    }

    /// Return all doc IDs stored in the memtable (in insertion order).
    pub fn all_doc_ids(&self) -> Vec<String> {
        self.docs.iter().map(|e| e.doc_id.clone()).collect()
    }

    /// First `n` doc ids (insertion order) — see
    /// `ShardedFtsMemtable::doc_ids_bounded`.
    pub fn doc_ids_up_to(&self, n: usize) -> Vec<String> {
        self.docs.iter().take(n).map(|e| e.doc_id.clone()).collect()
    }

    /// The WAL `seq_no` of the document at shard-local position `pos`
    /// (`u64::MAX` when the position is out of range).
    ///
    /// #191 — the bounded memtable paths must rank candidates ACROSS shards
    /// by the same key the final page sort uses, and `seq_no` is that key's
    /// tie-break.  Positions are only comparable within a shard; `seq_no` is
    /// global.
    pub fn seq_no_at(&self, pos: usize) -> u64 {
        self.docs.get(pos).map_or(u64::MAX, |e| e.seq_no)
    }

    /// `(seq_no, doc_id)` for the first `n` buffered documents, stopping early
    /// once an entry's `seq_no` exceeds `cutoff`.
    ///
    /// #191 — `docs` is append-ordered and `remove` preserves that order, so a
    /// shard's entries ascend by `seq_no`: once one entry is past the caller's
    /// running cap-th key, so is every entry after it, and neither needs its
    /// `doc_id` cloned.  That early break is what keeps the per-shard bound
    /// (which replaced a greedy "fill from shard 0" walk) from multiplying the
    /// id clones by the shard count.
    pub fn ranked_ids_up_to(&self, n: usize, cutoff: u64) -> Vec<(u64, String)> {
        let mut out = Vec::new();
        for e in self.docs.iter().take(n) {
            if e.seq_no > cutoff {
                break;
            }
            out.push((e.seq_no, e.doc_id.clone()));
        }
        out
    }

    /// Resolve a MemEntry's source Value — if `source` is Null but
    /// `source_bytes` is non-empty, lazily parse the bytes.  This is
    /// the M5.11 deferred-parse path used by `insert_raw_bytes_with_seq`.
    fn resolve_source(entry: &MemEntry) -> Value {
        if entry.source.is_null() && !entry.source_bytes.is_empty() {
            (**Self::memoized_parsed(entry)).clone()
        } else {
            (*entry.source).clone()
        }
    }

    /// Arc-sharing twin of `resolve_source`: no deep clone for the common
    /// HTTP-bulk path where the parsed source is already Arc-stored.
    fn resolve_source_arc(entry: &MemEntry) -> Arc<Value> {
        if entry.source.is_null() && !entry.source_bytes.is_empty() {
            Arc::clone(Self::memoized_parsed(entry))
        } else {
            Arc::clone(&entry.source)
        }
    }

    /// Parse `entry.source_bytes` at most once, caching the result in the
    /// entry's `parsed_memo`.  All read paths for M5.11 raw-bytes entries
    /// funnel through here so a scan of the buffered memtable parses each
    /// doc a single time for the memtable's lifetime instead of once per
    /// query.  The bytes are immutable for a given entry (a doc update
    /// creates a fresh `MemEntry` with a fresh memo), so the cache never
    /// goes stale.
    fn memoized_parsed(entry: &MemEntry) -> &Arc<Value> {
        entry.parsed_memo.get_or_init(|| {
            Arc::new(serde_json::from_slice(&entry.source_bytes).unwrap_or(Value::Null))
        })
    }

    /// All (doc_id, source) pairs WITHOUT deep-cloning the source trees.
    /// The fast-agg path was deep-cloning the entire memtable per agg
    /// request via `all_docs_with_sources` (~100-300 ms/query at 1e5
    /// buffered docs under a bulk writer).
    pub fn all_docs_with_sources_arc(&self) -> Vec<(String, Arc<Value>)> {
        self.docs
            .iter()
            .map(|e| (e.doc_id.clone(), Self::resolve_source_arc(e)))
            .collect()
    }

    /// `seq_no`-carrying twin of [`Self::all_docs_with_sources_arc`] (#191).
    pub fn all_docs_with_seq_arc(&self) -> Vec<(u64, String, Arc<Value>)> {
        self.docs
            .iter()
            .map(|e| (e.seq_no, e.doc_id.clone(), Self::resolve_source_arc(e)))
            .collect()
    }

    /// Return the full original source JSON for a document by ID.
    /// O(1) doc lookup via `doc_id_index` (maintained by every insert /
    /// remove path), with a linear-scan fallback in case an entry is ever
    /// missing from the index.
    ///
    /// Pre-fix this was an unconditional `docs.iter().find(..)` — a full
    /// linear walk of the shard per call.  `fill_memtable_sources`
    /// hydrates up to `materialisation_limit` (256) hits per search, so
    /// with a bulk writer keeping the memtable at 10⁵ docs every search
    /// paid 256 × O(shard) string compares; `get_doc_source_as_value` was
    /// the single hottest frame in the read-under-write profile.
    fn entry_for(&self, doc_id: &str) -> Option<&MemEntry> {
        if let Some(&i) = self.doc_id_index.get(doc_id) {
            if let Some(e) = self.docs.get(i) {
                if e.doc_id == doc_id {
                    return Some(e);
                }
            }
        }
        self.docs.iter().find(|e| e.doc_id == doc_id)
    }

    pub fn get_doc_source_as_value(&self, doc_id: &str) -> Option<Value> {
        self.entry_for(doc_id).map(Self::resolve_source)
    }

    /// Return the Arc-wrapped source for a document by ID.
    pub fn get_doc_source_arc(&self, doc_id: &str) -> Option<Arc<Value>> {
        self.entry_for(doc_id).map(Self::resolve_source_arc)
    }

    /// Iterate all stored documents as (doc_id, original_source) pairs.
    /// Lazily parses M5.11 raw-bytes entries.
    pub fn all_docs(&self) -> Vec<(String, Value)> {
        self.docs
            .iter()
            .map(|e| (e.doc_id.clone(), Self::resolve_source(e)))
            .collect()
    }

    /// Collect all docs as owned (doc_id, source) pairs for predicate-based scan.
    pub fn all_docs_with_sources(&self) -> Vec<(String, Value)> {
        self.docs
            .iter()
            .map(|e| (e.doc_id.clone(), Self::resolve_source(e)))
            .collect()
    }

    /// Check if a document exists in the memtable.  O(1) via
    /// `doc_id_index`, with linear fallback (see `entry_for`).
    pub fn contains(&self, doc_id: &str) -> bool {
        self.entry_for(doc_id).is_some()
    }

    /// Number of documents in the memtable.
    pub fn doc_count(&self) -> usize {
        self.docs.len()
    }

    /// Approximate total byte size of buffered data.
    pub fn size_bytes(&self) -> usize {
        self.total_bytes
    }

    /// Drain all documents (for flush).
    ///
    /// NOTE: uses `= HashMap::new()` (not `.clear()`) so the HashMap backing
    /// storage is actually released back to the allocator.  `HashMap::clear`
    /// keeps the internal bucket capacity around for reuse, which for a
    /// log-ingest workload means memory never shrinks after a flush —
    /// RSS grows monotonically until OOM.  See CAPPED_RAM_BATTLE for the bug.
    pub fn drain(&mut self) -> Vec<(String, HashMap<String, String>)> {
        // Sort by seq_no — see `drain_with_sources` for the rationale.
        let mut drained: Vec<MemEntry> = std::mem::take(&mut self.docs);
        drained.sort_by_key(|e| e.seq_no);
        let result = drained
            .into_iter()
            .map(|e| {
                let fields = extract_text_fields_from(e.source.as_ref());
                (e.doc_id, fields)
            })
            .collect();
        self.index = FxHashMap::default();
        self.doc_values = DocValues::default();
        self.aggregate_bytes
            .fetch_sub(self.total_bytes, Ordering::Relaxed);
        self.total_bytes = 0;
        self.field_lengths = FxHashMap::default();
        self.avg_field_lengths = FxHashMap::default();
        self.doc_id_index = FxHashMap::default();
        // Flush is the equivalent of a Lucene merge: tombstone contributions
        // are purged, so delete-aware ghost statistics reset to empty.
        self.ghost_docs = 0;
        self.ghost_field_len = FxHashMap::default();
        self.ghost_doc_freq = FxHashMap::default();
        // `self.docs` was drained in place; its Vec backing is kept (cheap).
        self.docs.shrink_to_fit();
        result
    }

    /// Return a snapshot of the current analyzed field encodings.
    ///
    /// Each entry represents a field whose sample threshold was reached and
    /// whose optimal encoding was determined by `FieldAnalyzer`.
    pub fn analyzed_field_encodings(&self) -> &HashMap<String, FieldEncoding> {
        &self.doc_values.analyzed_encodings
    }

    /// Drain all documents with their original source values (for flush to segment).
    ///
    /// Returns `(doc_id, text_fields, source)` in insertion order.
    pub fn drain_with_sources(&mut self) -> Vec<(String, HashMap<String, String>, Value)> {
        // M5.0 — sort drained docs by global WAL seq_no before
        // returning so the downstream FTS / stored-section writers
        // produce matching ordinals.  Out-of-order in-memory pushes
        // (introduced when we lift the WAL append out of the engine
        // memtable write lock so concurrent bulk batches pipeline)
        // get canonicalised here.  For workloads where all entries
        // carry `seq_no = 0` (the legacy `insert` path), `sort_by_key`
        // is stable so existing insertion order is preserved.
        let mut drained: Vec<MemEntry> = std::mem::take(&mut self.docs);
        drained.sort_by_key(|e| e.seq_no);
        let result = drained
            .into_iter()
            .map(|e| {
                let val = Arc::try_unwrap(e.source).unwrap_or_else(|arc| (*arc).clone());
                let fields = extract_text_fields_from(&val);
                (e.doc_id, fields, val)
            })
            .collect();
        // See `drain()` for why we must reassign these (not `.clear()`):
        // `HashMap::clear` retains internal bucket capacity and the RSS
        // never shrinks between flushes.
        self.index = FxHashMap::default();
        self.doc_values = DocValues::default();
        self.aggregate_bytes
            .fetch_sub(self.total_bytes, Ordering::Relaxed);
        self.total_bytes = 0;
        self.field_lengths = FxHashMap::default();
        self.avg_field_lengths = FxHashMap::default();
        self.doc_id_index = FxHashMap::default();
        // Flush is the equivalent of a Lucene merge: tombstone contributions
        // are purged, so delete-aware ghost statistics reset to empty.
        self.ghost_docs = 0;
        self.ghost_field_len = FxHashMap::default();
        self.ghost_doc_freq = FxHashMap::default();
        self.docs.shrink_to_fit();
        result
    }

    /// Return the source Value for a document by its internal index (O(1)).
    ///
    /// Unlike `get_doc_source_as_value` this uses the pre-known index position
    /// instead of scanning by doc_id, making it O(1) rather than O(N).
    pub fn get_doc_source_by_index(&self, idx: usize) -> Option<Value> {
        self.docs.get(idx).map(|e| (*e.source).clone())
    }

    /// Return a reference to the numeric DocValues column for `field`.
    ///
    /// Each slot corresponds to a document in insertion order (parallel to `docs`).
    /// `None` slots mean the document had no value for this field.
    pub fn doc_values_numeric_column(&self, field: &str) -> Option<&[Option<f64>]> {
        self.doc_values.numeric.get(field).map(|v| v.as_slice())
    }

    /// Return a reference to the keyword DocValues column for `field`.
    pub fn doc_values_keyword_column(&self, field: &str) -> Option<&[Option<String>]> {
        self.doc_values.keyword.get(field).map(|v| v.as_slice())
    }

    /// O(1) memtable count for `field == value` over the keyword column.
    /// Backed by the bounded-delta per-field `keyword_counts` map (folds
    /// only this field's appended tail).  `&self` — the fold uses interior
    /// mutability, so the caller holds only the shard READ lock.
    pub fn doc_values_keyword_count(&self, field: &str, value: &str) -> Option<u32> {
        self.doc_values.with_keyword_field(field, |c| {
            c.keyword_counts
                .get(field)
                .map(|m| m.get(value).copied().unwrap_or(0))
        })
    }

    /// O(1) memtable count for `field == value` over the numeric column.
    pub fn doc_values_numeric_count(&self, field: &str, value: f64) -> Option<u32> {
        self.doc_values.with_numeric_field(field, |c| {
            c.numeric_counts
                .get(field)
                .map(|m| m.get(&value.to_bits()).copied().unwrap_or(0))
        })
    }

    // ── Fast DocValues queries ────────────────────────────────────────────────

    /// Incremental per-shard sorted-candidate extraction — see
    /// `ShardedFtsMemtable::sort_candidates_numeric` for the contract and
    /// `SortCandCache` for the state.  Returns `(top page-order candidates
    /// as (key, doc_id), missing doc_ids, shard doc count)`; `None` when
    /// the shard is ineligible (un-normalisable value or array-valued
    /// field — poisoned until the next drain resets the memtable, so the
    /// fallback decision itself is O(1) per query, not a re-walk).
    pub fn sort_candidates_cached(
        &self,
        field: &str,
        desc: bool,
        cap: usize,
        normalize: &dyn Fn(&str) -> Option<f64>,
    ) -> Option<(Vec<(f64, String)>, Vec<String>, u64)> {
        let n = self.docs.len();
        let cap = cap.max(1);
        if self.doc_values.array_fields.contains(field) {
            return None;
        }
        let key = if desc {
            format!("{field}\u{1}d")
        } else {
            format!("{field}\u{1}a")
        };
        // Step 4: interior-mutable cache — held under the shard READ lock.
        let mut cache = self.doc_values.sort_cand_cache.lock();
        let entry = cache.entry(key).or_insert_with(|| SortCandCache {
            seen_docs: 0,
            cap,
            top: Vec::new(),
            missing: Vec::new(),
            poisoned: false,
        });
        if entry.poisoned {
            return None;
        }
        if entry.cap < cap || entry.seen_docs > n {
            // Larger page than cached, or the docs vec shrank without a
            // cache clear (defensive) — rebuild from scratch.
            entry.cap = cap;
            entry.seen_docs = 0;
            entry.top.clear();
            entry.missing.clear();
        }
        // Fold in the docs inserted since the last query.
        let ncol = self.doc_values.numeric.get(field);
        let kcol = self.doc_values.keyword.get(field);
        for i in entry.seen_docs..n {
            let nv: Option<f64> = ncol.and_then(|c| c.get(i).copied().flatten());
            let v: Option<f64> = match nv {
                Some(x) => Some(x),
                None => match kcol.and_then(|c| c.get(i)).and_then(|o| o.as_deref()) {
                    Some(s) => match normalize(s) {
                        Some(x) => Some(x),
                        None => {
                            entry.poisoned = true;
                            return None;
                        }
                    },
                    None => None,
                },
            };
            match v {
                Some(x) => entry.top.push((x, i)),
                None => {
                    if entry.missing.len() < entry.cap {
                        entry.missing.push(i);
                    }
                }
            }
        }
        entry.seen_docs = n;
        // Keep the pool bounded: page-order sort + cut to cap (+boundary
        // ties) once it doubles.  Top-cap of (old top-cap ∪ new docs) ==
        // top-cap of all docs, so the incremental cut is exact.
        let cut = |top: &mut Vec<(f64, usize)>, cap: usize| {
            if desc {
                top.sort_unstable_by(|a, b| {
                    b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)
                });
            } else {
                top.sort_unstable_by(|a, b| {
                    a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)
                });
            }
            let mut keep = top.len().min(cap);
            while keep < top.len() && keep > 0 && top[keep].0 == top[keep - 1].0 {
                keep += 1;
            }
            top.truncate(keep);
        };
        if entry.top.len() > entry.cap.saturating_mul(2).max(64) {
            cut(&mut entry.top, entry.cap);
        }
        // Result view: an exact top-`cap` (+ties) cut of the pool.
        let mut pool = entry.top.clone();
        cut(&mut pool, cap);
        let out: Vec<(f64, String)> = pool
            .into_iter()
            .map(|(v, i)| (v, self.docs[i].doc_id.clone()))
            .collect();
        let miss: Vec<String> = entry
            .missing
            .iter()
            .take(cap)
            .map(|&i| self.docs[i].doc_id.clone())
            .collect();
        Some((out, miss, n as u64))
    }

    /// Per-shard fused bool walk — see `ShardedFtsMemtable::doc_values_bool_query`.
    ///
    /// Column resolution and per-predicate match semantics mirror the
    /// standalone `doc_values_term_query` (keyword column, case-sensitive
    /// whole value, analyzed-text whitespace bailout) and
    /// `doc_values_range_query` (numeric column, gte/gt/lte/lt) exactly, so
    /// a bool of term+range predicates matches the same doc set the
    /// per-child queries would intersect to — without intermediate
    /// per-child hit vectors.
    ///
    /// #332 — the array bailout is part of "mirror the standalone paths
    /// exactly" and was MISSING here: `doc_values_term_query`,
    /// `doc_values_terms_query` and `filtered_docs_arc` all refuse a field
    /// that has ever carried an array, because `push_field` stores only the
    /// array's FIRST element in these single-valued columns. This walk did
    /// not, so it answered a bare `term` on a multi-valued keyword field
    /// from the lossy column and silently dropped every doc whose match was
    /// in a non-first element (`{"tags":["red","blue"]}` → `term tags:blue`
    /// = 0 hits) for as long as the doc stayed memtable-resident. It is the
    /// arm a plain `term` search actually takes (`is_doc_scan_query` →
    /// fused columnar, ahead of `try_doc_values_query`), so the standalone
    /// path's bailout never got a chance to fire.
    pub fn doc_values_bool_hits(
        &self,
        preds: &[MemBoolPred],
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        enum Col<'a> {
            Kw(&'a Vec<Option<String>>, &'a str, Option<f64>),
            KwAny(&'a Vec<Option<String>>, FxHashSet<String>, FxHashSet<u64>),
            NumEq(&'a Vec<Option<f64>>, f64),
            NumAny(&'a Vec<Option<f64>>, FxHashSet<u64>),
            Num(
                &'a Vec<Option<f64>>,
                Option<f64>,
                Option<f64>,
                Option<f64>,
                Option<f64>,
            ),
        }
        let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());
        for p in preds {
            match p {
                MemBoolPred::Term { field, value, num } => {
                    // Array-valued field: the column is lossy (first element
                    // only) → bail so a later matching element can't be
                    // dropped (#332; mirrors `filtered_docs_arc`).
                    if self.doc_values.array_fields.contains(field.as_str()) {
                        return None;
                    }
                    // #677: a dotted path (object sub-field / `.keyword`
                    // multi-field) is not faithfully represented by a single
                    // DV column — it resolves through `_source`. Bail to the
                    // source scan, else a `term` on `meta.owner` counts 0 while
                    // buffered but matches after flush (size:0 undercount).
                    if field.contains('.') {
                        return None;
                    }
                    if let Some(col) = self.doc_values.keyword.get(field.as_str()) {
                        // Step 2: analyzed-text bailout via the insert-time
                        // cached flag instead of an O(N) per-query column
                        // prescan.
                        if self
                            .doc_values
                            .keyword_has_whitespace
                            .contains(field.as_str())
                        {
                            return None;
                        }
                        cols.push(Col::Kw(col, value.as_str(), *num));
                    } else {
                        // Numeric term predicate: exact f64 equality (the
                        // predicate value was lowered to its string form).
                        let col = self.doc_values.numeric.get(field.as_str())?;
                        let needle = value.parse::<f64>().ok()?;
                        cols.push(Col::NumEq(col, needle));
                    }
                }
                MemBoolPred::Terms {
                    field,
                    values,
                    nums,
                } => {
                    if self.doc_values.array_fields.contains(field.as_str()) {
                        return None;
                    }
                    // #677: dotted path → source scan (see the Term arm).
                    if field.contains('.') {
                        return None;
                    }
                    // CIDR is not a literal in the column; the source scan
                    // expands it. An empty value list matches nothing and
                    // stays on this path — `None` would clone the memtable.
                    if values.iter().any(|v| v.contains('/')) {
                        return None;
                    }
                    if let Some(col) = self.doc_values.keyword.get(field.as_str()) {
                        if self
                            .doc_values
                            .keyword_has_whitespace
                            .contains(field.as_str())
                        {
                            return None;
                        }
                        let set: FxHashSet<String> = values.iter().cloned().collect();
                        let num_set: FxHashSet<u64> =
                            nums.iter().map(|n| (n + 0.0).to_bits()).collect();
                        cols.push(Col::KwAny(col, set, num_set));
                    } else {
                        let col = self.doc_values.numeric.get(field.as_str())?;
                        let mut set: FxHashSet<u64> = FxHashSet::default();
                        for v in values {
                            let needle = v.parse::<f64>().ok()?;
                            set.insert(needle.to_bits());
                        }
                        cols.push(Col::NumAny(col, set));
                    }
                }
                MemBoolPred::Range {
                    field,
                    gte,
                    gt,
                    lte,
                    lt,
                } => {
                    if self.doc_values.array_fields.contains(field.as_str()) {
                        return None;
                    }
                    // #677: dotted path → source scan (see the Term arm).
                    if field.contains('.') {
                        return None;
                    }
                    let col = self.doc_values.numeric.get(field.as_str())?;
                    cols.push(Col::Num(col, *gte, *gt, *lte, *lt));
                }
            }
        }
        let n = self.docs.len();
        let mut out: Vec<(String, usize)> = Vec::new();
        let mut total: u64 = 0;
        'doc: for idx in 0..n {
            for c in &cols {
                let ok = match c {
                    Col::Kw(col, want, num) => col
                        .get(idx)
                        .and_then(|o| o.as_deref())
                        .is_some_and(|v| v == *want || kw_equals_number(v, *num)),
                    Col::KwAny(col, set, nums) => {
                        col.get(idx).and_then(|o| o.as_deref()).is_some_and(|v| {
                            set.contains(v)
                                || (!nums.is_empty()
                                    && v.parse::<f64>()
                                        .is_ok_and(|f| nums.contains(&(f + 0.0).to_bits())))
                        })
                    }
                    Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),
                    Col::NumAny(col, set) => col
                        .get(idx)
                        .copied()
                        .flatten()
                        .is_some_and(|v| set.contains(&v.to_bits())),
                    Col::Num(col, gte, gt, lte, lt) => match col.get(idx).copied().flatten() {
                        None => false,
                        Some(v) => {
                            let pl = match (gte, gt) {
                                (Some(b), _) => v >= *b,
                                (None, Some(b)) => v > *b,
                                (None, None) => true,
                            };
                            let pu = match (lte, lt) {
                                (Some(b), _) => v <= *b,
                                (None, Some(b)) => v < *b,
                                (None, None) => true,
                            };
                            pl && pu
                        }
                    },
                };
                if !ok {
                    continue 'doc;
                }
            }
            total += 1;
            if out.len() < limit {
                out.push((self.docs[idx].doc_id.clone(), idx));
            }
        }
        Some((out, total))
    }

    /// Push `(doc_id, source_arc)` for every buffered position whose
    /// doc-values columns satisfy EVERY predicate (a pure conjunction of
    /// Term/Range).  Column resolution + per-predicate match semantics are the
    /// exact ones `doc_values_bool_hits` uses, so this enumerates the same doc
    /// set the fused-bool search walk would — but UNBOUNDED (all matches, the
    /// aggregation contract) and materialising the matching sources instead of
    /// just ids.
    ///
    /// Returns `None` (caller must fall back to the full-corpus walk) whenever
    /// column↔source equivalence with the JSON matcher isn't provable for a
    /// predicate field:
    ///   * the field has EVER carried an ARRAY value in this shard — the column
    ///     keeps only the first element, so a later element that matches under
    ///     `doc_matches_filter` would be silently dropped (a false negative the
    ///     aggregation could NOT recover);
    ///   * a keyword predicate field holds analyzed-text (whitespace) values;
    ///   * a predicate field has no column at all.
    ///
    /// For the shapes it DOES accept (scalar numeric/keyword columns) the
    /// columnar match set equals the `doc_matches_filter` set — the fast-agg
    /// caller still re-applies the JSON matcher as a safety net, so any residual
    /// false POSITIVE is removed and only the provable no-false-negative
    /// guarantee is load-bearing.
    pub fn filtered_docs_arc_into(
        &self,
        preds: &[MemBoolPred],
        out: &mut Vec<(String, Arc<Value>)>,
    ) -> Option<()> {
        enum Col<'a> {
            Kw(&'a Vec<Option<String>>, &'a str, Option<f64>),
            KwAny(&'a Vec<Option<String>>, FxHashSet<String>, FxHashSet<u64>),
            NumEq(&'a Vec<Option<f64>>, f64),
            NumAny(&'a Vec<Option<f64>>, FxHashSet<u64>),
            Num(
                &'a Vec<Option<f64>>,
                Option<f64>,
                Option<f64>,
                Option<f64>,
                Option<f64>,
            ),
        }
        let mut cols: Vec<Col<'_>> = Vec::with_capacity(preds.len());
        for p in preds {
            match p {
                MemBoolPred::Term { field, value, num } => {
                    // Array-valued field: the column is lossy (first element
                    // only) → bail so a later matching element can't be dropped.
                    if self.doc_values.array_fields.contains(field.as_str()) {
                        return None;
                    }
                    // #677: dotted path (object sub-field / `.keyword`) → source
                    // scan, so the count and filter paths agree (see the count
                    // fn `doc_values_bool_hits`).
                    if field.contains('.') {
                        return None;
                    }
                    if let Some(col) = self.doc_values.keyword.get(field.as_str()) {
                        if self
                            .doc_values
                            .keyword_has_whitespace
                            .contains(field.as_str())
                        {
                            return None;
                        }
                        cols.push(Col::Kw(col, value.as_str(), *num));
                    } else {
                        let col = self.doc_values.numeric.get(field.as_str())?;
                        let needle = value.parse::<f64>().ok()?;
                        cols.push(Col::NumEq(col, needle));
                    }
                }
                MemBoolPred::Terms {
                    field,
                    values,
                    nums,
                } => {
                    if self.doc_values.array_fields.contains(field.as_str()) {
                        return None;
                    }
                    // #677: dotted path → source scan (see the Term arm).
                    if field.contains('.') {
                        return None;
                    }
                    if values.iter().any(|v| v.contains('/')) {
                        return None;
                    }
                    if let Some(col) = self.doc_values.keyword.get(field.as_str()) {
                        if self
                            .doc_values
                            .keyword_has_whitespace
                            .contains(field.as_str())
                        {
                            return None;
                        }
                        let set: FxHashSet<String> = values.iter().cloned().collect();
                        let num_set: FxHashSet<u64> =
                            nums.iter().map(|n| (n + 0.0).to_bits()).collect();
                        cols.push(Col::KwAny(col, set, num_set));
                    } else {
                        let col = self.doc_values.numeric.get(field.as_str())?;
                        let mut set: FxHashSet<u64> = FxHashSet::default();
                        for v in values {
                            let needle = v.parse::<f64>().ok()?;
                            set.insert(needle.to_bits());
                        }
                        cols.push(Col::NumAny(col, set));
                    }
                }
                MemBoolPred::Range {
                    field,
                    gte,
                    gt,
                    lte,
                    lt,
                } => {
                    if self.doc_values.array_fields.contains(field.as_str()) {
                        return None;
                    }
                    // #677: dotted path → source scan (see the Term arm).
                    if field.contains('.') {
                        return None;
                    }
                    let col = self.doc_values.numeric.get(field.as_str())?;
                    cols.push(Col::Num(col, *gte, *gt, *lte, *lt));
                }
            }
        }
        let n = self.docs.len();
        'doc: for idx in 0..n {
            for c in &cols {
                let ok = match c {
                    Col::Kw(col, want, num) => col
                        .get(idx)
                        .and_then(|o| o.as_deref())
                        .is_some_and(|v| v == *want || kw_equals_number(v, *num)),
                    Col::KwAny(col, set, nums) => {
                        col.get(idx).and_then(|o| o.as_deref()).is_some_and(|v| {
                            set.contains(v)
                                || (!nums.is_empty()
                                    && v.parse::<f64>()
                                        .is_ok_and(|f| nums.contains(&(f + 0.0).to_bits())))
                        })
                    }
                    Col::NumEq(col, want) => col.get(idx).copied().flatten() == Some(*want),
                    Col::NumAny(col, set) => col
                        .get(idx)
                        .copied()
                        .flatten()
                        .is_some_and(|v| set.contains(&v.to_bits())),
                    Col::Num(col, gte, gt, lte, lt) => match col.get(idx).copied().flatten() {
                        None => false,
                        Some(v) => {
                            let pl = match (gte, gt) {
                                (Some(b), _) => v >= *b,
                                (None, Some(b)) => v > *b,
                                (None, None) => true,
                            };
                            let pu = match (lte, lt) {
                                (Some(b), _) => v <= *b,
                                (None, Some(b)) => v < *b,
                                (None, None) => true,
                            };
                            pl && pu
                        }
                    },
                };
                if !ok {
                    continue 'doc;
                }
            }
            let e = &self.docs[idx];
            out.push((e.doc_id.clone(), Self::resolve_source_arc(e)));
        }
        Some(())
    }

    /// Fast term query using the keyword column — O(N * string_compare).
    ///
    /// Returns `(doc_id, doc_index)` pairs for documents where the keyword
    /// column for `field` equals `value` (case-sensitive exact match).
    ///
    /// Returns `None` when the field has no keyword column (fall back to JSON scan).
    pub fn doc_values_term_query(
        &self,
        field: &str,
        value: &str,
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        if let Some(col) = self.doc_values.keyword.get(field) {
            // Step 2: text-field bailout via the insert-time cached flag — a
            // whitespace-containing keyword value means the column stores the
            // full analyzed-text source; a `term` query expects a token match
            // which only the FTS / scan path can serve.  Was an O(N) per-query
            // column prescan.
            //
            // Array/multi-valued bailout: a keyword ARRAY stores only its FIRST
            // element in this single-valued column (see `push_field`), so a
            // `term` on any non-first element would be silently missed. Bail to
            // the array-aware source scan (`json_values_equal` matches on array
            // membership). This is the fix for ES-incompatible term-on-array.
            if self.doc_values.keyword_has_whitespace.contains(field)
                || self.doc_values.array_fields.contains(field)
            {
                return None;
            }
            // Step 1: walk the whole column for an exact total but only clone
            // the doc_id String for the first `limit` matches — mirrors
            // `doc_values_bool_hits`.  The unbounded clone was ~all-matching-doc
            // String allocations per query at a drain-lagged memtable.
            let mut out: Vec<(String, usize)> = Vec::new();
            let mut total: u64 = 0;
            for (idx, opt) in col.iter().enumerate() {
                if opt.as_deref() == Some(value) {
                    total += 1;
                    if out.len() < limit {
                        out.push((self.docs[idx].doc_id.clone(), idx));
                    }
                }
            }
            return Some((out, total));
        }
        // Numeric column: `term` on a numeric field is lowered to its string
        // form by the caller; parse it back to f64 and match by exact equality
        // (mirrors the numeric matching in `doc_values_range_indices`). Without
        // this a `term` on a numeric field found no keyword column and returned
        // `None`, dropping the query onto the O(N) per-doc `_source` scan — a
        // selective numeric term (`code:500`, an id lookup) then paid a full
        // memtable walk. Bail (→ JSON scan) if the value isn't numeric.
        if let Some(col) = self.doc_values.numeric.get(field) {
            let needle = value.parse::<f64>().ok()?;
            let mut out: Vec<(String, usize)> = Vec::new();
            let mut total: u64 = 0;
            for (idx, opt) in col.iter().enumerate() {
                if *opt == Some(needle) {
                    total += 1;
                    if out.len() < limit {
                        out.push((self.docs[idx].doc_id.clone(), idx));
                    }
                }
            }
            return Some((out, total));
        }
        None
    }

    /// Index-only variant of `doc_values_term_query` that returns only
    /// the matching memtable positions, no cloned doc-id Strings. Used
    /// by the bool intersection shortcut to avoid 170k String allocations
    /// per query.
    pub fn doc_values_term_indices(&self, field: &str, value: &str) -> Option<Vec<usize>> {
        // Array/multi-valued keyword fields keep only their first element in the
        // single-valued column; term matching must fall back to the array-aware
        // source scan (mirrors the bail in `doc_values_term_query`).
        if self.doc_values.array_fields.contains(field) {
            return None;
        }
        let col = self.doc_values.keyword.get(field)?;
        let results: Vec<usize> = col
            .iter()
            .enumerate()
            .filter_map(|(idx, opt)| {
                if opt.as_deref() == Some(value) {
                    Some(idx)
                } else {
                    None
                }
            })
            .collect();
        Some(results)
    }

    /// Index-only variant of `doc_values_range_query`.
    pub fn doc_values_range_indices(
        &self,
        field: &str,
        gte: Option<f64>,
        gt: Option<f64>,
        lte: Option<f64>,
        lt: Option<f64>,
    ) -> Option<Vec<usize>> {
        // #1750 — the column holds only an array's first element.
        if self.doc_values.array_fields.contains(field) {
            return None;
        }
        let col = self.doc_values.numeric.get(field)?;
        let results: Vec<usize> = col
            .iter()
            .enumerate()
            .filter_map(|(idx, opt)| {
                let v = (*opt)?;
                let pl = match (gte, gt) {
                    (Some(b), _) => v >= b,
                    (None, Some(b)) => v > b,
                    (None, None) => true,
                };
                let pu = match (lte, lt) {
                    (Some(b), _) => v <= b,
                    (None, Some(b)) => v < b,
                    (None, None) => true,
                };
                if pl && pu {
                    Some(idx)
                } else {
                    None
                }
            })
            .collect();
        Some(results)
    }

    /// Fast terms query (multi-value term filter) using the keyword column.
    ///
    /// Returns `(doc_id, doc_index)` pairs for documents where the keyword
    /// column for `field` matches any of the given `values`.
    ///
    /// Returns `None` when the field has no keyword column (fall back to JSON scan).
    pub fn doc_values_terms_query(
        &self,
        field: &str,
        values: &[String],
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        let Some(col) = self.doc_values.keyword.get(field) else {
            // Numeric column: a `terms` over a numeric field — parse every
            // query value to f64 and match by exact equality (mirrors the
            // single-value numeric path in `doc_values_term_query`). Without
            // this a numeric `terms` fell onto the O(N) `_source` scan.
            if let Some(col) = self.doc_values.numeric.get(field) {
                let needles: Vec<f64> = values
                    .iter()
                    .filter_map(|v| v.parse::<f64>().ok())
                    .collect();
                if needles.len() != values.len() {
                    return None; // some value isn't numeric — JSON scan
                }
                let mut out: Vec<(String, usize)> = Vec::new();
                let mut total: u64 = 0;
                for (idx, opt) in col.iter().enumerate() {
                    if let Some(v) = *opt {
                        if needles.contains(&v) {
                            total += 1;
                            if out.len() < limit {
                                out.push((self.docs[idx].doc_id.clone(), idx));
                            }
                        }
                    }
                }
                return Some((out, total));
            }
            return None;
        };
        // Step 2: analyzed-text bailout via the insert-time cached flag.
        // If any stored keyword value in this column contains whitespace
        // it's likely an analyzed text field whose doc-values were built
        // from the full source string (not the token stream). A `terms`
        // query compares against tokens in that case, which doc-values
        // can't serve — bail so callers fall through to the scan path.
        // Array/multi-valued keyword fields (only the first element is stored
        // in the single-valued column) likewise bail to the array-aware scan.
        if self.doc_values.keyword_has_whitespace.contains(field)
            || self.doc_values.array_fields.contains(field)
        {
            return None;
        }
        // Step 1: exact total, bounded doc_id materialisation.
        let mut out: Vec<(String, usize)> = Vec::new();
        let mut total: u64 = 0;
        for (idx, opt) in col.iter().enumerate() {
            if let Some(v) = opt.as_deref() {
                if values.iter().any(|qv| qv == v) {
                    total += 1;
                    if out.len() < limit {
                        out.push((self.docs[idx].doc_id.clone(), idx));
                    }
                }
            }
        }
        Some((out, total))
    }

    /// Fast range query using the numeric column — O(N * f64_compare).
    ///
    /// Parameters mirror the ES range query: `gte` / `gt` / `lte` / `lt`.
    ///
    /// Returns `None` when the field has no numeric column, or has carried an
    /// array (fall back to JSON scan).
    pub fn doc_values_range_query(
        &self,
        field: &str,
        gte: Option<f64>,
        gt: Option<f64>,
        lte: Option<f64>,
        lt: Option<f64>,
        limit: usize,
    ) -> Option<(Vec<(String, usize)>, u64)> {
        // #1750 — an array-valued field keeps only its first element in the
        // numeric column; bail to the array-aware source scan, as
        // `doc_values_bool_hits` already does for the same predicate.
        if self.doc_values.array_fields.contains(field) {
            return None;
        }
        let col = self.doc_values.numeric.get(field)?;
        // Step 1: exact total, bounded doc_id materialisation — the
        // unbounded id clone was the biggest read-under-write term for
        // broad range filters (~all-of-memtable String allocations/query).
        let mut out: Vec<(String, usize)> = Vec::new();
        let mut total: u64 = 0;
        for (idx, opt) in col.iter().enumerate() {
            let v = match *opt {
                Some(v) => v,
                None => continue,
            };
            let passes_lower = match (gte, gt) {
                (Some(b), _) => v >= b,
                (None, Some(b)) => v > b,
                (None, None) => true,
            };
            let passes_upper = match (lte, lt) {
                (Some(b), _) => v <= b,
                (None, Some(b)) => v < b,
                (None, None) => true,
            };
            if passes_lower && passes_upper {
                total += 1;
                if out.len() < limit {
                    out.push((self.docs[idx].doc_id.clone(), idx));
                }
            }
        }
        Some((out, total))
    }

    /// Return all (term, frequency) pairs for a given field from the inverted index.
    ///
    /// Used by the term suggester to find candidate terms via edit distance, and
    /// by the completion suggester for prefix-based autocomplete over indexed terms.
    pub fn all_terms_for_field(&self, field: &str) -> Vec<(String, usize)> {
        match self.index.get(field) {
            Some(postings) => postings
                .iter()
                .map(|(term, docs)| (term.clone(), docs.len()))
                .collect(),
            None => Vec::new(),
        }
    }

    /// Return all distinct keyword values for a field (from the DocValues keyword column).
    ///
    /// Used by the completion suggester for fast prefix autocomplete on keyword fields.
    pub fn all_keyword_values_for_field(&self, field: &str) -> Vec<(String, usize)> {
        self.doc_values
            .with_keyword_field(field, |c| match c.keyword_set.get(field) {
                Some(set) => set.iter().map(|v| (v.clone(), 1)).collect(),
                None => Vec::new(),
            })
    }

    // ── Helpers ───────────────────────────────────────────────────────────────

    fn avg_field_length(&self, field_name: &str) -> f32 {
        match self.avg_field_lengths.get(field_name) {
            Some(&(total, count)) if count > 0 => (total / count as f64) as f32,
            _ => 1.0,
        }
    }
}

impl Default for FtsMemtable {
    fn default() -> Self {
        Self::new()
    }
}

// ── Helpers ───────────────────────────────────────────────────────────────────

/// Collect any string-valued field not in the schema (dynamic mapping).
/// Walks nested objects with dotted paths so a doc `{a: {b: {c: "x"}}}`
/// indexes `a.b.c: "x"` — queries targeting the leaf-specific dotted
/// path match only docs that carry that leaf. Also keeps a root-level
/// JSON-blob entry for each top-level object key so types like
/// flattened (queried at the root path) still find their inner tokens.
///
/// Hoisted out of `FtsMemtable::insert` so [`analyze_doc`] can run the
/// identical walk outside the shard write lock.
fn collect_text_fields(
    v: &Value,
    prefix: &str,
    out: &mut HashMap<String, String>,
    excluded: &std::collections::HashSet<String>,
) {
    if let Value::Object(obj) = v {
        for (k, val) in obj {
            let path = if prefix.is_empty() {
                k.clone()
            } else {
                format!("{}.{}", prefix, k)
            };
            // Matched on the FULL dotted path, not just the root key — #328.
            // Every pre-#328 member of the set is a root-level name, for which
            // `path == k` and this is the check that was here before; a
            // `dense_vector` nested under an object mapping is the first member
            // that can arrive as `passages.vec`.
            if excluded.contains(&path) {
                continue;
            }
            match val {
                Value::Object(_) => {
                    // Root-level JSON-blob for flattened-style
                    // whole-object queries.
                    //
                    // Built from a PRUNED copy of the object (#328): the blob
                    // is where a nested `dense_vector` actually landed. The
                    // recursion below skips `passages.vec`, but this line
                    // serialised the whole `passages` object — vector included
                    // — into one text field, so excluding the leaf alone moved
                    // no bytes at all. Pinned by
                    // `nested_dense_vector_is_excluded_from_its_parent_objects_term_dictionary`,
                    // which reports `<seg>.passages.fst` and requires it under
                    // 4 KiB — the fixture's 19,200 vector components cannot fit
                    // in that. Both mapping shapes are covered there: the
                    // dotted top-level name `"passages.vec"` and a `vec`
                    // sub-mapping under a `passages` object.
                    if prefix.is_empty() {
                        let t = extract_text_value_excluding(val, &path, excluded);
                        if !t.is_empty() && !out.contains_key(&path) {
                            out.insert(path.clone(), t);
                        }
                    }
                    collect_text_fields(val, &path, out, excluded);
                }
                Value::Array(arr) => {
                    let joined: String = arr
                        .iter()
                        .map(extract_text_value)
                        .collect::<Vec<_>>()
                        .join(" ");
                    if !joined.is_empty() && !out.contains_key(&path) {
                        out.insert(path, joined);
                    }
                }
                _ => {
                    let t = extract_text_value(val);
                    if !t.is_empty() && !out.contains_key(&path) {
                        out.insert(path, t);
                    }
                }
            }
        }
    }
}

/// Front half of `FtsMemtable::insert`, factored out as a pure function
/// so it can run OUTSIDE the memtable shard write locks (e.g. on the
/// rayon pool, one call per doc, before the bucketed per-shard insert).
///
/// Extracts the schema Text fields plus the dynamic-mapping dotted-path
/// text fields (exactly like `insert` did in-lock) and tokenises each
/// with `analyzer`.  Feed the result to [`FtsMemtable::insert_analyzed`].
pub fn analyze_doc(
    source: &Value,
    schema: &Schema,
    analyzer: &AnalyzerPipeline,
) -> Vec<(String, Vec<Token>)> {
    let excluded = fts_excluded_fields(schema);
    analyze_doc_excluding(source, schema, analyzer, &excluded)
}

pub fn analyze_doc_excluding(
    source: &Value,
    schema: &Schema,
    analyzer: &AnalyzerPipeline,
    excluded: &std::collections::HashSet<String>,
) -> Vec<(String, Vec<Token>)> {
    let mut text_fields: HashMap<String, String> = HashMap::new();

    // Index fields that are defined as Text in the schema.
    for field_cfg in &schema.fields {
        // `excluded` is authoritative for schema fields too, not only for the
        // dynamic-path sweep below. Before `index: false` was honoured the only
        // members were semantic-derived vector fields, which are never
        // `FieldType::Text`, so this loop could skip the check; a declared
        // `"index": false` Text field is exactly the case that needs it.
        if excluded.contains(&field_cfg.name) {
            continue;
        }
        if matches!(field_cfg.field_type, FieldType::Text) {
            if let Some(val) = source.get(&field_cfg.name) {
                let text = extract_text_value(val);
                if !text.is_empty() {
                    text_fields.insert(field_cfg.name.clone(), text);
                }
            }
        }
    }

    // Also index any string-valued field not in the schema (dynamic
    // mapping) — see `collect_text_fields`.
    if source.is_object() {
        collect_text_fields(source, "", &mut text_fields, excluded);
    }

    text_fields
        .into_iter()
        .map(|(field_name, text)| {
            let tokens = analyzer.analyze(&text);
            (field_name, tokens)
        })
        .collect()
}

/// Exact internal fields generated from semantic embedding mappings.
///
/// This is schema-derived: similarly named user fields are unaffected unless
/// an embedding config explicitly designates them as its target.
pub fn semantic_derived_vector_fields(schema: &Schema) -> std::collections::HashSet<String> {
    let mut excluded = std::collections::HashSet::new();
    for field in &schema.fields {
        let Some(embedding) = &field.embedding else {
            continue;
        };
        let target = embedding
            .target_field
            .clone()
            .unwrap_or_else(|| format!("{}_vector", field.name));
        excluded.insert(target.clone());
        excluded.insert(format!("{target}_chunks"));
        excluded.insert(format!(
            "{}{}",
            xerj_query::executor::PASSAGE_METADATA_PREFIX,
            target
        ));
    }
    excluded
}

/// Every field that must be kept OUT of the full-text inverted index.
///
/// Three disjoint reasons, unioned into one set so flush, merge, the memtable
/// analyser and `build_fts_field_configs` cannot disagree:
///
///  * semantic-derived vector fields (see above) — engine-internal payloads
///    that were never user text;
///  * fields the user MAPPED as `dense_vector` (`FieldType::Vector`) — #328.
///    See "Why a declared vector has no lexical form" below;
///  * fields whose mapping declares `"index": false` — #204's last open
///    accepted-and-ignored instance. The option was echoed by `GET _mapping`
///    and then ignored, so the field kept a full inverted index and stayed
///    searchable.
///
/// Derived from the SCHEMA rather than from the documents, exactly like
/// `doc_values_skip_set` — a merge must not resurrect postings that flush
/// correctly skipped. Dynamic (unmapped) fields are absent from the schema
/// and are unaffected: there is no declared `index: false` to honour.
///
/// ## Why `index: false` alone is not enough to drop the postings
///
/// Dropping a field's postings does not make it unmatchable here: the
/// per-segment `fts_has_field` gate routes the query onto the stored-doc
/// scan instead. That fallback is only usable where it is *equivalent* —
/// #204's second rule, "degrade loudly or not at all", applies to us as much
/// as to anyone.
///
///  * A **Text** field's scan comparison is analysed, which is exactly what
///    the postings did. Equivalent → the postings go.
///  * An **exact-typed** field (keyword / ip / date / numeric / boolean) is
///    FTS-indexed with the `keyword` analyzer: one whole-value token. The
///    scan's `match` arm splits on non-alphanumerics, so `192.168.0.1` would
///    start matching `192.168.0.2`. Measured, not theorised — it is the one
///    case `search/390_doc_values_search.yml` fails on. Weaker → the postings
///    stay, and they are what answers ES 8.1's "doc values search" exactly.
///    Nothing observable changes for those fields; the footprint saving is
///    knowingly forgone rather than bought with wrong answers.
///  * A field with **no doc values either** is unsearchable in ES's sense
///    (`MappedFieldType.isSearchable()` = has terms OR has doc values), and
///    `unsearchable_query_field` in `index.rs` rejects every query naming it
///    before execution — so no fallback of any strength ever runs, and the
///    postings are pure waste whatever the type.
///
/// ## Why a declared vector has no lexical form (#328)
///
/// `#12` removed the term dictionary for the vectors XERJ *generates* from a
/// semantic mapping. A field the user maps as `dense_vector` took a different
/// route through the same code and kept its own: kNN needs the HNSW graph, so
/// the field is `"index": true`, so the `index: false` arm above never fires.
/// The field then flowed through `extract_fts_fields_excluding` into
/// `extract_field_text`, whose `Value::Array` arm joins the elements with
/// spaces — turning a 128-dim vector into one enormous decimal-string term per
/// document. kNN is served by `hnsw/graph.bin`; `exists` is resolved from
/// `_source` / doc values; `_field_caps` and highlighting never open the
/// field's FST — so every one of those bytes is unreadable by construction.
///
/// Lucene draws the type line in exactly the same place and gets the same
/// consequence for free: `KnnFloatVectorField.createFieldType` sets vector
/// attributes and then `freeze()`s, and sets nothing else
/// (`lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java:70`;
/// `KnnByteVectorField.java:82` is identical), so `IndexOptions` stays `NONE`
/// (`lucene/core/src/java/org/apache/lucene/index/FieldInfo.java:565`) and a
/// term dictionary for the field cannot exist. `ColumnValidation` enumerates
/// "vectors" as an indexing feature DISJOINT from "index options"
/// (`lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java:104`)
/// — the same type-level separation this is.
///
/// Measured — one harness, one corpus, built twice and diffed file by file
/// after `force_merge(1)`. 5,000 docs × 128 dim: `text` + `keyword` + `long` +
/// a `dense_vector` + its `_chunks` companion (the corpus carries the companion
/// because that is the shape RFC #148 reports). Every byte below is a
/// `std::fs::metadata().len()` sum over the data dir, not an estimate:
///
/// | file, 5,000 × 128 | `main` | this branch |
/// |---|---:|---:|
/// | `<seg>.emb.fst` | 13,256,659 B | 0 |
/// | `<seg>.emb.post` | 10,834 B | 0 |
/// | `<seg>.emb.norms` | 48 B | 0 |
/// | `<seg>.emb_chunks.fst` | 26,518,005 B | 0 |
/// | `<seg>.emb_chunks.post` | 10,834 B | 0 |
/// | `<seg>.body.fst` | 278 B | 278 B |
/// | `<seg>.cat.fst` | 53 B | 53 B |
/// | `<seg>.n.fst` | 209 B | 209 B |
/// | **durable index** (all files bar `.wal`) | **54,068,549 B** | **14,251,975 B** (−73.6%, 3.79×) |
///
/// Two observations the table is here to make. First, the term dictionary of a
/// vector-carrying index essentially *is* the vector field: the three lexical
/// `.fst` files total 540 B against 39.8 MB of vector postings. Second — and
/// this is why the exclusion cannot stop at the base name — the `_chunks`
/// companion is the BIGGER half. At 26,518,005 B it is twice `emb.fst` and 49%
/// of `main`'s whole index on its own, so a base-name-only exclusion would
/// leave more behind than it removes.
///
/// The row is the DURABLE index — every file in the data directory except
/// `.wal` — because the write-ahead log's tail is reclaimed asynchronously and
/// the whole-directory total is therefore not reproducible: runs of the 300-doc
/// regression test have put it anywhere between 858,147 B and 1,105,001 B while
/// the durable bytes moved by ~500 B. `.seg` (the stored documents, 11.2 MB
/// here) is counted on both sides and is why the ratio is 3.79× rather than the
/// 39.8 MB the term dictionaries cost outright.
///
/// The 300-doc × 128-dim fixture the regression tests actually run is the same
/// shape one order of magnitude down: `emb.fst` 796,274 B, `emb.post` 591 B,
/// `emb.norms` 313 B, `emb_chunks.fst` 1,592,118 B, `emb_chunks.post` 591 B,
/// durable index 3,249,323 B → 857,265–857,769 B (eight runs on this revision
/// printed 857,265 / 857,529 / 857,533 / 857,545 / 857,549 / 857,657 /
/// 857,657 / 857,769, a 504 B spread). That test prints both totals and asserts
/// a ceiling on the durable one, so this figure is enforced rather than merely
/// published. The same eight runs are quoted verbatim in `CHANGELOG.md` and in
/// the comment above the `eprintln!` in
/// `user_mapped_dense_vector_builds_no_fts_term_dictionary`; all three move
/// together or none of them do.
///
/// TWO RULES DECIDE THIS SET, and they are stated separately because they do
/// not have the same strength. [`lexically_typeless_fields`] has the full
/// statement; the summary is:
///
///  * BY DECLARED TYPE, for the field itself: `FieldType::Vector` and nothing
///    else. Not by value shape — an unmapped numeric array stays lexical
///    exactly as before (the same line HNSW draws in
///    `load_hnsw_artifacts_sync`, which refuses a graph pinned to a
///    non-`dense_vector` field). `FieldType::Chunk` is deliberately NOT
///    excluded: its declared payload is chunk *text* plus a vector, and only
///    the vector half is droppable — that needs a value-shape-aware rule, not
///    a type rule.
///  * BY NAME, for the two companions `<field>_chunks` and
///    `__xerj_passage_meta__<field>`, which arrive in `_source` with no
///    `FieldConfig` behind them, so there is no declared type to read. This
///    rule is the weaker of the two and YIELDS to ANY declaration: a field
///    mapped as anything other than `dense_vector` under one of those names —
///    `text`, `keyword`, `long`, `double`, `date`, `boolean`, `ip`,
///    `geo_point`, `binary`, `object`, `nested`, `chunk`, or an ES type string
///    `es_type_to_field_type` does not recognise — keeps its full lexical
///    surface. That is only sound because the companion is prevented from ever
///    ACQUIRING a declaration: `index::is_undeclared_multi_vector_companion`
///    stops dynamic mapping registering a `<vector>_chunks` key whose value is
///    a real multi-vector, so an "absent" answer here means absent.
///
/// The bytes in the table above are almost all the second rule's: `emb_chunks`
/// in that fixture is absent from the user's mapping — and, since this change,
/// absent from the evolved schema too — and it is 26,518,005 B of the
/// 39,796,380 B removed.
///
/// Removing the postings is only half of it, and the other half is NOT the
/// `*`-expansion filter. Every other member of this set is protected from the
/// stored-doc scan by `unsearchable_query_field`, which rejects the query
/// before execution; a `dense_vector` is not, because it IS indexed. So two
/// things happen on the query side:
///
///  * `text_fields` / `exact_fields` in `search_inner` drop vector fields, so a
///    `*` expansion stops projecting a clause onto them;
///  * and `index::lower_lexically_typeless_clauses` lowers any clause that
///    NAMES one to `match_none` at plan time. The expansion filter alone does
///    nothing for `{"term":{"emb":…}}` or `{"multi_match":{"fields":["emb"]}}`,
///    which name the field outright — those fall through the per-segment
///    `fts_has_field` gate onto the stored-doc scan, whose `Term` arm matches
///    ANY ELEMENT of a JSON array (`json_values_equal`) and whose `match` arm
///    splits on non-alphanumerics.
///
/// Measured on the corpus above, post-`force_merge`, three source states —
/// `main` / write-side exclusion only (postings gone, no plan-time lowering) /
/// this branch. Timings are the FIRST call for each shape, before the result
/// cache has seen it; a repeat of any row here is ~0.002 ms on all three and
/// measures the cache, not the plan:
///
/// | query | `main` | postings gone, no lowering | this branch |
/// |---|---|---|---|
/// | `term {emb: "<component>"}` | 0 hits, 89.2 ms | 0 hits, 186.8 ms | 0 hits, 0.061 ms |
/// | `term {emb: <component>}` numeric | **1 hit**, 87.6 ms | **1 hit**, 187.6 ms | **0 hits**, 0.011 ms |
/// | `terms {emb: [<component>]}` numeric | **1 hit**, 85.6 ms | **1 hit**, 112.0 ms | **0 hits**, 0.012 ms |
/// | `prefix {emb: "0"}` | **2,500 hits**, 231.8 ms | 0 hits, 95.0 ms | **0 hits**, 0.008 ms |
/// | `wildcard {emb: "0*"}` | **5,000 hits**, 316.6 ms | 0 hits, 95.1 ms | **0 hits**, 0.008 ms |
/// | `fuzzy {emb: "0"~2}` | **5,000 hits**, 798.5 ms | 0 hits, 97.7 ms | **0 hits**, 0.011 ms |
/// | `match_phrase_prefix {emb: "0"}` | **50 hits**, 84.6 ms | 0 hits, 89.3 ms | **0 hits**, 0.009 ms |
/// | `range {emb: {gte:-2, lte:2}}` | 0 hits, 119.8 ms | 0 hits, 134.5 ms | 0 hits, 0.009 ms |
/// | `multi_match {fields:["emb"], query:"0"}` | 0 hits, 0.409 ms | **5,000 hits**, 123.5 ms | 0 hits, 0.012 ms |
/// | `simple_query_string {fields:["emb"]}` | 0 hits, 0.033 ms | 0 hits, 114.5 ms | 0 hits, 0.015 ms |
/// | `term {emb_chunks: "<component>"}` | 0 hits, 86.0 ms | 0 hits, 116.6 ms | 0 hits, 0.008 ms |
/// | `wildcard {emb_chunks: "0*"}` | **5,000 hits**, 626.9 ms | 0 hits, 89.9 ms | **0 hits**, 0.007 ms |
/// | `multi_match {fields:["emb_chunks"], query:"0"}` | 0 hits, 2.24 ms | **5,000 hits**, 138.5 ms | 0 hits, 0.014 ms |
/// | `bool{should:[term emb], must_not:[term cat]}` | 0 hits, 81.4 ms | 0 hits, 92.4 ms | 0 hits, 0.011 ms |
/// | `bool{must_not:[term emb]}` | 5,000 hits, 84.9 ms | 5,000 hits, 98.9 ms | 5,000 hits, 2.31 ms |
/// | `constant_score{filter: term emb}` | 0 hits, 83.0 ms | 0 hits, 110.3 ms | 0 hits, 0.012 ms |
/// | `exists {field: emb}` | 5,000 hits, 83.0 ms | 5,000 hits, 108.0 ms | 5,000 hits, 88.5 ms |
/// | `match {body: "liquidity"}` | 5,000 hits, 5.18 ms | 5,000 hits, 5.96 ms | 5,000 hits, 4.95 ms |
/// | `term {cat: "even"}` | 2,500 hits, 2.32 ms | 2,500 hits, 2.87 ms | 2,500 hits, 2.27 ms |
///
/// The two `multi_match` rows are the ones that are not merely slow. Without the
/// lowering they answer EVERY document, because the scan renders the float array
/// to text and every component contains a `0` — a 0-hit query silently becomes
/// a match-all. Those rows are the reason the write-side change cannot ship on
/// its own.
///
/// A WHOLE CLASS OF ANSWERS CHANGES, and it is stated rather than buried. SEVEN
/// rows above differ between `main` and this branch (nine shapes on the 300-doc
/// regression fixture, which probes both companions), and they are all the same
/// thing: a `dense_vector` and its UNDECLARED `_chunks` companion no longer
/// have any lexical surface, so every lexical leaf naming one OUTRIGHT answers
/// 0. Two exclusions from that sentence, both measured and both stated where
/// they live: a `fields: ["emb.*"]` PATTERN is not lowered and still answers
/// 300 of 300 for `"0"` on the fixture, exactly as on the merge base
/// (`index::lower_lexically_typeless_clauses`'s `is_typeless_spec`), and a
/// `<field>_chunks` the user mapped `text`/`keyword`/`date`/`boolean`/`ip` is
/// not in the set at all, so nothing naming it changes
/// ([`lexically_typeless_fields`]). On `main` those
/// clauses reached the stored-doc scan, which renders the float array back to
/// decimal text and matches against THAT — its `Term` arm matches any ELEMENT of
/// a JSON array (`json_values_equal`) and its `match` arm splits on
/// non-alphanumerics — so a shape answered 0 or answered most of the corpus
/// purely according to whether the rendered decimals happened to contain the
/// probe. `wildcard {emb:"0*"}` and `fuzzy {emb:"0"~2}` matched EVERY document.
/// None of that is a retrieval semantic anyone asked for: ES rejects these
/// queries on a `dense_vector` outright (`DenseVectorFieldType` has no term
/// query) and Lucene gives a field with no terms an empty scorer
/// (`lucene/core/src/java/org/apache/lucene/index/Terms.java:40`). 0 is the
/// Lucene answer. It is still a behaviour change and the CHANGELOG describes the
/// class; `user_mapped_dense_vector_builds_no_fts_term_dictionary`'s (d) loop
/// pins it shape by shape.
///
/// `_score` MOVES TOO, for bools whose hit count does not. Dropping the dead
/// clause out of `should` makes the surviving bool projectable through
/// `query_node_to_fts_with_keyword_fields`, so BM25 scores it where the
/// stored-doc scan used to. On the 300-doc fixture,
/// `bool{should:[term emb, term cat:"even"]}` keeps its 150 hits but goes
/// `_score` 0.008402659 → 0.6931471 and `max_score` 1.6931472 → 0.6931471, and
/// `bool{must:[match body], must_not:[term emb]}` keeps its 300 hits and its
/// `_score` but goes `max_score` 1.6931472 → 0.008402659. Both moves are toward
/// consistency: on `main` these reported a `max_score` no returned hit carried.
/// Isolated to the plan-time fold — with the write-side exclusion alone and the
/// lowering disabled, both are byte-identical to `main`. Pinned in that test's
/// (d2c) block.
///
/// Not claimed and not changed: XERJ still answers 200 rather than 400 for
/// these shapes. Adding ES's rejection is a separate, breaking change.
pub fn fts_excluded_fields(schema: &Schema) -> std::collections::HashSet<String> {
    let mut excluded = semantic_derived_vector_fields(schema);
    excluded.extend(lexically_typeless_fields(schema));
    for field in &schema.fields {
        if field.options.indexed {
            continue;
        }
        let fallback_is_equivalent =
            matches!(field.field_type, FieldType::Text) || !field.options.doc_values;
        if fallback_is_equivalent {
            excluded.insert(field.name.clone());
        }
    }
    excluded
}

pub const MULTI_VECTOR_COMPANION_SUFFIX: &str = "_chunks";

/// Is `val` the exact `_source` shape RFC #148 gives a per-document
/// MULTI-VECTOR — a non-empty array of equal-length, non-empty, all-numeric
/// arrays?
///
/// This is the value-shape half of the companion rule. The schema alone cannot
/// tell a multi-vector from a user's own numeric field (see
/// [`declares_non_vector_shaped_field`]); the VALUE can, and it is available at
/// exactly the moment that matters — when dynamic mapping is about to invent a
/// `FieldConfig` for an unmapped `<vector>_chunks` key
/// (`index::evolve_schema_from_doc`/`_docs`). Refusing to invent one there is
/// what keeps the schema honest, so the plan-time side of the rule can then
/// defer to ANY declaration it does find.
///
/// The equal-length requirement is not decoration: every chunk of one document
/// embeds with the same model, so a real multi-vector is rectangular. A ragged
/// array of numeric arrays is therefore NOT one, is left to dynamic mapping,
/// and keeps its term dictionary — the fail-safe direction.
///
/// Lucene models the same object as its own field type rather than inferring a
/// numeric one: `LateInteractionField extends BinaryDocValuesField`
/// (`lucene/core/src/java/org/apache/lucene/document/LateInteractionField.java:36`),
/// constructed from a `float[][]` and encoded straight to a `BytesRef`
/// (`:44`, `:45`) — it never reaches an analyzer or a term dictionary at all.
pub(crate) fn looks_like_multi_vector(val: &serde_json::Value) -> bool {
    let Some(rows) = val.as_array() else {
        return false;
    };
    let mut dims: Option<usize> = None;
    if rows.is_empty() {
        return false;
    }
    for row in rows {
        let width = if let Some(text) = row.as_str() {
            // Packed chunk vectors (#1940) are one string per passage, not
            // a number array. Same rectangular rule as the array form.
            match xerj_common::packed_f32::unpack_f32_vector(text) {
                Some(values) => values.len(),
                None => return false,
            }
        } else {
            let Some(components) = row.as_array() else {
                return false;
            };
            if components.is_empty() || !components.iter().all(serde_json::Value::is_number) {
                return false;
            }
            components.len()
        };
        match dims {
            None => dims = Some(width),
            Some(d) if d == width => {}
            Some(_) => return false,
        }
    }
    true
}

/// Fields whose DECLARED type has no lexical representation at all — #328.
///
/// Today that is exactly `FieldType::Vector` (`dense_vector`), so the base list
/// is deliberately the SAME list, from the same walk, that decides which fields
/// get an HNSW graph: [`crate::index::collect_dense_vector_fields`]. Two
/// independent notions of "which fields are vectors" is how you end up with a
/// field that has both a graph and a term dictionary, or neither — so there is
/// one walk and this is a view of it.
///
/// Each vector contributes THREE names, exactly the shape
/// [`semantic_derived_vector_fields`] already emits per target, because a
/// `dense_vector` never arrives alone: `passage_scored_vector_fields` unions
/// user-mapped vectors with semantic targets and both grow a `<field>_chunks`
/// companion (the per-document MULTI-vector) and a
/// `__xerj_passage_meta__<field>` sidecar. Excluding only the base name leaves
/// the companion behind, and the companion is the BIGGER half: on the 5,000-doc
/// × 128-dim corpus measured in [`fts_excluded_fields`], `<seg>.emb_chunks.fst`
/// is 26,518,005 B against `<seg>.emb.fst`'s 13,256,659 B — twice the size, and
/// 49% of `main`'s whole 54,068,549 B durable index on its own.
///
/// Lucene draws the same line and never lets a multi-vector reach an analyzer:
/// its per-document float[][] shape is `LateInteractionField`, which `extends
/// BinaryDocValuesField`
/// (`lucene/core/src/java/org/apache/lucene/document/LateInteractionField.java:36`)
/// and hands its constructor straight to `super(name, encode(value))` (`:45`;
/// the constructor signature is `:44`)
/// — doc values only, no index options, so no term dictionary can exist for it
/// any more than for the single-vector `KnnFloatVectorField`
/// (`KnnFloatVectorField.java:70`: vector attributes, then `freeze()`).
///
/// TWO RULES, and they are not the same rule. The first is BY DECLARED TYPE:
/// `FieldType::Vector` has no lexical form, so the base name is excluded
/// unconditionally. The second is BY NAME: the two companions are synthesised
/// from the base name because no USER mapping declares them — they arrive in
/// `_source` from the passage pipeline, and whatever `FieldConfig` they end up
/// with is one dynamic mapping inferred from their values. A name-based rule
/// can collide with a field the user actually mapped, so it yields to ANY
/// declaration it finds:
///
///  * declared as anything other than `dense_vector` — `text`, `keyword`,
///    `long`, `double`, `date`, `boolean`, `ip`, `geo_point`, `binary`,
///    `object`, `nested`, `chunk`, or a mapping type
///    `es_type_to_field_type` did not recognise — the declaration wins and the
///    field keeps its term dictionary and its whole lexical surface. Lucene
///    draws exactly this line: whether a field gets a term dictionary is read
///    off the field's own `IndexableFieldType` (`IndexingChain.invertAndStore`,
///    `lucene/core/src/java/org/apache/lucene/index/IndexingChain.java:1416`),
///    never off its name, and one name meaning two different things is an
///    outright error rather than a silent downgrade
///    (`IndexingChain.FieldSchema.setIndexOptions` → `assertSame` →
///    `raiseNotSame`, `IndexingChain.java:2213`, `:2187` and `:2193`);
///  * absent → the name rule applies, and this is the companion's own case.
///
/// THAT SECOND BULLET IS ONLY SAFE BECAUSE THE COMPANION IS KEPT UNDECLARED.
/// `<field>_chunks` is user-supplied `_source` (an array of float arrays), so
/// dynamic mapping used to register it like any other unmapped key —
/// `infer_field_type` walks to the first scalar and lands on
/// `FieldType::Double`, a `FieldConfig` byte-identical to a user's own
/// `"type": "double"`. That is what forced an earlier revision to carve
/// `double`/`long` out of the yield set above, and with them a real user field
/// under that name. `index::is_undeclared_multi_vector_companion` removes the
/// ambiguity at its source instead: dynamic mapping refuses to invent the
/// config when the value [`looks_like_multi_vector`], so the only `double` or
/// `long` under this name is one somebody wrote.
///
/// BOTH HALVES ARE LOAD-BEARING, measured by disabling each on its own against
/// the committed 300-doc × 128-dim fixture and the 50-doc collision fixture:
///
/// | | `.emb_chunks.fst` (300-doc) | durable index | declared `long` sibling |
/// |---|---|---|---|
/// | here | **0 B** | 857,265–857,769 B | 44 B `.fst`, answers 25 |
/// | yield-to-numeric restored | 0 B | 857,445 B | **0 B `.fst`, answers 0** |
/// | refusal disabled | **1,592,118 B** | **2,451,336 B** | 44 B, answers 25 |
///
/// The middle row is the residual this replaces; the bottom row is the 67% of
/// the saving an unconditional "any declaration wins" gives up. Neither is
/// paid for here.
///
/// This is deliberately NOT the contract `semantic_derived_vector_fields`
/// carries: that one synthesises `<target>_chunks` unconditionally, and a
/// declared field of that name has always lost its postings to it. That
/// behaviour is untouched here — it is `main`'s, on a mapping shape this PR
/// does not introduce — but it is not a precedent this rule reuses, because it
/// fires only when an embedding config designates the target, whereas this rule
/// fires for every user-mapped `dense_vector`. Nor is
/// `observe_passage_chunks_in_source` a precedent: it reserves `<field>_chunks`
/// only when `_source` holds a NON-EMPTY ARRAY there
/// (`index.rs`'s `Some(Value::Array(chunks)) if !chunks.is_empty()`), so a
/// string or keyword value was never captured by it. Measured on
/// `{text body, keyword emb_chunks, dense_vector emb}`, 50 docs × 16 dim, half
/// `tenant-a` — three source states, all three run in one worktree: the merge
/// base `ca4d75a` (`main` when this branch was cut), the unconditional
/// name-only rule this replaces (`e629a58`), and here.
///
/// | | `main` | name-only | here |
/// |---|---|---|---|
/// | `<seg>.emb_chunks.fst` | 54 B | **0** | 54 B |
/// | `<seg>.emb.fst` | 16,713 B | 0 | 0 |
/// | `term {emb_chunks:"tenant-a"}` | 25 | **0** | 25 |
/// | `terms {emb_chunks:["tenant-a"]}` | 25 | **0** | 25 |
/// | `bool{filter:[term emb_chunks]}` | 25 | **0** | 25 |
/// | `bool{must:[match body],filter:[term …]}` | 25 | **0** | 25 |
/// | `constant_score{filter:term emb_chunks}` | 25 | **0** | 25 |
/// | `bool{must_not:[term emb_chunks]}` | 25 | **50** | 25 |
/// | `bool{must:[match_all],must_not:[term …]}` | 25 | **50** | 25 |
/// | `terms` agg on `emb_chunks` | 25/25 | 25/25 | 25/25 |
///
/// The same schema with `emb_chunks` mapped `text` (values `tenant-a widgets` /
/// `tenant-b gadgets`, so one token is unique to each half):
///
/// | | `main` | name-only | here |
/// |---|---|---|---|
/// | `<seg>.emb_chunks.fst` | 71 B | **0** | 71 B |
/// | `match {emb_chunks:"widgets"}` | 25 | **0** | 25 |
/// | `match_phrase {emb_chunks:"tenant-a widgets"}` | 25 | **0** | 25 |
/// | `multi_match {fields:["emb_chunks"]}` | 25 | **0** | 25 |
/// | `simple_query_string {fields:["emb_chunks"]}` | 25 | **0** | 25 |
/// | `prefix {emb_chunks:"widget"}` | 25 | **0** | 25 |
/// | `bool{must:[match_all],must_not:[match …]}` | 25 | **50** | 25 |
/// | unfielded `query_string {"query":"widgets"}` | 25 | 25 | 25 |
///
/// Every ANSWER row is back to `main`'s value while `emb.fst` stays gone, which
/// is the whole claim: the vector loses its term dictionary, the declared field
/// beside it loses nothing.
///
/// The `must_not` rows are why this is a correctness rule and not a footprint
/// one: under the name-only rule the clause lowered to `match_none`, the Bool
/// fold dropped it as "excludes nothing", and the query RETURNED the 25
/// documents the exclusion was written to remove — with the aggregation proving
/// the data was present and correctly populated the whole time. The unfielded
/// `query_string` row is the self-inconsistency it produced: 25 through the
/// stored-doc scan while every fielded shape said 0.
///
/// One shape is NOT in these tables and deliberately not asserted anywhere:
/// bare `bool{must_not:[match …]}` with no `must`/`filter` answers 0 on
/// `ca4d75a` (and 50 under the name-only rule). That is a PRE-EXISTING
/// XERJ quirk with nothing to do with #328 — a control index with no vector
/// field at all, `{text body, text tags, keyword kw}`, answers
/// `bool{must_not:[match tags:"widgets"]}` = 0 and
/// `bool{must:[match_all],must_not:[match tags:"widgets"]}` = 25 there too.
/// The `match_all`-anchored form is the one the tests pin, because it is the
/// one that is correct on `main` to begin with.
///
/// THE NUMERIC ARM, which an earlier revision of this branch shipped as an
/// accepted residual and which is now closed. On that revision a field the user
/// declared `long` or `double` under one of these two names was still excluded
/// by name and the plan-time lowering still zeroed every lexical clause on it.
/// Measured on `{text body, long emb_chunks, dense_vector emb}`, 50 docs,
/// values 7 and 9 — merge base `ca4d75a` → that revision → here:
///
///   `term {emb_chunks:7}`                      25 → **0** → 25
///   `terms {emb_chunks:[7]}`                   25 → **0** → 25
///   `range {emb_chunks:{gte:8}}`               25 → **0** → 25
///   `bool{filter:[term emb_chunks 7]}`         25 → **0** → 25
///   `bool{must:[match_all],must_not:[term 7]}` 25 → **50** → 25
///   `terms` agg / `stats` on `emb_chunks`      25/25 on all three
///
/// The `double` arm and the DYNAMICALLY mapped numeric case (a scalar `7`/`9`
/// under this name with no mapping at all, which dynamic mapping registers as
/// `long`) behave identically and are pinned alongside it. It was NOT closed by
/// per-field schema provenance and NOT by giving up the by-name rule — the two
/// options an earlier revision of this comment called exhaustive, wrongly. It
/// is closed by not manufacturing the ambiguous declaration in the first place
/// (`index::is_undeclared_multi_vector_companion`), which costs nothing: the
/// 300-doc fixture still reports `emb_chunks.fst=0` and durable 857,265–857,769 B.
///
/// THE LIMIT THAT REMAINS, and it fails safe: the refusal needs the
/// `dense_vector` declared BEFORE the documents arrive. Index first and map
/// afterwards and `emb_chunks` already carries a dynamic `double` config, which
/// this function then yields to — measured `.emb_chunks.fst` = 9,467 B on a
/// 50-doc × 16-dim fixture, with `exists {field: emb_chunks}` still answering
/// 50. That costs bytes, not answers, and is pinned by
/// `a_companion_mapped_before_its_vector_keeps_its_dictionary_and_its_answers`
/// in `tests/integration.rs`. The nested-under-an-object spelling has the same
/// shape of limit and the same direction.
///
/// Pinned by `a_declared_sibling_keeps_its_lexical_surface` here and by
/// `declared_chunks_sibling_of_a_dense_vector_stays_queryable_unless_it_looks_like_one` in
/// `tests/integration.rs`.
///
/// ALLOCATION: this runs once per SEARCH (`index::search_inner` calls it to
/// decide whether the plan-time lowering has anything to do), and the vast
/// majority of indices have no vector field at all. `collect_dense_vector_fields`
/// clones the name of EVERY field it walks past, not just the vector ones, so
/// the no-vector case is short-circuited by the allocation-free predicate
/// below and returns an empty `HashSet`, which does not allocate either. The
/// two must agree by construction, and
/// `the_fast_path_predicate_agrees_with_the_walk` pins that they do.
pub fn lexically_typeless_fields(schema: &Schema) -> std::collections::HashSet<String> {
    let mut excluded = std::collections::HashSet::new();
    if !has_vector_field(&schema.fields) {
        return excluded;
    }
    for field in crate::index::collect_dense_vector_fields(schema) {
        // BY NAME, and only where no declaration contradicts it.
        for synthesised in [
            format!("{field}{MULTI_VECTOR_COMPANION_SUFFIX}"),
            format!("{}{}", xerj_query::executor::PASSAGE_METADATA_PREFIX, field),
        ] {
            if !declares_non_vector_shaped_field(schema, &synthesised) {
                excluded.insert(synthesised);
            }
        }
        // BY DECLARED TYPE: unconditional.
        excluded.insert(field);
    }
    excluded
}

/// Does `schema` declare `path` as a field whose type a MULTI-VECTOR could not
/// have produced — i.e. does the name-based companion rule have to yield to it?
///
/// `path` is dotted exactly as [`crate::index::collect_dense_vector_fields`]
/// emits it, so the walk descends `FieldConfig::fields` segment by segment
/// rather than using `Schema::has_field`, which only sees root names and would
/// answer `false` for every nested companion.
///
/// THE RULE IS ITS COMPLEMENT, NOT A LIST: this yields for EVERY declared type
/// except `dense_vector`. It is stated that way on purpose — an earlier
/// revision enumerated `Text`/`Keyword`/`Date`/`Boolean`/`Ip`/`Chunk`/`Object`
/// and read as exhaustive when it was not (`GeoPoint`, `Binary` and `Nested`
/// also yielded, and `es_type_to_field_type` maps every UNRECOGNISED ES type
/// string to `Object`, so unknown mapping types yielded too). So:
///
///  * `Vector` — already in the set by declared type; `false` is a no-op, and
///    the companion of a companion is not a thing.
///  * anything else — a declaration is a declaration. The user mapped a field
///    under this name, so the name rule yields and the field keeps its term
///    dictionary and its whole lexical surface: `text`, `keyword`, `long`,
///    `double`, `date`, `boolean`, `ip`, `geo_point`, `binary`, `object`,
///    `nested`, `chunk`, and anything `es_type_to_field_type` did not
///    recognise.
///  * absent — nothing to defer to; the name rule applies. This is the
///    companion's own case, and it is the ONLY one, because
///    `index::evolve_schema_from_doc`/`_docs` refuse to invent a `FieldConfig`
///    for an unmapped `<vector>_chunks` key whose value
///    [`looks_like_multi_vector`]. Without that refusal this predicate would be
///    answering about a declaration DYNAMIC MAPPING wrote: `infer_field_type`
///    walks an array of float arrays to its first scalar and lands on
///    `FieldType::Double`, indistinguishable from a user's own `double`. That
///    is why an earlier revision had to carve `Double`/`Long` out of the yield
///    set, and why a user's own numeric field under this name silently lost its
///    lexical surface. The carve-out is gone; the ambiguity is prevented at the
///    only place it was ever manufactured.
///
/// Lucene draws exactly this line: whether a field gets a term dictionary is
/// read off the field's own `IndexableFieldType`
/// (`IndexingChain.invertAndStore`,
/// `lucene/core/src/java/org/apache/lucene/index/IndexingChain.java:1416`),
/// never off its name, and one name meaning two different things is an outright
/// error rather than a silent downgrade (`FieldSchema.setIndexOptions` →
/// `assertSame` → `raiseNotSame`, `IndexingChain.java:2213`, `:2187`, `:2193`).
///
/// BOTH SPELLINGS of a nested name are resolved, because both reach the schema.
/// `put_mapping` stores a dotted path as ONE top-level `FieldConfig` named
/// `"passages.vec_chunks"`, while `es_properties_to_fields` stores it as a
/// `vec_chunks` child under a `passages` object — the same two shapes
/// `nested_dense_vector_is_excluded_from_its_parent_objects_term_dictionary`
/// exercises for the vector itself. The literal lookup runs first so the flat
/// spelling cannot fall through the segmented walk and be treated as undeclared.
fn declares_non_vector_shaped_field(schema: &Schema, path: &str) -> bool {
    let is_lexical =
        |fc: &xerj_common::types::FieldConfig| !matches!(fc.field_type, FieldType::Vector);
    if let Some(field) = schema.fields.iter().find(|fc| fc.name == path) {
        return is_lexical(field);
    }
    let mut fields: &[xerj_common::types::FieldConfig] = &schema.fields;
    let mut segments = path.split('.').peekable();
    while let Some(segment) = segments.next() {
        let Some(field) = fields.iter().find(|fc| fc.name == segment) else {
            return false;
        };
        if segments.peek().is_none() {
            return is_lexical(field);
        }
        fields = &field.fields;
    }
    false
}

/// Does this field list contain a `dense_vector` anywhere? Allocation-free.
///
/// The recursion condition is `collect_dense_vector_fields`'s, deliberately: it
/// stops at a `Vector` rather than descending into its sub-fields, so the two
/// walks visit exactly the same nodes and cannot disagree about whether the set
/// would come back empty.
fn has_vector_field(fields: &[xerj_common::types::FieldConfig]) -> bool {
    fields.iter().any(|fc| {
        if matches!(fc.field_type, FieldType::Vector) {
            true
        } else {
            !fc.fields.is_empty() && has_vector_field(&fc.fields)
        }
    })
}

#[cfg(test)]
mod semantic_derived_vector_exclusion_tests {
    use super::*;
    use serde_json::json;
    use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType};

    #[test]
    fn derives_default_and_custom_targets_without_suffix_guessing() {
        let mut schema = Schema::empty();
        schema
            .add_field(
                FieldConfig::new("body", FieldType::Text).with_embedding(EmbeddingConfig {
                    endpoint: None,
                    model: None,
                    target_field: None,
                }),
            )
            .unwrap();
        schema
            .add_field(FieldConfig::new("summary", FieldType::Text).with_embedding(
                EmbeddingConfig {
                    endpoint: None,
                    model: None,
                    target_field: Some("summary_model_output".to_string()),
                },
            ))
            .unwrap();
        schema
            .add_field(FieldConfig::new("user_vector", FieldType::Text))
            .unwrap();

        let excluded = semantic_derived_vector_fields(&schema);
        assert_eq!(
            excluded,
            std::collections::HashSet::from([
                "body_vector".to_string(),
                "body_vector_chunks".to_string(),
                "__xerj_passage_meta__body_vector".to_string(),
                "summary_model_output".to_string(),
                "summary_model_output_chunks".to_string(),
                "__xerj_passage_meta__summary_model_output".to_string(),
            ])
        );

        let source = json!({
            "body": "cash and equivalents",
            "page": 7,
            "user_vector": [1.0, 2.0],
            "body_vector_backup": [3.0, 4.0],
            "body_vector": [0.1, 0.2],
            "body_vector_chunks": [[0.1, 0.2], [0.3, 0.4]],
            "__xerj_passage_meta__body_vector": {"field": "body", "chunks": [[0, 20]]},
            "summary_model_output": [0.5, 0.6],
            "summary_model_output_chunks": [[0.5, 0.6], [0.7, 0.8]],
            "__xerj_passage_meta__summary_model_output": {
                "field": "summary",
                "chunks": [[0, 7]]
            }
        });
        let fields = extract_text_fields_from_excluding(&source, &excluded);
        assert_eq!(fields.get("page").map(String::as_str), Some("7"));
        assert_eq!(
            fields.get("user_vector").map(String::as_str),
            Some("1.0 2.0")
        );
        assert_eq!(
            fields.get("body_vector_backup").map(String::as_str),
            Some("3.0 4.0")
        );
        for derived in &excluded {
            assert!(!fields.contains_key(derived));
        }
    }
}

#[cfg(test)]
mod index_false_exclusion_tests {
    //! `"index": false` must remove the field from the full-text index —
    //! the storage half of #204's last accepted-and-ignored instance. Before
    //! the fix `FieldOptions::indexed` was never read here, so a field the
    //! mapping declared non-indexed got a full set of postings anyway.
    use super::*;
    use serde_json::json;
    use xerj_common::types::{FieldConfig, FieldType};

    fn schema_with_a_non_indexed_field() -> Schema {
        let mut schema = Schema::empty();
        let mut note = FieldConfig::new("note", FieldType::Text);
        note.options.indexed = false;
        note.options.doc_values = false;
        schema.add_field(note).unwrap();
        let mut code = FieldConfig::new("code", FieldType::Keyword);
        code.options.indexed = false; // keeps doc values — still queryable
        schema.add_field(code).unwrap();
        schema
            .add_field(FieldConfig::new("body", FieldType::Text))
            .unwrap();
        schema
    }

    #[test]
    fn non_indexed_fields_join_the_fts_exclusion_set() {
        let schema = schema_with_a_non_indexed_field();
        let excluded = fts_excluded_fields(&schema);
        assert!(
            excluded.contains("note"),
            "a non-indexed Text field: the stored-doc scan analyses it the \
             same way the postings did, so dropping them is equivalent: \
             {excluded:?}"
        );
        assert!(
            !excluded.contains("code"),
            "a non-indexed KEYWORD that kept its doc values is still \
             searchable in ES's sense, and the scan fallback is weaker than \
             its whole-value postings (it would split `192.168.0.1` into \
             tokens) — the postings stay: {excluded:?}"
        );
        assert!(!excluded.contains("body"), "{excluded:?}");

        // …but with no doc values either, nothing can answer it, the query is
        // rejected outright, and the postings are pure waste whatever the type.
        let mut schema = Schema::empty();
        let mut opaque = FieldConfig::new("opaque", FieldType::Keyword);
        opaque.options.indexed = false;
        opaque.options.doc_values = false;
        schema.add_field(opaque).unwrap();
        assert!(fts_excluded_fields(&schema).contains("opaque"));
    }

    #[test]
    fn a_non_indexed_schema_text_field_is_never_analysed() {
        // The schema-Text loop in `analyze_doc_excluding` used to ignore the
        // exclusion set entirely (its only members were semantic-derived
        // vector fields, which are never Text). A declared `index: false`
        // Text field is exactly the case that made the omission visible.
        let schema = schema_with_a_non_indexed_field();
        let source = json!({ "note": "zzquagga", "code": "AB-1234", "body": "ordinary text" });
        let analyzer = xerj_fts::analyzer::AnalyzerPipeline::new(
            vec![],
            std::sync::Arc::new(xerj_fts::analyzer::StandardTokenizer),
            vec![std::sync::Arc::new(xerj_fts::analyzer::LowercaseFilter)
                as std::sync::Arc<dyn xerj_fts::analyzer::TokenFilter>],
        );
        let analysed: std::collections::HashMap<String, Vec<Token>> =
            analyze_doc(&source, &schema, &analyzer)
                .into_iter()
                .collect();
        let names: Vec<&str> = analysed.keys().map(String::as_str).collect();
        assert!(
            !analysed.contains_key("note"),
            "a non-indexed Text field must contribute no tokens: {names:?}"
        );
        assert!(
            analysed.contains_key("code"),
            "`code` kept its doc values, so it kept its whole-value postings \
             too — see `fts_excluded_fields`: {names:?}"
        );
        assert!(
            analysed.contains_key("body"),
            "the indexed control field must still be analysed: {names:?}"
        );
    }
}

/// Extract a string value from a JSON value for text indexing.
fn extract_text_value(val: &Value) -> String {
    match val {
        Value::String(s) => s.clone(),
        Value::Number(n) => n.to_string(),
        Value::Bool(b) => b.to_string(),
        Value::Array(arr) => arr
            .iter()
            .map(extract_text_value)
            .collect::<Vec<_>>()
            .join(" "),
        Value::Object(_) => serde_json::to_string(val).unwrap_or_default(),
        Value::Null => String::new(),
    }
}

/// Does `excluded` hold anything strictly BELOW `path`?
///
/// Allocation-free on the hot path: the set is schema-derived and normally
/// holds zero dotted entries, so this is a handful of length compares per
/// object-valued field per document.
pub(crate) fn has_excluded_descendant(
    path: &str,
    excluded: &std::collections::HashSet<String>,
) -> bool {
    excluded.iter().any(|entry| {
        entry.len() > path.len() && entry.as_bytes()[path.len()] == b'.' && entry.starts_with(path)
    })
}

/// [`extract_text_value`], with every descendant named in `excluded` removed
/// from the flattened object first — #328.
///
/// `path` is the dotted path of `val` itself. Only objects can have
/// descendants, so every other shape short-circuits to `extract_text_value`,
/// and an object with no excluded descendant does too — the clone is paid only
/// by the documents that actually carry an excluded sub-field.
pub(crate) fn extract_text_value_excluding(
    val: &Value,
    path: &str,
    excluded: &std::collections::HashSet<String>,
) -> String {
    fn prune(val: &Value, path: &str, excluded: &std::collections::HashSet<String>) -> Value {
        match val {
            Value::Object(map) => {
                let mut kept = serde_json::Map::with_capacity(map.len());
                for (key, child) in map {
                    let child_path = if path.is_empty() {
                        key.clone()
                    } else {
                        format!("{path}.{key}")
                    };
                    if excluded.contains(&child_path) {
                        continue;
                    }
                    kept.insert(key.clone(), prune(child, &child_path, excluded));
                }
                Value::Object(kept)
            }
            other => other.clone(),
        }
    }
    if !matches!(val, Value::Object(_)) || !has_excluded_descendant(path, excluded) {
        return extract_text_value(val);
    }
    extract_text_value(&prune(val, path, excluded))
}

/// Unit coverage for the three helpers #328 adds — the widest new surface in
/// this file is [`lexically_typeless_fields`], which is `pub`.
#[cfg(test)]
mod lexically_typeless_fields_tests {
    use super::*;
    use serde_json::json;
    use xerj_common::types::{FieldConfig, FieldType, Schema};

    fn vector(name: &str, dims: usize) -> FieldConfig {
        let mut f = FieldConfig::new(name, FieldType::Vector);
        f.options.dimensions = Some(dims);
        f.options.similarity = Some("cosine".to_string());
        f
    }

    /// Every `dense_vector` contributes THREE names, because a vector never
    /// arrives alone: the base field, the `_chunks` multi-vector companion and
    /// the passage-metadata sidecar. Excluding only the base name is what left
    /// `<seg>.emb_chunks.fst` — the bigger of the two artifacts — behind.
    #[test]
    fn every_vector_contributes_its_whole_family() {
        let mut schema = Schema::empty();
        schema
            .add_field(FieldConfig::new("body", FieldType::Text))
            .unwrap();
        schema.add_field(vector("emb", 8)).unwrap();

        let typeless = lexically_typeless_fields(&schema);
        assert_eq!(
            typeless,
            std::collections::HashSet::from([
                "emb".to_string(),
                "emb_chunks".to_string(),
                "__xerj_passage_meta__emb".to_string(),
            ])
        );
        assert!(!typeless.contains("body"), "a text field is not typeless");
    }

    /// The name-based half YIELDS to a declaration. A user who maps a field
    /// called `emb_chunks` beside a `dense_vector emb` gets a real keyword
    /// field, not a silently unqueryable one — the previous revision of this
    /// function excluded the name unconditionally, which took the postings away
    /// from a field the schema declares and (through the plan-time lowering)
    /// made `bool{must_not:[term emb_chunks]}` RETURN the documents it excludes.
    ///
    /// Lucene reads "does this field get a term dictionary" off the field's own
    /// `IndexableFieldType` and never off its name
    /// (`IndexingChain.java:1416`), and treats one name carrying two different
    /// index options as an error rather than a silent downgrade (`:2213`).
    #[test]
    fn a_declared_sibling_keeps_its_lexical_surface() {
        for declared in [
            FieldType::Keyword,
            FieldType::Text,
            FieldType::Date,
            FieldType::Boolean,
            FieldType::Ip,
        ] {
            let mut schema = Schema::empty();
            schema
                .add_field(FieldConfig::new("body", FieldType::Text))
                .unwrap();
            schema
                .add_field(FieldConfig::new("emb_chunks", declared))
                .unwrap();
            schema
                .add_field(FieldConfig::new("__xerj_passage_meta__emb", declared))
                .unwrap();
            schema.add_field(vector("emb", 8)).unwrap();

            let typeless = lexically_typeless_fields(&schema);
            assert_eq!(
                typeless,
                std::collections::HashSet::from(["emb".to_string()]),
                "a declared `{declared:?}` sibling must keep its term dictionary"
            );
            // And the union the write path actually reads agrees.
            let excluded = fts_excluded_fields(&schema);
            assert!(excluded.contains("emb"));
            assert!(
                !excluded.contains("emb_chunks"),
                "declared sibling excluded from the inverted index: {excluded:?}"
            );
        }
    }

    /// THE NUMERIC ARM, which used to be the accepted residual: a `Double` or
    /// `Long` declaration is exactly what dynamic mapping USED to produce from
    /// a real multi-vector, so the name rule could not yield to it without
    /// losing the whole companion saving. It yields now, because
    /// `index::is_undeclared_multi_vector_companion` stops that config from
    /// being invented — so a `long`/`double` under this name is necessarily
    /// somebody's own field. This assertion is the inverse of the one it
    /// replaces; if the carve-out ever comes back it fails here first.
    #[test]
    fn a_numerically_declared_sibling_keeps_its_lexical_surface() {
        for declared in [FieldType::Double, FieldType::Long] {
            let mut schema = Schema::empty();
            schema
                .add_field(FieldConfig::new("emb_chunks", declared))
                .unwrap();
            schema.add_field(vector("emb", 8)).unwrap();
            assert!(
                !lexically_typeless_fields(&schema).contains("emb_chunks"),
                "a `{declared:?}` declaration is somebody's own field and the \
                 name rule must yield to it"
            );
            assert!(declares_non_vector_shaped_field(&schema, "emb_chunks"));
        }
    }

    /// The yield set is the COMPLEMENT of `Vector`, not the enumerated list an
    /// earlier doc comment gave. `GeoPoint`, `Binary` and `Nested` were never
    /// in that list and always yielded, and `es_type_to_field_type` maps every
    /// unrecognised ES type string to `Object`, so an unknown mapping type
    /// yields too. Pinned across every variant so the doc and the code cannot
    /// drift apart again.
    #[test]
    fn every_declared_type_except_dense_vector_keeps_its_lexical_surface() {
        for declared in [
            FieldType::Text,
            FieldType::Keyword,
            FieldType::Long,
            FieldType::Double,
            FieldType::Boolean,
            FieldType::Date,
            FieldType::Ip,
            FieldType::GeoPoint,
            FieldType::Binary,
            FieldType::Object,
            FieldType::Nested,
        ] {
            let mut schema = Schema::empty();
            schema
                .add_field(FieldConfig::new("emb_chunks", declared))
                .unwrap();
            schema.add_field(vector("emb", 8)).unwrap();
            assert!(
                declares_non_vector_shaped_field(&schema, "emb_chunks"),
                "`{declared:?}` is a declaration and must be yielded to"
            );
            let typeless = lexically_typeless_fields(&schema);
            assert!(
                !typeless.contains("emb_chunks"),
                "`{declared:?}` sibling excluded anyway: {typeless:?}"
            );
            // …while the vector itself is excluded on every one of them.
            assert!(typeless.contains("emb"));
        }
    }

    /// The value-shape half of the rule, exhaustively. Only a rectangular,
    /// non-empty array of non-empty all-numeric arrays is a multi-vector;
    /// everything else is somebody's field and keeps its mapping.
    #[test]
    fn looks_like_multi_vector_accepts_only_the_rfc_148_shape() {
        for yes in [
            serde_json::json!([[1.0, 2.0], [3.0, 4.0]]),
            serde_json::json!([[1, 2, 3]]),
            serde_json::json!([[0.5]]),
        ] {
            assert!(looks_like_multi_vector(&yes), "should accept {yes}");
        }
        for no in [
            // scalars and strings — a user's own field
            serde_json::json!(7),
            serde_json::json!(7.5),
            serde_json::json!("tenant-a"),
            serde_json::json!(true),
            serde_json::json!(null),
            serde_json::json!({"a": 1}),
            // a FLAT numeric array is a single vector, not a multi-vector, and
            // is not what this rule is about
            serde_json::json!([1.0, 2.0, 3.0]),
            // empty outer, empty inner
            serde_json::json!([]),
            serde_json::json!([[]]),
            // RAGGED — cannot have come from one embedding model, so it is left
            // to dynamic mapping (the fail-safe direction)
            serde_json::json!([[1.0, 2.0], [3.0]]),
            // non-numeric elements
            serde_json::json!([["a", "b"]]),
            serde_json::json!([[1.0, "b"]]),
            serde_json::json!([[1.0], null]),
            serde_json::json!([[1.0], 2.0]),
            // array of objects
            serde_json::json!([{"a": 1}]),
        ] {
            assert!(!looks_like_multi_vector(&no), "should reject {no}");
        }
    }

    /// A sibling declared as a `dense_vector` is excluded — by the TYPE rule,
    /// which is unconditional — so yielding to the declaration must not
    /// accidentally rescue it.
    #[test]
    fn a_sibling_declared_as_a_vector_is_still_typeless() {
        let mut schema = Schema::empty();
        schema.add_field(vector("emb_chunks", 8)).unwrap();
        schema.add_field(vector("emb", 8)).unwrap();

        let typeless = lexically_typeless_fields(&schema);
        for name in [
            "emb",
            "emb_chunks",
            "emb_chunks_chunks",
            "__xerj_passage_meta__emb",
        ] {
            assert!(typeless.contains(name), "{name} missing from {typeless:?}");
        }
    }

    /// The declaration check walks DOTTED paths, because a nested vector's
    /// companion is nested too. `Schema::has_field` only sees root names and
    /// would answer "not declared" for every one of these.
    #[test]
    fn a_declared_nested_sibling_keeps_its_lexical_surface() {
        let mut schema = Schema::empty();
        let mut parent = FieldConfig::new("passages", FieldType::Object);
        parent.fields.push(vector("vec", 4));
        parent
            .fields
            .push(FieldConfig::new("vec_chunks", FieldType::Keyword));
        schema.add_field(parent).unwrap();

        let typeless = lexically_typeless_fields(&schema);
        assert!(typeless.contains("passages.vec"), "got {typeless:?}");
        assert!(
            !typeless.contains("passages.vec_chunks"),
            "declared nested sibling excluded: {typeless:?}"
        );
        assert!(declares_non_vector_shaped_field(
            &schema,
            "passages.vec_chunks"
        ));
        assert!(
            !declares_non_vector_shaped_field(&schema, "passages.vec"),
            "a declared vector is not a lexical declaration"
        );
        assert!(
            !declares_non_vector_shaped_field(&schema, "passages.absent"),
            "an undeclared name has no declaration to defer to"
        );
        assert!(
            !declares_non_vector_shaped_field(&schema, "passages.vec_chunks.deeper"),
            "walking past a leaf must not resolve"
        );
    }

    /// The FLAT spelling of a nested name — one top-level `FieldConfig` called
    /// `"passages.vec_chunks"`, which is what `put_mapping` produces — must
    /// resolve too. Without the literal lookup the segmented walk looks for a
    /// root `passages` object, does not find one, and treats a declared field
    /// as undeclared.
    #[test]
    fn the_flat_dotted_spelling_of_a_declared_sibling_resolves() {
        let mut schema = Schema::empty();
        schema.add_field(vector("passages.vec", 4)).unwrap();
        schema
            .add_field(FieldConfig::new("passages.vec_chunks", FieldType::Keyword))
            .unwrap();

        assert!(declares_non_vector_shaped_field(
            &schema,
            "passages.vec_chunks"
        ));
        let typeless = lexically_typeless_fields(&schema);
        assert!(typeless.contains("passages.vec"), "got {typeless:?}");
        assert!(
            !typeless.contains("passages.vec_chunks"),
            "the flat spelling was treated as undeclared: {typeless:?}"
        );
    }

    /// The exclusion is by DECLARED type, not by value shape: a numeric array
    /// under a non-vector mapping stays lexical exactly as before.
    #[test]
    fn a_non_vector_field_is_never_typeless() {
        let mut schema = Schema::empty();
        schema
            .add_field(FieldConfig::new("body", FieldType::Text))
            .unwrap();
        schema
            .add_field(FieldConfig::new("cat", FieldType::Keyword))
            .unwrap();
        schema
            .add_field(FieldConfig::new("n", FieldType::Long))
            .unwrap();
        assert!(lexically_typeless_fields(&schema).is_empty());
    }

    /// A vector nested under an object mapping arrives as a DOTTED path — the
    /// first member of this set that is not a root-level name, and the reason
    /// `collect_text_fields` had to start matching the full path.
    #[test]
    fn a_nested_vector_is_collected_by_its_dotted_path() {
        let mut schema = Schema::empty();
        let mut parent = FieldConfig::new("passages", FieldType::Object);
        parent.fields.push(vector("vec", 4));
        schema.add_field(parent).unwrap();

        let typeless = lexically_typeless_fields(&schema);
        assert!(typeless.contains("passages.vec"), "got {typeless:?}");
        assert!(typeless.contains("passages.vec_chunks"));
        assert!(
            !typeless.contains("vec"),
            "the leaf name alone must not match"
        );
    }

    /// `fts_excluded_fields` is the union, so the vector family must appear
    /// there too — that union is what flush, merge and `build_fts_field_configs`
    /// all read.
    #[test]
    fn fts_excluded_fields_unions_the_vector_family_in() {
        let mut schema = Schema::empty();
        schema
            .add_field(FieldConfig::new("body", FieldType::Text))
            .unwrap();
        schema.add_field(vector("emb", 8)).unwrap();

        let excluded = fts_excluded_fields(&schema);
        for name in ["emb", "emb_chunks", "__xerj_passage_meta__emb"] {
            assert!(excluded.contains(name), "{name} missing from {excluded:?}");
        }
        assert!(!excluded.contains("body"));
    }

    /// The allocation-free short-circuit and the allocating walk must agree on
    /// "is this set empty", for every shape either of them can see — otherwise
    /// the fast path would silently disable the whole #328 query half on some
    /// schema.
    #[test]
    fn the_fast_path_predicate_agrees_with_the_walk() {
        let mut nested = FieldConfig::new("passages", FieldType::Object);
        nested.fields.push(vector("vec", 4));
        let mut deep = FieldConfig::new("a", FieldType::Object);
        let mut mid = FieldConfig::new("b", FieldType::Object);
        mid.fields.push(vector("c", 2));
        deep.fields.push(mid);
        let mut object_without_a_vector = FieldConfig::new("meta", FieldType::Object);
        object_without_a_vector
            .fields
            .push(FieldConfig::new("note", FieldType::Text));

        for (label, fields) in [
            ("empty", vec![]),
            ("text only", vec![FieldConfig::new("body", FieldType::Text)]),
            ("object with no vector", vec![object_without_a_vector]),
            ("root vector", vec![vector("emb", 8)]),
            ("nested vector", vec![nested]),
            ("two levels deep", vec![deep]),
            (
                "vector beside text",
                vec![FieldConfig::new("body", FieldType::Text), vector("emb", 8)],
            ),
        ] {
            let mut schema = Schema::empty();
            for f in fields {
                schema.add_field(f).unwrap();
            }
            assert_eq!(
                has_vector_field(&schema.fields),
                !crate::index::collect_dense_vector_fields(&schema).is_empty(),
                "predicate and walk disagree on `{label}`"
            );
            assert_eq!(
                has_vector_field(&schema.fields),
                !lexically_typeless_fields(&schema).is_empty(),
                "short-circuit changed the answer on `{label}`"
            );
        }
    }

    #[test]
    fn has_excluded_descendant_matches_only_strictly_below_the_path() {
        let set = std::collections::HashSet::from(["passages.vec".to_string(), "emb".to_string()]);
        assert!(has_excluded_descendant("passages", &set));
        // The path ITSELF is not a descendant of itself — the caller checks
        // `excluded.contains(&path)` separately, and double-counting it here
        // would make the prune fire for a leaf that was already skipped.
        assert!(!has_excluded_descendant("passages.vec", &set));
        assert!(!has_excluded_descendant("emb", &set));
        // A shared PREFIX is not a path prefix: `passages_extra` must not match
        // `passages.vec`, which is why the check tests for the `.` separator
        // rather than calling `starts_with` alone.
        assert!(!has_excluded_descendant("passages_extra", &set));
        assert!(!has_excluded_descendant("other", &set));
    }

    /// The flattened root-level blob is where a nested vector actually lands,
    /// so the prune has to happen there and not only in the recursion.
    #[test]
    fn extract_text_value_excluding_drops_the_nested_vector_from_the_blob() {
        let set = std::collections::HashSet::from(["passages.vec".to_string()]);
        let val = json!({
            "text": "quarterly liquidity",
            "vec": [0.125, 0.25, 0.5],
            "nested": {"keep": "yes"}
        });
        let pruned = extract_text_value_excluding(&val, "passages", &set);
        assert!(
            pruned.contains("quarterly liquidity"),
            "text kept: {pruned}"
        );
        assert!(
            pruned.contains("yes"),
            "unrelated sub-object kept: {pruned}"
        );
        assert!(
            !pruned.contains("0.125") && !pruned.contains("0.25"),
            "vector components must be gone: {pruned}"
        );
    }

    /// No excluded descendant → identical to `extract_text_value`, and no clone
    /// is paid. This is the path every document without a nested vector takes.
    #[test]
    fn extract_text_value_excluding_is_a_no_op_without_a_descendant() {
        let set = std::collections::HashSet::from(["other.vec".to_string()]);
        let val = json!({"text": "quarterly liquidity", "n": 7});
        assert_eq!(
            extract_text_value_excluding(&val, "passages", &set),
            extract_text_value(&val)
        );
        // A non-object short-circuits regardless of what the set holds.
        let scalar = json!("plain string");
        assert_eq!(
            extract_text_value_excluding(&scalar, "passages", &set),
            extract_text_value(&scalar)
        );
    }
}

#[cfg(test)]
mod bounded_delta_counts_tests {
    use super::*;
    use serde_json::json;

    /// Brute-force full recount of a DocValues store's columns, matching
    /// the OLD `ensure_counts_built` semantics exactly.
    fn recount(
        dv: &DocValues,
    ) -> (
        FxHashMap<String, FxHashMap<String, u32>>,
        FxHashMap<String, FxHashMap<u64, u32>>,
        FxHashMap<String, FxHashSet<String>>,
    ) {
        let mut kc: FxHashMap<String, FxHashMap<String, u32>> = FxHashMap::default();
        let mut nc: FxHashMap<String, FxHashMap<u64, u32>> = FxHashMap::default();
        let mut ks: FxHashMap<String, FxHashSet<String>> = FxHashMap::default();
        for (field, col) in &dv.keyword {
            let counts = kc.entry(field.clone()).or_default();
            let set = ks.entry(field.clone()).or_default();
            for s in col.iter().flatten() {
                *counts.entry(s.clone()).or_insert(0) += 1;
                set.insert(s.clone());
            }
        }
        for (field, col) in &dv.numeric {
            let counts = nc.entry(field.clone()).or_default();
            for f in col.iter().flatten() {
                *counts.entry(f.to_bits()).or_insert(0) += 1;
            }
        }
        (kc, nc, ks)
    }

    /// Fold every field per-field (as the query path does) and assert the
    /// maintained maps equal a full brute-force recount.
    fn assert_maintained_eq_recount(dv: &DocValues) {
        let (kc, nc, ks) = recount(dv);
        // Fold each keyword + numeric field via the per-field accessors,
        // exactly as the query path does, then compare the whole state.
        for field in dv.keyword.keys() {
            dv.with_keyword_field(field, |_| {});
        }
        for field in dv.numeric.keys() {
            dv.with_numeric_field(field, |_| {});
        }
        let cs = dv.counts.lock();
        assert_eq!(cs.keyword_counts, kc, "keyword_counts drift");
        assert_eq!(cs.numeric_counts, nc, "numeric_counts drift");
        assert_eq!(cs.keyword_set, ks, "keyword_set drift");
    }

    #[test]
    fn bounded_delta_matches_full_recount() {
        let mut dv = DocValues::default();
        // Push a first batch spanning keyword + numeric + mixed-number fields.
        let models = ["haiku", "sonnet", "opus", "haiku", "opus"];
        for i in 0..2500usize {
            let doc = json!({
                "model": models[i % models.len()],
                "status": if i % 3 == 0 { "ok" } else { "err" },
                "latency_ms": (i % 400) as i64,
                "cost_usd": (i as f64) * 0.001,
            });
            dv.push(&doc, i);
        }
        // First fold (folds the whole batch) must equal a full recount.
        assert_maintained_eq_recount(&dv);

        // Incremental: append MORE docs past the watermark, fold again — must
        // still equal a full recount (bounded-delta folds only the new tail).
        for i in 2500..6000usize {
            let doc = json!({
                "model": models[i % models.len()],
                "status": if i % 5 == 0 { "ok" } else { "err" },
                "latency_ms": (i % 400) as i64,
                "cost_usd": (i as f64) * 0.001,
            });
            dv.push(&doc, i);
        }
        assert_maintained_eq_recount(&dv);

        // Per-field isolation: folding ONE field must not populate a sibling's
        // maps (a `terms` on `model` must not build `cost_usd`'s high-card map).
        {
            let mut fresh = DocValues::default();
            for i in 0..1000usize {
                fresh.push(&json!({ "model": "x", "cost_usd": i as f64 }), i);
            }
            fresh.with_keyword_field("model", |c| {
                assert!(c.keyword_counts.contains_key("model"));
                assert!(!c.keyword_counts.contains_key("cost_usd"), "sibling folded");
                assert!(!c.numeric_counts.contains_key("cost_usd"), "sibling folded");
            });
        }

        // Delete resets the watermarks; the next fold re-derives from scratch
        // and must again equal a full recount over the (shifted) columns.
        dv.remove_at(1234);
        dv.remove_at(0);
        assert_maintained_eq_recount(&dv);
    }
}

#[cfg(test)]
mod filtered_docs_arc_tests {
    use super::*;
    use crate::aggs::doc_matches_filter;
    use serde_json::json;

    /// Build a sharded memtable of `n_docs` docs with scalar fields:
    ///   n (long = i), status (keyword, 5-card = i%5), lat (double).
    fn build_mem(n_docs: usize) -> ShardedFtsMemtable {
        let mem = ShardedFtsMemtable::new();
        let schema = Schema::default();
        for i in 0..n_docs {
            let doc = json!({
                "n": i as i64,
                "status": format!("s{}", i % 5),
                "lat": (i % 1000) as f64 * 0.01,
            });
            mem.insert(format!("d{i}"), &doc, &schema, i as u64);
        }
        mem
    }

    /// Brute reference: every buffered doc whose source matches `filter_json`
    /// under the exact JSON matcher, as a sorted `doc_id` vec.
    fn brute_ids(mem: &ShardedFtsMemtable, filter_json: &Value) -> Vec<String> {
        let mut ids: Vec<String> = mem
            .all_docs_with_sources_arc()
            .into_iter()
            .filter(|(_id, src)| doc_matches_filter(src, filter_json))
            .map(|(id, _)| id)
            .collect();
        ids.sort();
        ids
    }

    fn columnar_ids(mem: &ShardedFtsMemtable, preds: &[MemBoolPred]) -> Vec<String> {
        let mut ids: Vec<String> = mem
            .filtered_docs_arc(preds)
            .expect("columnar path should apply for scalar fields")
            .into_iter()
            .map(|(id, _)| id)
            .collect();
        ids.sort();
        ids
    }

    /// The columnar candidate set must EQUAL the `doc_matches_filter` set —
    /// including on a filter that matches FAR more than the historical
    /// `materialisation_limit` (256) so a bounded/truncated fold would be
    /// caught (the all-matches aggregation contract).
    #[test]
    fn columnar_range_matches_brute_over_256() {
        let mem = build_mem(4000);
        // Range n ∈ [500, 3500) → 3000 matches, an order of magnitude past 256.
        let preds = vec![MemBoolPred::Range {
            field: "n".to_string(),
            gte: Some(500.0),
            gt: None,
            lte: None,
            lt: Some(3500.0),
        }];
        let filter = json!({ "range": { "n": { "gte": 500, "lt": 3500 } } });
        let brute = brute_ids(&mem, &filter);
        let columnar = columnar_ids(&mem, &preds);
        assert_eq!(brute.len(), 3000, "sanity: 3000 docs in [500,3500)");
        assert_eq!(columnar, brute, "columnar range set diverged from brute");
        // Value fold parity: sum over the columnar candidate set (metric-agg
        // fold input) equals the brute sum over the matcher set.
        let sum_of = |ids: &[String]| -> f64 {
            ids.iter()
                .map(|id| id.trim_start_matches('d').parse::<f64>().unwrap())
                .sum()
        };
        assert_eq!(sum_of(&columnar), sum_of(&brute));
    }

    /// Term + Range conjunction (a `bool{filter}`), also > 256 matches.
    #[test]
    fn columnar_bool_conjunction_matches_brute() {
        let mem = build_mem(4000);
        // status == "s2" (800 docs) AND n ∈ [0,4000) → the 800 status-s2 docs.
        let preds = vec![
            MemBoolPred::Term {
                field: "status".to_string(),
                value: "s2".to_string(),
                num: None,
            },
            MemBoolPred::Range {
                field: "n".to_string(),
                gte: Some(0.0),
                gt: None,
                lte: None,
                lt: Some(4000.0),
            },
        ];
        let filter = json!({
            "bool": { "filter": [
                { "term": { "status": "s2" } },
                { "range": { "n": { "gte": 0, "lt": 4000 } } },
            ]}
        });
        let brute = brute_ids(&mem, &filter);
        assert_eq!(brute.len(), 800);
        assert_eq!(columnar_ids(&mem, &preds), brute);
    }

    /// Bare keyword term.
    #[test]
    fn columnar_term_matches_brute() {
        let mem = build_mem(2000);
        let preds = vec![MemBoolPred::Term {
            field: "status".to_string(),
            value: "s3".to_string(),
            num: None,
        }];
        let filter = json!({ "term": { "status": "s3" } });
        let brute = brute_ids(&mem, &filter);
        assert_eq!(brute.len(), 400);
        assert_eq!(columnar_ids(&mem, &preds), brute);
    }

    /// `terms` disjunction, alone and conjoined with a `term` — the catalog
    /// finalize-verify shape (`terms file_key` AND `term run_id`). Must equal
    /// the JSON matcher and must not bail to the full-source walk.
    #[test]
    fn columnar_terms_conjunction_matches_brute() {
        let mem = build_mem(4000);
        let terms = vec![MemBoolPred::Terms {
            field: "status".to_string(),
            values: vec!["s1".to_string(), "s2".to_string()],
            nums: Vec::new(),
        }];
        let terms_filter = json!({ "terms": { "status": ["s1", "s2"] } });
        let terms_brute = brute_ids(&mem, &terms_filter);
        assert_eq!(terms_brute.len(), 1600);
        assert_eq!(columnar_ids(&mem, &terms), terms_brute);

        // status ∈ {s1, s2} AND status = s1 → the 800 s1 docs.
        let conj = vec![
            MemBoolPred::Terms {
                field: "status".to_string(),
                values: vec!["s1".to_string(), "s2".to_string()],
                nums: Vec::new(),
            },
            MemBoolPred::Term {
                field: "status".to_string(),
                value: "s1".to_string(),
                num: None,
            },
        ];
        let conj_filter = json!({
            "bool": { "filter": [
                { "terms": { "status": ["s1", "s2"] } },
                { "term": { "status": "s1" } },
            ]}
        });
        let conj_brute = brute_ids(&mem, &conj_filter);
        assert_eq!(conj_brute.len(), 800);
        assert_eq!(columnar_ids(&mem, &conj), conj_brute);
    }

    /// A predicate field that ever carried an ARRAY value makes the column
    /// lossy (first element only), so the columnar path MUST bail (`None`) so
    /// the caller keeps the exact full-corpus walk — otherwise a later matching
    /// array element would be a silent false negative.
    #[test]
    fn columnar_bails_on_array_predicate_field() {
        let mem = ShardedFtsMemtable::new();
        let schema = Schema::default();
        for i in 0..2000usize {
            // Every 500th doc stores `n` as an array — poisons the shard's
            // `array_fields` for whichever shard owns it.
            let n_val = if i % 500 == 0 {
                json!([i as i64, (i as i64) + 1])
            } else {
                json!(i as i64)
            };
            mem.insert(format!("d{i}"), &json!({ "n": n_val }), &schema, i as u64);
        }
        let preds = vec![MemBoolPred::Range {
            field: "n".to_string(),
            gte: Some(0.0),
            gt: None,
            lte: None,
            lt: Some(2000.0),
        }];
        assert!(
            mem.filtered_docs_arc(&preds).is_none(),
            "array-valued predicate field must force the full-corpus fallback"
        );
    }
}

#[cfg(test)]
mod external_scalar_n_ghost_tests {
    use super::*;
    use serde_json::json;

    /// #193 item 2 — the external-mode (`Some(cs)`) scalar `N` must be
    /// ghost-inclusive, exactly like the historical `None` branch.
    ///
    /// The scalar only engages as the per-field fallback: a scored field the
    /// supplied union does not carry.  This test pins the seam directly with
    /// ONE shard, so the local per-shard fallbacks for `df` and `avgdl`
    /// coincide with the `None` branch's cross-shard aggregation by
    /// construction, and the ghost carries NO occurrence of the queried
    /// field — leaving the scalar `N` as the ONLY quantity the two modes
    /// could disagree on.  Before the fix the external branch used the
    /// live-only `doc_count()` (2) where the `None` branch used
    /// live + ghosts (3), so the same query on the same memtable scored
    /// differently depending on which mode the engine happened to take.
    #[test]
    fn external_mode_scalar_n_matches_none_mode_under_ghosts() {
        let registry = Arc::new(AnalyzerRegistry::default());
        let mem = ShardedFtsMemtable::with_registry_and_shards(registry, 1);
        let schema = Schema::empty();

        // Two live docs carrying the queried term in `body`, one doomed doc
        // that never mentions `body` at all.
        mem.insert("a".into(), &json!({"body": "quicklist alpha"}), &schema, 1);
        mem.insert(
            "b".into(),
            &json!({"body": "quicklist beta gamma delta"}),
            &schema,
            2,
        );
        mem.insert(
            "ghost".into(),
            &json!({"other": "unrelated text"}),
            &schema,
            3,
        );
        mem.remove("ghost");

        // Union that does NOT carry `body` — the scored field falls back to
        // the scalar N (plus local df/avgdl, identical across modes here).
        let mut cs = xerj_fts::CollectionStats::new();
        cs.add_field(
            "other",
            &xerj_fts::FieldStats {
                total_docs: 1,
                total_field_length: 2,
            },
        );

        let boosts = std::collections::HashMap::new();
        let (none_hits, none_total) =
            mem.search_text_boosted_with_total("quicklist", &["body"], 10, &boosts);
        let (ext_hits, ext_total) = mem.search_text_boosted_with_total_using(
            "quicklist",
            &["body"],
            10,
            &boosts,
            Some(&cs),
        );

        assert_eq!(none_total, 2, "sanity: two live matches");
        assert_eq!(ext_total, none_total, "totals must agree between modes");
        assert_eq!(none_hits.len(), 2);
        assert_eq!(ext_hits.len(), 2);
        for (n, e) in none_hits.iter().zip(ext_hits.iter()) {
            assert_eq!(n.doc_id, e.doc_id, "hit order diverged between modes");
            assert_eq!(
                n.score.to_bits(),
                e.score.to_bits(),
                "{}: external-mode fallback scored {} vs None-mode {} — the \
                 scalar N dropped the ghost from IDF",
                n.doc_id,
                e.score,
                n.score
            );
        }
    }
}

#[cfg(test)]
mod aggregate_bytes_tests {
    use super::*;
    use serde_json::json;
    use std::sync::mpsc;
    use std::time::Duration;

    /// #872 — the sampler thread calls `size_bytes()` every 100 ms; it must
    /// never block on a shard write lock held by a turbo batch.  Pre-fix the
    /// sum read-locked every shard, so a held write lock stalled it and this
    /// test's `recv_timeout` expired.
    #[test]
    fn size_bytes_does_not_take_shard_locks() {
        let registry = Arc::new(AnalyzerRegistry::default());
        let mem = ShardedFtsMemtable::with_registry_and_shards(registry, 4);
        mem.insert(
            "doc-1".to_string(),
            &json!({"body": "buffered text"}),
            &Schema::default(),
            1,
        );

        let (locked_tx, locked_rx) = mpsc::channel();
        let (release_tx, release_rx) = mpsc::channel::<()>();
        let (size_tx, size_rx) = mpsc::channel();
        std::thread::scope(|scope| {
            let mem_ref = &mem;
            let holder = scope.spawn(move || {
                mem_ref.with_shard_mut(0, |_| {
                    locked_tx.send(()).unwrap();
                    // Bounded hold: release on our own after 4 s even if the
                    // release signal never arrives, so a lock-summing
                    // `size_bytes` makes this test FAIL (timeout above) rather
                    // than deadlock the scope join.
                    let _ = release_rx.recv_timeout(Duration::from_secs(4));
                });
            });
            locked_rx.recv().unwrap();
            let reader = scope.spawn(move || {
                size_tx.send(mem_ref.size_bytes()).unwrap();
            });
            let size = size_rx
                .recv_timeout(Duration::from_secs(2))
                .expect("size_bytes must not block on a held shard write lock");
            assert!(size > 0, "one buffered doc must report a non-zero size");
            // Best-effort: the holder also self-releases on its 4 s bound.
            let _ = release_tx.send(());
            holder.join().unwrap();
            reader.join().unwrap();
        });
    }

    /// The invariant `#872` rests on: the shared aggregate equals a full
    /// per-shard recount after every mutator that touches `total_bytes`.
    /// #1015: the recount must fold the FROZEN generations in — freeze
    /// moves docs behind the reader view without touching their charge,
    /// so an active-only sum diverges the moment a flush claims a shard.
    fn recount(mem: &ShardedFtsMemtable) -> usize {
        (0..mem.shard_count())
            .map(|i| {
                let g = mem.shards[i].read();
                g.active.size_bytes()
                    + g.frozen
                        .iter()
                        .map(|fg| fg.gen.read().size_bytes())
                        .sum::<usize>()
            })
            .sum()
    }

    #[test]
    fn aggregate_bytes_matches_full_recount_after_every_mutator() {
        let registry = Arc::new(AnalyzerRegistry::default());
        // Arc: `freeze_shard_accounted` hands the claim a handle to the
        // whole memtable so `retire`/`release` can find the shard again.
        let mem = Arc::new(ShardedFtsMemtable::with_registry_and_shards(registry, 4));
        let schema = Schema::default();
        let check = |step: &str| {
            assert_eq!(
                mem.size_bytes(),
                recount(&mem),
                "aggregate diverged from full recount after {step}"
            );
        };

        (0..8u64).for_each(|i| {
            mem.insert(
                format!("doc-{i}"),
                &json!({"body": format!("buffered text {i}")}),
                &schema,
                i + 1,
            );
            check("insert");
        });

        mem.insert_pretokenized_with_seq(
            9,
            "pretok-1".to_string(),
            Arc::new(json!({"body": "pretokenized doc"})),
            &[],
        );
        check("insert_pretokenized_with_seq");

        mem.insert_raw_bytes_with_seq(
            10,
            "raw-1".to_string(),
            Arc::from(&b"{\"body\":\"raw doc\"}"[..]),
        );
        check("insert_raw_bytes_with_seq");

        // Turbo bypass shape: fresh raw insert under one held shard lock.
        mem.with_shard_mut(0, |m| {
            m.insert_raw_bytes_fresh(
                11,
                "fresh-1".to_string(),
                Arc::from(&b"{\"body\":\"f\"}"[..]),
            )
        });
        check("insert_raw_bytes_fresh");

        mem.remove("doc-0");
        check("remove of an existing doc");
        mem.remove("no-such-doc");
        check("remove of a missing doc");

        // #1015 — the flush path freezes a generation (bytes stay charged:
        // the docs remain reader-visible through the build) and retires it
        // at publish (bytes leave the aggregate).  A released claim (failed
        // flush) must keep its bytes charged — release moves nothing.
        let (_, claim) = mem.freeze_shard_accounted(0, false);
        check("freeze_shard_accounted(0)");
        let frozen_bytes = mem.size_bytes();
        let _retained = claim.retire();
        assert!(
            mem.size_bytes() <= frozen_bytes,
            "retire must not grow the aggregate"
        );
        check("retire");

        let (_, claim) = mem.freeze_shard_accounted(1, false);
        check("freeze_shard_accounted(1)");
        claim.release();
        check("release (failed flush keeps its bytes charged)");

        (0..4u64).for_each(|i| {
            mem.insert(
                format!("redoc-{i}"),
                &json!({"body": format!("second wave {i}")}),
                &schema,
                20 + i,
            );
            check("re-insert");
        });
        let _ = mem.drain();
        check("drain");

        (0..4u64).for_each(|i| {
            mem.insert(
                format!("tridoc-{i}"),
                &json!({"body": format!("third wave {i}")}),
                &schema,
                30 + i,
            );
            check("re-insert");
        });
        let _ = mem.drain_with_sources();
        check("drain_with_sources");
        assert_eq!(mem.size_bytes(), 0, "fully drained memtable must read 0");
    }

    /// Issue #948: the raw-bytes entry charge must track what the entry
    /// actually retains. A mailbox-shaped doc (~10 KB of NDJSON held
    /// verbatim until flush) must charge at least its payload length — the
    /// pre-fix flat 800 bytes let the process-wide memtable budget admit
    /// ~12× more live bytes than it was told about. Small (log-shaped) docs
    /// must keep the historical 800-byte floor so the M5.17 flush-cadence
    /// behaviour is byte-for-byte unchanged for them.
    ///
    /// Uses only the public insert/size_bytes surface (no helper symbols
    /// introduced by the fix) so the mailbox assertion is a mechanical
    /// FAIL-BEFORE on pre-fix source: it charged 800 there.
    #[test]
    fn raw_bytes_charge_tracks_payload_with_an_800_byte_floor() {
        use std::sync::Arc as StdArc;

        let registry = StdArc::new(AnalyzerRegistry::default());
        let mem = ShardedFtsMemtable::with_registry_and_shards(registry, 2);

        // Small (log-shaped) doc: keeps the historical flat charge exactly.
        mem.insert_raw_bytes_with_seq(
            1,
            "id-1".to_string(),
            StdArc::from(&b"{\"body\":\"log line\"}"[..]),
        );
        assert_eq!(
            mem.size_bytes(),
            800,
            "small docs must keep the M5.6/M5.17 flat charge and flush cadence"
        );

        // Mailbox-shaped doc: must charge at least its own payload length.
        let payload = vec![b'x'; 9_830];
        mem.insert_raw_bytes_with_seq(
            2,
            "message-42".to_string(),
            StdArc::from(payload.clone().into_boxed_slice()),
        );
        assert!(
            mem.size_bytes() >= 800 + payload.len(),
            "memtable size_bytes must not under-count a mailbox-sized doc by ~12x (pre-fix: 800/doc)"
        );
        assert_eq!(mem.size_bytes(), recount(&mem));
    }
}