Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): answer text exists without hydrating the corpus (#2130)

xerj-org1c314d49c6cd
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index ad585e02c..3a19e6d73 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -383,6 +383,25 @@ pub(crate) fn note_fast_path_agg() {
FAST_PATH_AGGS_SERVED.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
+/// Documents cloned out of segment stored sections into the brute
+/// aggregation corpus. Memtable docs are not counted: that buffer is
+/// flush-bounded. A verify query that stays on the columnar path leaves
+/// this unchanged no matter how many documents the index holds.
+static CORPUS_DOCS_HYDRATED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+
+/// Segment documents copied into an owned aggregation corpus since process
+/// start. Read two of these around a search to see whether that search
+/// materialised stored documents.
+pub fn corpus_docs_hydrated() -> u64 {
+ CORPUS_DOCS_HYDRATED.load(std::sync::atomic::Ordering::Relaxed)
+}
+
+pub(crate) fn note_corpus_hydration(n: u64) {
+ if n > 0 {
+ CORPUS_DOCS_HYDRATED.fetch_add(n, std::sync::atomic::Ordering::Relaxed);
+ }
+}
+
/// Per-call aggregation limits. Passed to [`run_aggs_with_limits`] by callers
/// that need a limit other than the node-wide default (a per-request
/// `search.max_buckets` override, or a test pinning the cap for one call).
diff --git a/engine/crates/xerj-engine/src/fast_aggs.rs b/engine/crates/xerj-engine/src/fast_aggs.rs
index a496da6cb..3929d7604 100644
--- a/engine/crates/xerj-engine/src/fast_aggs.rs
+++ b/engine/crates/xerj-engine/src/fast_aggs.rs
@@ -65,7 +65,7 @@ use crate::aggs::{
use crate::memtable::MemBoolPred;
use serde_json::{json, Map, Value};
use std::collections::HashMap;
-use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn};
+use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn, PresenceColumn};
/// Compile the top-level agg filter JSON (the restricted shape
/// `query_node_to_agg_filter` emits: `term`/`range`/`bool{must,filter}`) into a
@@ -599,6 +599,10 @@ impl<'a> FastCtx<'a> {
enum ColKind {
Numeric,
Keyword,
+ /// `doc_values: false` field. The column records which rows have a
+ /// value, not the value. Term and metric executors must not read it
+ /// as a dictionary or a number.
+ Presence,
}
#[derive(Clone, Copy, PartialEq, Eq)]
@@ -1038,6 +1042,7 @@ impl<'a> FastCtx<'a> {
let k = match s.col(field) {
Some(Column::Numeric(_)) => ColKind::Numeric,
Some(Column::Keyword(_)) => ColKind::Keyword,
+ Some(Column::Presence(_)) => ColKind::Presence,
None => continue,
};
match kind {
@@ -1745,6 +1750,9 @@ impl<'a> FastCtx<'a> {
// Only reachable for value_count.
acc.count += (k.doc_count as u64).saturating_sub(k.null_bitmap.len());
}
+ // Text presence is not a value_count of tokens. Bail so the
+ // brute extractor answers.
+ Some(Column::Presence(_)) => return None,
None => {}
}
}
@@ -1831,7 +1839,7 @@ impl<'a> FastCtx<'a> {
}
}
},
- Some(Column::Keyword(_)) => return None,
+ Some(Column::Keyword(_)) | Some(Column::Presence(_)) => return None,
None => {}
}
}
@@ -1927,7 +1935,7 @@ impl<'a> FastCtx<'a> {
}
}
}
- Some(Column::Keyword(_)) => return None,
+ Some(Column::Keyword(_)) | Some(Column::Presence(_)) => return None,
None => {}
}
}
@@ -2146,6 +2154,13 @@ impl<'a> FastCtx<'a> {
match seg.col(field) {
Some(Column::Numeric(n)) => missing += n.null_bitmap.len(),
Some(Column::Keyword(k)) => missing += k.null_bitmap.len(),
+ // Complement of `exists`: the bitmap is `value_present`.
+ Some(Column::Presence(p)) => {
+ if p.doc_count != seg.docs {
+ return None;
+ }
+ missing += (seg.docs as u64).saturating_sub(p.count());
+ }
// Field absent from this segment's columns → every row missing.
None => missing += seg.docs as u64,
}
@@ -2347,6 +2362,7 @@ impl<'a> FastCtx<'a> {
any = true;
}
}
+ Some(Column::Presence(_)) => return None,
Some(Column::Keyword(k)) => {
let ord_ms = self.date_ord_index(seg, field)?;
for (ord, &ms) in ord_ms.iter().enumerate() {
@@ -2433,6 +2449,12 @@ impl<'a> FastCtx<'a> {
)?),
None => None,
};
+ // A terms window that names no ordinal in this segment resolves to
+ // `Never` before any row is visited. Zero-hit finalize probes stop
+ // here instead of walking the segment.
+ if matches!(top_sp, Some(SegPred::Never)) {
+ return Some(());
+ }
// Resolve metric columns once.
let mut num_cols: Vec<Option<&NumericColumn>> = Vec::with_capacity(plan.metrics.len());
let mut kw_cols: Vec<Option<&KeywordColumn>> = Vec::with_capacity(plan.metrics.len());
@@ -2449,6 +2471,7 @@ impl<'a> FastCtx<'a> {
num_cols.push(None);
kw_cols.push(Some(k));
}
+ Some(Column::Presence(_)) => return None,
None => {
num_cols.push(None);
kw_cols.push(None);
@@ -4099,9 +4122,12 @@ impl<'a> FastCtx<'a> {
Ok(Some(ColKind::Keyword)) | Ok(None) => Some(()),
_ => None,
},
- // Either column kind can answer "row is non-null".
+ // Keyword, numeric, and presence columns can answer "row is non-null".
Pred::Exists { field } => match self.seg_field_kind(field) {
- Ok(Some(ColKind::Keyword)) | Ok(Some(ColKind::Numeric)) | Ok(None) => Some(()),
+ Ok(Some(ColKind::Keyword))
+ | Ok(Some(ColKind::Numeric))
+ | Ok(Some(ColKind::Presence))
+ | Ok(None) => Some(()),
_ => None,
},
Pred::And(subs) => {
@@ -5596,6 +5622,9 @@ enum SegPred<'a> {
/// `exists` — the row carries a non-null keyword / numeric value.
KwExists(&'a KeywordColumn),
NumExists(&'a NumericColumn),
+ /// `exists` on a `doc_values: false` field. The bitmap is the set of
+ /// rows `value_present` accepted at flush.
+ PresExists(&'a PresenceColumn),
/// Conjunction — every sub-predicate must match the row.
And(Vec<SegPred<'a>>),
}
@@ -5626,7 +5655,7 @@ fn resolve_pred<'a>(
Some(ord) => SegPred::KwEq(k, ord, k.null_bitmap.is_empty()),
None => SegPred::Never,
},
- Some(Column::Numeric(_)) => return None,
+ Some(Column::Numeric(_)) | Some(Column::Presence(_)) => return None,
None => miss_is_genuinely_empty(field, segs, mapped_fields)?,
},
Pred::TermsKw { field, values } => match dv_col(cols, field) {
@@ -5638,7 +5667,7 @@ fn resolve_pred<'a>(
SegPred::KwIn(k, ords, k.null_bitmap.is_empty())
}
}
- Some(Column::Numeric(_)) => return None,
+ Some(Column::Numeric(_)) | Some(Column::Presence(_)) => return None,
None => miss_is_genuinely_empty(field, segs, mapped_fields)?,
},
Pred::RangeKw {
@@ -5698,7 +5727,7 @@ fn resolve_pred<'a>(
SegPred::KwRange(k, lo_ord, hi_ord, k.null_bitmap.is_empty())
}
}
- Some(Column::Numeric(_)) => return None,
+ Some(Column::Numeric(_)) | Some(Column::Presence(_)) => return None,
None => miss_is_genuinely_empty(field, segs, mapped_fields)?,
},
Pred::RangeNum {
@@ -5711,12 +5740,13 @@ fn resolve_pred<'a>(
Some(Column::Numeric(n)) => {
SegPred::Num(n, *lo, *lo_incl, *hi, *hi_incl, n.null_bitmap.is_empty())
}
- Some(Column::Keyword(_)) => return None,
+ Some(Column::Keyword(_)) | Some(Column::Presence(_)) => return None,
None => miss_is_genuinely_empty(field, segs, mapped_fields)?,
},
Pred::Exists { field } => match dv_col(cols, field) {
Some(Column::Keyword(k)) => SegPred::KwExists(k),
Some(Column::Numeric(n)) => SegPred::NumExists(n),
+ Some(Column::Presence(p)) => SegPred::PresExists(p),
None => miss_is_genuinely_empty(field, segs, mapped_fields)?,
},
Pred::And(subs) => {
@@ -5778,6 +5808,7 @@ fn seg_pred_matches(sp: &SegPred<'_>, row: u32) -> bool {
}
SegPred::KwExists(k) => k.ord_for(row).is_some(),
SegPred::NumExists(n) => !n.null_bitmap.contains(row),
+ SegPred::PresExists(p) => p.contains(row),
SegPred::And(subs) => subs.iter().all(|s| seg_pred_matches(s, row)),
}
}
@@ -5804,6 +5835,9 @@ fn seg_pred_count(sp: &SegPred<'_>, docs: u32) -> u64 {
// the numeric column's own precomputed non-null count.
SegPred::KwExists(k) => k.per_ord_count.iter().map(|c| *c as u64).sum(),
SegPred::NumExists(n) => n.live_count,
+ // Exact for any density. Delete-blind, like the other O(1) arms;
+ // dirty segments row-scan via `seg_pred_matches` instead.
+ SegPred::PresExists(p) => p.count(),
// Conjunction has no O(1) form — count matching rows directly. Only
// reached from the top-level filter count (compile_pred, used by the
// filter/filters executors, never yields `And`).
diff --git a/engine/crates/xerj-engine/src/graph.rs b/engine/crates/xerj-engine/src/graph.rs
index 4ed07d1d3..6663572fb 100644
--- a/engine/crates/xerj-engine/src/graph.rs
+++ b/engine/crates/xerj-engine/src/graph.rs
@@ -334,9 +334,10 @@ impl Index {
let n_invalid = match cols.get("invalid_at") {
None => None,
Some(Column::Numeric(n)) => Some(n),
- // Present but keyword-typed = a writer stored ISO strings;
- // an as-of compare against strings would be a lie.
- Some(Column::Keyword(_)) => {
+ // Present but not numeric = a writer stored ISO strings,
+ // or only a presence bit. An as-of compare against either
+ // would be a lie.
+ Some(Column::Keyword(_)) | Some(Column::Presence(_)) => {
stats.segments_without_columns += 1;
continue;
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index d4d1b9cfe..d3651e855 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -564,6 +564,60 @@ mod collection_publication_fail_closed_tests {
);
}
+ /// `doc_values: false` fields (text, semantic_text) record presence, not
+ /// values. The bit must follow `value_present`: empty string and a
+ /// non-null array element are present; null, `[]`, and `[null]` are not.
+ /// An all-absent field still ships a column so `exists` can answer
+ /// "no" without a source scan.
+ #[test]
+ fn build_doc_value_columns_presence_matches_value_present() {
+ use xerj_storage::doc_values::Column;
+ let mut skip = std::collections::HashSet::new();
+ skip.insert("msg".to_string());
+ skip.insert("never".to_string());
+ let docs = [
+ json!({"ax_file": "a"}),
+ json!({"ax_file": "a", "msg": null}),
+ json!({"ax_file": "a", "msg": []}),
+ json!({"ax_file": "a", "msg": ""}),
+ json!({"ax_file": "a", "msg": [null]}),
+ json!({"ax_file": "a", "msg": "hello"}),
+ json!({"ax_file": "a", "msg": [null, "x"]}),
+ ];
+ let cols = super::build_doc_value_columns(docs.iter().map(Some), &skip);
+ let Some(Column::Presence(msg)) = cols.get("msg") else {
+ panic!("msg must be a presence column, got {:?}", cols.get("msg"));
+ };
+ assert_eq!(msg.doc_count, 7);
+ assert!(!msg.contains(0) && !msg.contains(1) && !msg.contains(2) && !msg.contains(4));
+ assert!(msg.contains(3) && msg.contains(5) && msg.contains(6));
+ assert_eq!(msg.count(), 3);
+ let Some(Column::Presence(never)) = cols.get("never") else {
+ panic!("an all-absent skipped field must still ship a presence column");
+ };
+ assert_eq!(never.doc_count, 7);
+ assert_eq!(never.count(), 0);
+ assert!(matches!(cols.get("ax_file"), Some(Column::Keyword(_))));
+ }
+
+ /// A dotted mapping name is an object sub-field. The builder only walks
+ /// top-level `_source` keys, so it cannot see `user.bio` inside
+ /// `{"user": {"bio": ..}}`. It must ship no presence column for it (the
+ /// reader then scans source) rather than an empty one that says no row
+ /// has the field.
+ #[test]
+ fn build_doc_value_columns_ships_no_presence_for_a_dotted_text_field() {
+ let mut skip = std::collections::HashSet::new();
+ skip.insert("user.bio".to_string());
+ let docs = [json!({"user": {"bio": "hello"}}), json!({"user": {}})];
+ let cols = super::build_doc_value_columns(docs.iter().map(Some), &skip);
+ assert!(
+ !cols.contains_key("user.bio"),
+ "a dotted field must not get a presence column: {:?}",
+ cols.get("user.bio")
+ );
+ }
+
async fn assert_flush_freeze_keeps_readers_live_then_finishes(inject_error: bool) {
let dir = tempfile::tempdir().unwrap();
let mut config = Config::default();
@@ -25204,6 +25258,7 @@ impl Index {
)));
}
};
+ let mut segment_hydrated: u64 = 0;
for d in arr.iter() {
// Skip tombstoned docs.
let id_ref = d.get("_id").and_then(Value::as_str).unwrap_or("");
@@ -25228,7 +25283,9 @@ impl Index {
}
}
docs.push(src);
+ segment_hydrated = segment_hydrated.saturating_add(1);
}
+ crate::aggs::note_corpus_hydration(segment_hydrated);
}
// Enrich corpus docs with hit-level metadata so score/seq-sensitive
// sub-aggs reproduce ES semantics: `top_hits` default-sorts by
@@ -27221,13 +27278,29 @@ fn build_doc_value_columns<'a>(
skip: &std::collections::HashSet<String>,
) -> std::collections::BTreeMap<String, xerj_storage::doc_values::Column> {
use std::collections::BTreeMap;
- use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn};
+ use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn, PresenceColumn};
// Collect per-field per-doc raw values. We materialise into vectors
// so we can decide column-type once we've seen all values. This is
// O(docs × fields) memory but much smaller than the segment itself.
let mut numeric: BTreeMap<String, Vec<Option<i64>>> = BTreeMap::new();
let mut keyword: BTreeMap<String, Vec<Option<String>>> = BTreeMap::new();
+ // `doc_values: false` fields (text, semantic_text) get a presence
+ // bitmap instead of a value column. Pre-seed every skipped name so a
+ // segment where the field is absent still has a column: `exists` then
+ // answers "no row" without parsing source, and a missing column stays
+ // reserved for segments written before this bitmap existed.
+ //
+ // Only top-level names. The loop below sees `_source`'s top-level keys,
+ // so a dotted mapping name (`user.bio`, an object sub-field) never gets a
+ // bit set; seeding it would ship an all-absent bitmap and `exists` would
+ // answer "no row" for documents that have the field. Those fields get
+ // no column and keep the source scan.
+ let mut presence: BTreeMap<String, PresenceColumn> = skip
+ .iter()
+ .filter(|name| !name.contains('.'))
+ .map(|name| (name.clone(), PresenceColumn::empty()))
+ .collect();
// Fields that carry an ARRAY value in ANY doc of this segment. The
// one-slot-per-row column cannot represent a multi-valued doc, and
// filing it as null makes the column LIE: every consumer (count
@@ -27255,9 +27328,15 @@ fn build_doc_value_columns<'a>(
if field.starts_with('_') {
continue;
}
- // Honour the declared mapping. Checked before any allocation so a
- // skipped field costs nothing per doc.
+ // Honour the declared mapping. A skipped field records presence
+ // only — one bit, not the value — so `exists` does not need the
+ // stored JSON. Checked before any value allocation.
if skip.contains(field.as_str()) {
+ if value_present(val) {
+ if let Some(bits) = presence.get_mut(field.as_str()) {
+ bits.insert(doc_idx as u32);
+ }
+ }
continue;
}
// Lookup-first (`get_mut` before `entry`) so the common case
@@ -27360,6 +27439,10 @@ fn build_doc_value_columns<'a>(
}
}
}
+ for (name, mut present) in presence {
+ present.set_doc_count(total_docs as u32);
+ out.insert(name, Column::Presence(present));
+ }
out
}
@@ -27997,7 +28080,8 @@ impl Index {
n.null_bitmap.is_empty(),
));
}
- None => {
+ // Presence answers `exists`, not term equality.
+ Some(xerj_storage::doc_values::Column::Presence(_)) | None => {
abandoned = true;
break;
}
@@ -28283,7 +28367,8 @@ impl Index {
shadow, slo, shi, slo_incl, shi_incl,
));
}
- None => return None, // abandon — field absent in this segment
+ // A presence bitmap has no numeric or date values.
+ Some(xerj_storage::doc_values::Column::Presence(_)) | None => return None,
}
}
@@ -28468,19 +28553,22 @@ impl Index {
}
}
}
- None => return None, // no dv column → can't count safely
+ // Presence is not a term dictionary.
+ Some(Column::Presence(_)) | None => return None,
}
}
return Some(seg_matches);
}
- // `exists` — provable only for the DENSE case: a field whose column
- // has no nulls and covers every doc means every doc has a scalar value
- // (arrays / objects / json-null leave the null bitmap non-empty), so
- // all docs match. Any null / partial coverage / missing column, or a
- // non-empty memtable, abandons to the scan — which resolves
- // `get_field_value(...).is_some()` exactly (incl. arrays, dotted
- // paths). Meta fields are always present.
+ // `exists` — keyword and numeric columns answer only the DENSE case:
+ // no nulls and full coverage means every doc has a scalar value
+ // (arrays poison the column, so a present column has no hidden
+ // multi-values). A presence bitmap answers any density, because it
+ // records `value_present` per row — the same predicate the source
+ // scan uses — including empty strings and excluding null / `[]` /
+ // `[null]`. A missing column, a column that does not cover the
+ // segment, or a non-empty memtable abandons to the scan. Meta
+ // fields are always present.
if let QueryNode::Exists { field } = query {
use xerj_storage::doc_values::Column;
if self.store.version_map.ghost_events() > 0 || self.memtable.doc_count() > 0 {
@@ -28499,15 +28587,28 @@ impl Index {
continue;
}
let cols = self.dv_columns_for(&segments_dir, &meta.id)?;
- let (null_empty, dcount) = match cols.get(field.as_str()) {
- Some(Column::Keyword(k)) => (k.null_bitmap.is_empty(), k.doc_count as u64),
- Some(Column::Numeric(n)) => (n.null_bitmap.is_empty(), n.doc_count as u64),
- None => return None, // can't prove presence → scan
+ let add = match cols.get(field.as_str()) {
+ Some(Column::Keyword(k)) => {
+ if !k.null_bitmap.is_empty() || k.doc_count as u64 != meta.doc_count {
+ return None;
+ }
+ meta.doc_count
+ }
+ Some(Column::Numeric(n)) => {
+ if !n.null_bitmap.is_empty() || n.doc_count as u64 != meta.doc_count {
+ return None;
+ }
+ meta.doc_count
+ }
+ Some(Column::Presence(p)) => {
+ if p.doc_count as u64 != meta.doc_count {
+ return None;
+ }
+ p.count()
+ }
+ None => return None,
};
- if !null_empty || dcount != meta.doc_count {
- return None; // sparse / partial → scan (may still match arrays)
- }
- seg_matches = seg_matches.saturating_add(meta.doc_count);
+ seg_matches = seg_matches.saturating_add(add);
}
return Some(seg_matches);
}
@@ -28640,7 +28741,11 @@ impl Index {
served_by_dv = true;
}
}
- None => {}
+ // Presence is not a term dictionary. Leave `served_by_dv`
+ // false so the caller abandons to the source scan (#362):
+ // a `term` on text must agree with `_search`, which
+ // compares the whole source value, not a bit.
+ Some(xerj_storage::doc_values::Column::Presence(_)) | None => {}
}
}
@@ -30184,6 +30289,17 @@ impl Index {
});
}
ScoredFilterLeaf::SourceExists { field } => {
+ // A presence bitmap answers `exists` without parsing
+ // stored JSON. Segments written before the bitmap
+ // existed still fall through to the source slices.
+ if let Some(xerj_storage::doc_values::Column::Presence(p)) =
+ cols.get(field.as_str())
+ {
+ if p.doc_count == meta.doc_count as u32 {
+ fev.push(FilterEval::Presence { col: p });
+ continue;
+ }
+ }
// Cache-backed stored slices for this segment; the
// `?` bails the whole columnar path (brute answers).
let slices = self.stored_slices_for(meta.id.as_str(), meta.doc_count)?;
@@ -30818,6 +30934,15 @@ impl Index {
Some((hits, total, from_applied))
}
+ /// Stored sections this index currently holds decoded: the decompressed
+ /// byte cache, the per-doc slice cache, and the parsed-document cache.
+ /// A presence-backed `exists` leaves all three untouched.
+ pub fn stored_sections_cached(&self) -> usize {
+ self.decoded_stored_cache.len()
+ + self.stored_slices_cache.len()
+ + self.stored_value_cache.len()
+ }
+
/// Cache-backed reader for a segment's parsed stored section.
///
/// First call for a segment opens the segment, decodes the Stored
@@ -31173,7 +31298,7 @@ impl Index {
}
shadow_range_positions(shadow, slo, shi, slo_incl, shi_incl)
}
- None => return None,
+ Some(Column::Presence(_)) | None => return None,
};
let built: Arc<PrefilterSet> = Arc::new(PrefilterSet::from_vec(matching));
if self.range_prefilter_cache.len() >= 32 {
@@ -31269,6 +31394,8 @@ impl Index {
}
Some(Arc::new(PrefilterSet::from_set(set)))
}
+ // No ordinals to enumerate. A full scan stays correct.
+ Column::Presence(_) => None,
}
}
@@ -31915,6 +32042,7 @@ impl Index {
},
)
}
+ Column::Presence(_) => None,
}
}
@@ -49690,6 +49818,7 @@ fn seg_term_positions(
}
Some(set)
}
+ Column::Presence(_) => None,
}
}
@@ -49855,7 +49984,7 @@ fn build_sort_shadow(
});
Some(Arc::new(sorted))
}
- None => None,
+ Some(Column::Presence(_)) | None => None,
}
}
@@ -53221,10 +53350,17 @@ enum FilterEval<'a> {
/// presence from the segment's cached stored slices, using the SAME
/// `get_field_value` + `value_present` predicate as the brute scan.
/// Owned `Resident` so the eval outlives the segment iteration.
+ /// Reached only when the segment has no presence bitmap.
Source {
field: String,
slices: Resident<StoredSlices>,
},
+ /// `exists` answered from the segment's presence bitmap. No stored
+ /// bytes are decoded. `col.doc_count` was checked against the segment
+ /// length before this leaf was built.
+ Presence {
+ col: &'a xerj_storage::doc_values::PresenceColumn,
+ },
}
impl FilterEval<'_> {
#[inline]
@@ -53272,6 +53408,7 @@ impl FilterEval<'_> {
let source = doc.get("_source").unwrap_or(&doc);
get_field_value(source, field).is_some_and(|v| value_present(&v))
}
+ FilterEval::Presence { col } => col.contains(row),
}
}
@@ -53321,6 +53458,10 @@ impl FilterEval<'_> {
FilterEval::Source { .. } => {
unreachable!("source-backed exists has no closed-form count")
}
+ // Bitmap cardinality. Ghosts are excluded by the caller that
+ // walks rows; `constant_plan` still refuses SourceExists, so
+ // this arm is the exhaustive match, not the closed-form lane.
+ FilterEval::Presence { col } => col.count(),
}
}
}
@@ -53536,21 +53677,14 @@ enum ScoredFilterLeaf {
min_inc: bool,
max_inc: bool,
},
- /// `exists` on a field with NO doc-values column (text /
- /// semantic_text): per-row presence is only observable in the stored
- /// source. Evaluated from the segment's cached stored slices — the
- /// SAME `get_field_value` + `value_present` predicate the brute scan
- /// applies, so the hit set cannot drift — but only reached on rows
- /// that already passed every cheaper conjunct in the same loop (each
- /// row walk short-circuits in fev order), which is what turns
- /// `bool.filter: [term, exists]` from a whole-index source scan into
- /// a term-column walk plus a handful of source parses. #1183's
- /// finalize-verify query (`term: ax_file` + `exists: <semantic
- /// field>`) measured took=9638ms / 0 hits on a 91k-doc segment
- /// because `exists` in a bool forced the brute path; this leaf makes
- /// it columnar. Keyword/numeric `exists` does NOT come here — those
- /// lower to the empty-prefix dictionary range / unbounded window
- /// above and never touch the source.
+ /// `exists` on a field with no value column (text / semantic_text,
+ /// `doc_values: false`). A segment that carries a presence bitmap
+ /// answers from that bitmap. Older segments fall back to the cached
+ /// stored slices and the SAME `get_field_value` + `value_present`
+ /// predicate the brute scan applies. The leaf is only reached on rows
+ /// that already passed every cheaper conjunct in the same loop.
+ /// Keyword/numeric `exists` does NOT come here — those lower to the
+ /// empty-prefix dictionary range / unbounded window above.
SourceExists { field: String },
}
diff --git a/engine/crates/xerj-engine/tests/text_exists_does_not_hydrate.rs b/engine/crates/xerj-engine/tests/text_exists_does_not_hydrate.rs
new file mode 100644
index 000000000..c5793f8eb
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/text_exists_does_not_hydrate.rs
@@ -0,0 +1,210 @@
+//! Finalize-verify (`size:0` + `terms` agg + `exists` on a
+//! `doc_values: false` text field) must not materialise the stored corpus.
+//!
+//! Issue #1940: `exists` on `semantic_text` / `text` had no doc-values
+//! column, so the filtered fast path refused the request and the brute
+//! corpus cloned every stored document. A 10k-document window then grew
+//! RSS with the corpus. The presence bitmap answers the same
+//! `value_present` predicate from the `.dv` sidecar.
+//!
+//! The counters are the bound. They do not move with document count:
+//! `corpus_docs_hydrated` counts stored documents cloned into the brute
+//! aggregation corpus, and `stored_sections_cached` counts decoded stored
+//! sections. Both stay flat across a full window, a one-digest window, and
+//! a zero-hit digest.
+
+use serde_json::{json, Value};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::aggs::{corpus_docs_hydrated, fast_path_aggs_served};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+/// Clears `FAST_AGG_MIN_DOCS` (10_000). 100 groups × 100 docs.
+const GROUPS: usize = 100;
+const DOCS_PER_GROUP: usize = 100;
+const DOCS: usize = GROUPS * DOCS_PER_GROUP;
+
+static ONE_AT_A_TIME: std::sync::LazyLock<tokio::sync::Mutex<()>> =
+ std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn schema() -> Schema {
+ let mut s = Schema::empty();
+ s.fields
+ .push(FieldConfig::new("ax_file", FieldType::Keyword));
+ let mut msg = FieldConfig::new("msg", FieldType::Text);
+ msg.options.doc_values = false;
+ s.fields.push(msg);
+ s
+}
+
+fn digest(g: usize) -> String {
+ format!("axf-{g:04}")
+}
+
+/// Variant `i % 10`. Present (exists = true) for `""` and any non-null
+/// string, including a non-null element inside an array. Absent for a
+/// missing key, JSON null, `[]`, and `[null]`.
+fn msg_value(i: usize) -> Option<Value> {
+ match i % 10 {
+ 0 => None,
+ 1 => Some(Value::Null),
+ 2 => Some(json!([])),
+ 3 => Some(json!("")),
+ 4 => Some(json!([null])),
+ _ => Some(json!(format!("record {i} {}", "x".repeat(1024)))),
+ }
+}
+
+fn exists_variant(i: usize) -> bool {
+ matches!(i % 10, 3 | 5 | 6 | 7 | 8 | 9)
+}
+
+async fn seed(idx: &Arc<Index>) {
+ for i in 0..DOCS {
+ let g = i / DOCS_PER_GROUP;
+ let mut doc = serde_json::Map::new();
+ doc.insert("ax_file".to_string(), json!(digest(g)));
+ if let Some(msg) = msg_value(i) {
+ doc.insert("msg".to_string(), msg);
+ }
+ idx.index_document(Some(format!("doc-{i}")), Value::Object(doc))
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+}
+
+fn verify_query(digests: &[String]) -> Value {
+ json!({
+ "size": 0,
+ "query": {
+ "bool": {
+ "filter": [
+ { "terms": { "ax_file": digests } },
+ { "exists": { "field": "msg" } }
+ ]
+ }
+ },
+ "aggs": {
+ "values": {
+ "terms": { "field": "ax_file", "size": digests.len().max(1) }
+ }
+ }
+ })
+}
+
+async fn search(idx: &Arc<Index>, body: Value) -> xerj_query::executor::SearchResult {
+ let req = parse_request(&body).unwrap();
+ idx.search(&req).await.unwrap()
+}
+
+fn buckets(aggs: &Value) -> HashMap<String, u64> {
+ aggs["values"]["buckets"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|b| {
+ (
+ b["key"].as_str().unwrap().to_string(),
+ b["doc_count"].as_u64().unwrap(),
+ )
+ })
+ .collect()
+}
+
+#[tokio::test]
+async fn text_exists_verify_does_not_hydrate() {
+ let _sequential = ONE_AT_A_TIME.lock().await;
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ engine.create_index("verify", schema()).unwrap();
+ let idx = engine.get_index("verify").unwrap();
+ seed(&idx).await;
+
+ let expected_per_group = (0..DOCS_PER_GROUP).filter(|r| exists_variant(*r)).count() as u64;
+ assert_eq!(expected_per_group, 60);
+ let all: Vec<String> = (0..GROUPS).map(digest).collect();
+
+ let cached_before = idx.stored_sections_cached();
+ let hydrated_before = corpus_docs_hydrated();
+ let served_before = fast_path_aggs_served();
+
+ let full = search(&idx, verify_query(&all)).await;
+ assert_eq!(full.total.value, expected_per_group * GROUPS as u64);
+ let full_buckets = buckets(full.aggs.as_ref().unwrap());
+ assert_eq!(full_buckets.len(), GROUPS);
+ for g in 0..GROUPS {
+ assert_eq!(full_buckets[&digest(g)], expected_per_group);
+ }
+
+ let one = search(&idx, verify_query(&[digest(3)])).await;
+ assert_eq!(one.total.value, expected_per_group);
+ let one_buckets = buckets(one.aggs.as_ref().unwrap());
+ assert_eq!(one_buckets.len(), 1);
+ assert_eq!(one_buckets[&digest(3)], expected_per_group);
+
+ let miss = search(&idx, verify_query(&["no-such-digest".to_string()])).await;
+ assert_eq!(miss.total.value, 0);
+ assert!(buckets(miss.aggs.as_ref().unwrap()).is_empty());
+
+ // size:1 zero-hit, no aggs. The scored path must answer `exists` from
+ // the bitmap rather than decoding the stored section up front.
+ let page = search(
+ &idx,
+ json!({
+ "size": 1,
+ "query": {
+ "bool": {
+ "filter": [
+ { "terms": { "ax_file": ["no-such-digest"] } },
+ { "exists": { "field": "msg" } }
+ ]
+ }
+ }
+ }),
+ )
+ .await;
+ assert_eq!(page.total.value, 0);
+ assert!(page.hits.is_empty());
+
+ assert_eq!(
+ corpus_docs_hydrated() - hydrated_before,
+ 0,
+ "verify/exists cloned stored documents into the aggregation corpus"
+ );
+ assert_eq!(
+ idx.stored_sections_cached(),
+ cached_before,
+ "verify/exists decoded a stored section"
+ );
+ assert!(
+ fast_path_aggs_served() - served_before >= 3,
+ "the three size:0 windows must be served by the columnar fast path"
+ );
+
+ // Brute reference. `missing` is not a fast-path terms parameter, so the
+ // executor refuses and the source scan answers. Every document has
+ // `ax_file`, so the placeholder bucket stays empty and the live buckets
+ // match the columnar result.
+ let mut reference = verify_query(&all);
+ reference["aggs"]["values"]["terms"]["missing"] = json!("__absent__");
+ let ref_served = fast_path_aggs_served();
+ let brute = search(&idx, reference).await;
+ assert_eq!(
+ fast_path_aggs_served() - ref_served,
+ 0,
+ "the missing parameter must force the brute executor"
+ );
+ assert_eq!(brute.total.value, full.total.value);
+ assert_eq!(buckets(brute.aggs.as_ref().unwrap()), full_buckets);
+}
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 5f6cb19c0..43be64457 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -29,7 +29,46 @@
// fragmentation actively and madvise()'s freed regions back.
#[cfg(not(target_env = "msvc"))]
#[global_allocator]
-static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+static ALLOC: ReportingJemalloc = ReportingJemalloc;
+
+/// Jemalloc plus a FATAL line when an allocation returns null.
+///
+/// `handle_alloc_error` still runs afterwards and aborts, which raises
+/// SIGABRT. The line is written first, with `write(2)`, because the heap
+/// is the thing that just failed. SIGKILL cannot be reported.
+#[cfg(not(target_env = "msvc"))]
+struct ReportingJemalloc;
+
+#[cfg(not(target_env = "msvc"))]
+unsafe impl std::alloc::GlobalAlloc for ReportingJemalloc {
+ unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 {
+ let ptr = tikv_jemallocator::Jemalloc.alloc(layout);
+ if ptr.is_null() {
+ note_alloc_failure(layout.size(), layout.align());
+ }
+ ptr
+ }
+
+ unsafe fn alloc_zeroed(&self, layout: std::alloc::Layout) -> *mut u8 {
+ let ptr = tikv_jemallocator::Jemalloc.alloc_zeroed(layout);
+ if ptr.is_null() {
+ note_alloc_failure(layout.size(), layout.align());
+ }
+ ptr
+ }
+
+ unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) {
+ tikv_jemallocator::Jemalloc.dealloc(ptr, layout);
+ }
+
+ unsafe fn realloc(&self, ptr: *mut u8, layout: std::alloc::Layout, new_size: usize) -> *mut u8 {
+ let ptr = tikv_jemallocator::Jemalloc.realloc(ptr, layout, new_size);
+ if ptr.is_null() {
+ note_alloc_failure(new_size, layout.align());
+ }
+ ptr
+ }
+}
// jemalloc runtime config (read at process start via the `_rjem_malloc_conf`
// symbol — tikv-jemalloc is built with the `_rjem_` prefix, so plain
@@ -2268,9 +2307,252 @@ mod nofile_tests {
}
}
+#[cfg(test)]
+mod fatal_reporter_tests {
+ use super::{format_alloc_failure, format_fatal_signal};
+
+ #[test]
+ fn alloc_failure_line_names_size_and_align() {
+ let mut buf = [0u8; 128];
+ let n = format_alloc_failure(&mut buf, 4096, 8);
+ let line = std::str::from_utf8(&buf[..n]).unwrap();
+ assert!(line.starts_with("FATAL:"));
+ assert!(line.contains("allocation"));
+ assert!(line.contains("4096"));
+ assert!(line.contains("align 8"));
+ assert!(line.ends_with('\n'));
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn fatal_signal_line_names_sigabrt_without_installing_a_handler() {
+ let mut buf = [0u8; 128];
+ let n = format_fatal_signal(&mut buf, libc::SIGABRT);
+ let line = std::str::from_utf8(&buf[..n]).unwrap();
+ assert!(line.starts_with("FATAL:"));
+ assert!(line.contains("SIGABRT"));
+ assert!(line.contains(&libc::SIGABRT.to_string()));
+ assert!(line.ends_with('\n'));
+ }
+}
+
+/// `{data_dir}/crash.log`, opened once the data directory exists. `-1`
+/// until then. Fatal signal and allocation-failure hooks write this fd
+/// with `write(2)` and never allocate.
+static CRASH_LOG_FD: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(-1);
+
+fn append_bytes(out: &mut [u8], mut at: usize, bytes: &[u8]) -> usize {
+ for &b in bytes {
+ if at >= out.len() {
+ break;
+ }
+ out[at] = b;
+ at += 1;
+ }
+ at
+}
+
+fn append_usize(out: &mut [u8], at: usize, mut value: usize) -> usize {
+ let mut buf = [0u8; 20];
+ let mut i = buf.len();
+ if value == 0 {
+ i -= 1;
+ buf[i] = b'0';
+ } else {
+ while value > 0 {
+ i -= 1;
+ buf[i] = b'0' + (value % 10) as u8;
+ value /= 10;
+ }
+ }
+ append_bytes(out, at, &buf[i..])
+}
+
+/// Allocation-failure line. Replaces the default
+/// "memory allocation of N bytes failed" message, so the size and
+/// alignment have to be in the line. No heap use: the alloc hook runs
+/// because the heap just failed.
+fn format_alloc_failure(out: &mut [u8], size: usize, align: usize) -> usize {
+ let mut at = append_bytes(out, 0, b"FATAL: memory allocation of ");
+ at = append_usize(out, at, size);
+ at = append_bytes(out, at, b" bytes failed (align ");
+ at = append_usize(out, at, align);
+ append_bytes(out, at, b")\n")
+}
+
+fn format_fatal_signal(out: &mut [u8], sig: i32) -> usize {
+ let name: &[u8] = match sig {
+ #[cfg(unix)]
+ x if x == libc::SIGABRT => b"SIGABRT",
+ #[cfg(unix)]
+ x if x == libc::SIGSEGV => b"SIGSEGV",
+ #[cfg(unix)]
+ x if x == libc::SIGBUS => b"SIGBUS",
+ #[cfg(unix)]
+ x if x == libc::SIGILL => b"SIGILL",
+ _ => b"signal",
+ };
+ let mut at = append_bytes(out, 0, b"FATAL: xerj received ");
+ at = append_bytes(out, at, name);
+ at = append_bytes(out, at, b" (");
+ at = append_usize(out, at, sig.max(0) as usize);
+ at = append_bytes(out, at, b") and is aborting\n");
+ at
+}
+
+#[cfg(unix)]
+fn write_fd(fd: i32, bytes: &[u8]) {
+ if fd < 0 || bytes.is_empty() {
+ return;
+ }
+ let mut off = 0usize;
+ while off < bytes.len() {
+ let n = unsafe {
+ libc::write(
+ fd,
+ bytes[off..].as_ptr() as *const libc::c_void,
+ bytes.len() - off,
+ )
+ };
+ if n <= 0 {
+ break;
+ }
+ off += n as usize;
+ }
+}
+
+#[cfg(unix)]
+fn report_fatal(bytes: &[u8]) {
+ write_fd(libc::STDERR_FILENO, bytes);
+ let fd = CRASH_LOG_FD.load(std::sync::atomic::Ordering::Relaxed);
+ if fd >= 0 && fd != libc::STDERR_FILENO {
+ write_fd(fd, bytes);
+ }
+}
+
+#[cfg(not(unix))]
+fn report_fatal(bytes: &[u8]) {
+ let _ = std::io::Write::write_all(&mut std::io::stderr(), bytes);
+}
+
+#[cfg(not(target_env = "msvc"))]
+fn note_alloc_failure(size: usize, align: usize) {
+ let mut buf = [0u8; 128];
+ let n = format_alloc_failure(&mut buf, size, align);
+ report_fatal(&buf[..n]);
+}
+
+/// The disposition each fatal signal had before [`install_fatal_reporter`]
+/// replaced it. Written once at startup, before the handler can run, and
+/// only read inside the handler.
+#[cfg(unix)]
+static mut PREVIOUS_FATAL_ACTIONS: [Option<libc::sigaction>; 4] = [None; 4];
+
+#[cfg(unix)]
+const FATAL_SIGNALS: [libc::c_int; 4] = [libc::SIGABRT, libc::SIGSEGV, libc::SIGBUS, libc::SIGILL];
+
+/// Async-signal-safe. Writes a line, puts back the disposition the signal
+/// had before (the Rust runtime's guard-page handler for SIGSEGV/SIGBUS,
+/// which prints "has overflowed its stack"; the default elsewhere), and
+/// returns. A fault re-executes and reaches that handler with its real
+/// `siginfo`; `abort()` re-raises SIGABRT after a handler returns; so the
+/// process still dies with the original signal. SIGKILL cannot be caught.
+/// SIGTERM is left to the server's shutdown path.
+#[cfg(unix)]
+extern "C" fn fatal_signal(sig: libc::c_int) {
+ let mut buf = [0u8; 128];
+ let n = format_fatal_signal(&mut buf, sig);
+ report_fatal(&buf[..n]);
+ unsafe {
+ let previous = std::ptr::addr_of!(PREVIOUS_FATAL_ACTIONS).read();
+ let slot = FATAL_SIGNALS.iter().position(|s| *s == sig);
+ match slot.and_then(|i| previous[i]) {
+ Some(action) => {
+ libc::sigaction(sig, &action, std::ptr::null_mut());
+ }
+ None => {
+ libc::signal(sig, libc::SIG_DFL);
+ }
+ }
+ if sig != libc::SIGSEGV && sig != libc::SIGBUS && sig != libc::SIGILL {
+ // A raised signal (not a fault) does not re-execute.
+ libc::raise(sig);
+ }
+ }
+}
+
+fn install_fatal_reporter() {
+ // Chain to the hook that was installed before: it prints the thread
+ // name, the location and (with RUST_BACKTRACE) the backtrace. A panic
+ // can be caught and the node keep serving, so the crash.log line says
+ // "panicked", not FATAL; a panic that ends the process is followed by
+ // the abort or exit that does.
+ let previous = std::panic::take_hook();
+ std::panic::set_hook(Box::new(move |info| {
+ #[cfg(unix)]
+ {
+ let fd = CRASH_LOG_FD.load(std::sync::atomic::Ordering::Relaxed);
+ if fd >= 0 {
+ let thread = std::thread::current();
+ let line = format!(
+ "xerj panicked on thread {}: {info}\n",
+ thread.name().unwrap_or("<unnamed>")
+ );
+ write_fd(fd, line.as_bytes());
+ }
+ }
+ previous(info);
+ }));
+ #[cfg(unix)]
+ unsafe {
+ let mut action: libc::sigaction = std::mem::zeroed();
+ action.sa_sigaction = fatal_signal as *const () as libc::sighandler_t;
+ libc::sigemptyset(&mut action.sa_mask);
+ // SA_ONSTACK: a stack overflow faults with no stack left, so the
+ // handler must run on the alternate stack the Rust runtime sets up
+ // per thread.
+ action.sa_flags = libc::SA_ONSTACK;
+ let mut previous: [Option<libc::sigaction>; 4] = [None; 4];
+ for (i, sig) in FATAL_SIGNALS.iter().enumerate() {
+ let mut old: libc::sigaction = std::mem::zeroed();
+ if libc::sigaction(*sig, std::ptr::null(), &mut old) == 0 {
+ previous[i] = Some(old);
+ }
+ }
+ std::ptr::addr_of_mut!(PREVIOUS_FATAL_ACTIONS).write(previous);
+ for sig in FATAL_SIGNALS {
+ libc::sigaction(sig, &action, std::ptr::null_mut());
+ }
+ }
+}
+
+#[cfg(unix)]
+fn open_crash_log(data_dir: &Path) {
+ let path = data_dir.join("crash.log");
+ let Ok(c_path) = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) else {
+ return;
+ };
+ let fd = unsafe {
+ libc::open(
+ c_path.as_ptr(),
+ libc::O_WRONLY | libc::O_CREAT | libc::O_APPEND,
+ 0o644,
+ )
+ };
+ if fd >= 0 {
+ CRASH_LOG_FD.store(fd, std::sync::atomic::Ordering::Relaxed);
+ }
+}
+
+#[cfg(not(unix))]
+fn open_crash_log(_data_dir: &Path) {}
+
fn main() -> Result<()> {
// Before anything can write to stdout, including the `--help`/`--version`
- // fast paths inside `parse_args`.
+ // fast paths inside `parse_args`. A node that dies must say so: the
+ // allocation hook, the panic hook, and SIGABRT/SIGSEGV/SIGBUS/SIGILL
+ // each write a FATAL line. SIGKILL still cannot be reported.
+ install_fatal_reporter();
#[cfg(unix)]
restore_default_sigpipe();
@@ -3407,6 +3689,7 @@ async fn async_main() -> Result<()> {
// 4. Data directory
std::fs::create_dir_all(&cfg.server.data_dir)
.with_context(|| format!("create data dir {}", cfg.server.data_dir))?;
+ open_crash_log(Path::new(&cfg.server.data_dir));
// 5. Admin key (first-run)
ensure_admin_key(&mut cfg)?;
diff --git a/engine/crates/xerj-storage/src/doc_values.rs b/engine/crates/xerj-storage/src/doc_values.rs
index 1e6d6bf1b..f18c65045 100644
--- a/engine/crates/xerj-storage/src/doc_values.rs
+++ b/engine/crates/xerj-storage/src/doc_values.rs
@@ -8,7 +8,7 @@
//!
//! ## What we store
//!
-//! Two column kinds:
+//! Three column kinds:
//!
//! 1. **`NumericColumn`** — a per-doc `Option<i64>` array. The natural
//! home for `Long`, `Integer`, `Date`, and `Boolean` (as 0/1).
@@ -20,6 +20,11 @@
//! so lookups are O(log |terms|) and the cardinality footprint is
//! near-Lucene.
//!
+//! 3. **`PresenceColumn`** — a roaring bitmap of rows whose source field
+//! is present (non-null). Written for fields that deliberately have no
+//! value column (`text`, `semantic_text`: `doc_values: false`) so
+//! `exists` can be answered without parsing stored JSON.
+//!
//! Both columns track a `RoaringBitmap` of "doc has no value" so a missing
//! field doesn't waste a slot.
//!
@@ -42,8 +47,8 @@
//! u32 magic = 0x44_56_30_31 ("DV01")
//! u32 num_columns
//! for each column:
-//! u8 kind (0 = numeric, 1 = keyword; 0x80 bit = zstd payload,
-//! clear = legacy LZ4 — see KIND_FLAG_ZSTD)
+//! u8 kind (0 = numeric, 1 = keyword, 2 = presence; 0x80 bit =
+//! zstd payload, clear = legacy LZ4 — see KIND_FLAG_ZSTD)
//! u32 field_name_len
//! bytes field_name
//! u64 payload_len
@@ -87,6 +92,7 @@ use std::io::{Cursor, Read};
const MAGIC: u32 = 0x44_56_30_31; // "DV01"
const KIND_NUMERIC: u8 = 0;
const KIND_KEYWORD: u8 = 1;
+const KIND_PRESENCE: u8 = 2;
/// Per-doc numeric column. `None` = no value for that doc.
///
@@ -847,11 +853,76 @@ impl KeywordColumn {
}
}
+/// Per-row presence of a field that has no value column.
+///
+/// `text` and `semantic_text` ship no doc-values *values* (`doc_values:
+/// false` — a value column would be a second copy of the source). `exists`
+/// only needs to know which rows have a non-null value. The bitmap is that
+/// answer: one bit per present row, not the field bytes. `present` holds
+/// row ids in `0..doc_count`. A row absent from the bitmap has no value
+/// (null, `[]`, and `[null]` are absent).
+#[derive(Debug, Clone)]
+pub struct PresenceColumn {
+ pub doc_count: u32,
+ /// Rows whose field is present.
+ pub present: RoaringBitmap,
+}
+
+impl PresenceColumn {
+ /// Empty bitmap. `doc_count` is filled in once the segment length is known.
+ pub fn empty() -> Self {
+ Self {
+ doc_count: 0,
+ present: RoaringBitmap::new(),
+ }
+ }
+
+ pub fn insert(&mut self, row: u32) {
+ self.present.insert(row);
+ }
+
+ pub fn set_doc_count(&mut self, doc_count: u32) {
+ self.doc_count = doc_count;
+ }
+
+ pub fn contains(&self, row: u32) -> bool {
+ row < self.doc_count && self.present.contains(row)
+ }
+
+ pub fn count(&self) -> u64 {
+ self.present.len()
+ }
+
+ pub fn encode(&self) -> Vec<u8> {
+ let mut bitmap_buf = Vec::new();
+ self.present.serialize_into(&mut bitmap_buf).unwrap();
+ let mut out = Vec::with_capacity(8 + bitmap_buf.len());
+ out.write_u32::<LittleEndian>(self.doc_count).unwrap();
+ out.write_u32::<LittleEndian>(bitmap_buf.len() as u32)
+ .unwrap();
+ out.extend_from_slice(&bitmap_buf);
+ out
+ }
+
+ pub fn decode(bytes: &[u8]) -> Result<Self> {
+ let mut cur = Cursor::new(bytes);
+ let doc_count = cur.read_u32::<LittleEndian>().map_err(io_to_storage)?;
+ let bitmap_len = cur.read_u32::<LittleEndian>().map_err(io_to_storage)? as usize;
+ let mut bitmap_bytes = vec![0u8; bitmap_len];
+ cur.read_exact(&mut bitmap_bytes).map_err(io_to_storage)?;
+ let present = RoaringBitmap::deserialize_from(&bitmap_bytes[..])
+ .map_err(|e| StorageError::Other(anyhow::anyhow!("dv presence bitmap: {e}")))?;
+ Ok(Self { doc_count, present })
+ }
+}
+
/// One column of any kind.
#[derive(Debug, Clone)]
pub enum Column {
Numeric(NumericColumn),
Keyword(KeywordColumn),
+ /// Field presence only — no values. See [`PresenceColumn`].
+ Presence(PresenceColumn),
}
impl Column {
@@ -859,6 +930,7 @@ impl Column {
match self {
Column::Numeric(n) => n.doc_count,
Column::Keyword(k) => k.doc_count,
+ Column::Presence(p) => p.doc_count,
}
}
@@ -882,6 +954,7 @@ impl Column {
.saturating_add(capacity_bytes::<u32>(column.per_ord_count.capacity()))
.saturating_add(roaring_retained_bytes(&column.null_bitmap))
}
+ Column::Presence(column) => roaring_retained_bytes(&column.present),
}
}
}
@@ -943,6 +1016,7 @@ pub fn encode_columns_at_level(columns: &BTreeMap<String, Column>, level: i32) -
let kind = match col {
Column::Numeric(_) => KIND_NUMERIC,
Column::Keyword(_) => KIND_KEYWORD,
+ Column::Presence(_) => KIND_PRESENCE,
};
// `level` is DV_ZSTD_LEVEL (3) on flush — fast encode
// (~250 MB/s/core), reverted from level 19 because the flush path
@@ -987,6 +1061,7 @@ pub fn encode_columns_at_level(columns: &BTreeMap<String, Column>, level: i32) -
}
}
Column::Keyword(k) => compress(&k.encode()),
+ Column::Presence(p) => compress(&p.encode()),
};
out.write_u8(kind | KIND_FLAG_ZSTD).unwrap();
out.write_u32::<LittleEndian>(name.len() as u32).unwrap();
@@ -1029,15 +1104,17 @@ pub fn decode_columns(bytes: &[u8]) -> Result<BTreeMap<String, Column>> {
.map_err(|e| StorageError::Other(anyhow::anyhow!("dv lz4 decompress: {e}")))?
};
let column = match kind {
- KIND_NUMERIC => Column::Numeric(NumericColumn::decode(&payload)?),
- KIND_KEYWORD => Column::Keyword(KeywordColumn::decode(&payload)?),
- _ => {
- return Err(StorageError::Other(anyhow::anyhow!(
- "unknown doc-values kind {kind}"
- )))
- }
+ KIND_NUMERIC => Some(Column::Numeric(NumericColumn::decode(&payload)?)),
+ KIND_KEYWORD => Some(Column::Keyword(KeywordColumn::decode(&payload)?)),
+ KIND_PRESENCE => Some(Column::Presence(PresenceColumn::decode(&payload)?)),
+ // A kind this build does not know is skipped, not fatal: failing
+ // the whole sidecar would drop every keyword and numeric column
+ // beside it. The unknown field falls through to the source scan.
+ _ => None,
};
- out.insert(name, column);
+ if let Some(column) = column {
+ out.insert(name, column);
+ }
}
Ok(out)
}
@@ -1392,4 +1469,57 @@ mod tests {
+ keyword_column.per_ord_count.capacity() * std::mem::size_of::<u32>();
assert!(keyword_estimate >= vector_floor as u64);
}
+
+ #[test]
+ fn presence_roundtrip_and_unknown_kind_keeps_sibling_columns() {
+ let mut present = RoaringBitmap::new();
+ present.insert(1);
+ present.insert(4);
+ let col = PresenceColumn {
+ doc_count: 6,
+ present,
+ };
+ assert!(col.contains(1));
+ assert!(!col.contains(0));
+ assert!(!col.contains(6));
+ let back = PresenceColumn::decode(&col.encode()).unwrap();
+ assert_eq!(back.doc_count, 6);
+ assert_eq!(back.count(), 2);
+ assert!(back.contains(4));
+
+ let mut cols = BTreeMap::new();
+ cols.insert("msg".to_string(), Column::Presence(col));
+ cols.insert(
+ "ax_file".to_string(),
+ Column::Keyword(KeywordColumn::from_iter(vec![Some("d".into()), None]).unwrap()),
+ );
+ let bytes = encode_columns(&cols);
+ let decoded = decode_columns(&bytes).unwrap();
+ let Column::Presence(p) = &decoded["msg"] else {
+ panic!("presence column missing");
+ };
+ assert!(p.contains(1) && !p.contains(2));
+ assert!(matches!(decoded.get("ax_file"), Some(Column::Keyword(_))));
+
+ // Kind 7 is not a column this build knows. Splice it in front of
+ // the frame just decoded and confirm the known columns survive.
+ let mut framed = Vec::new();
+ framed.write_u32::<LittleEndian>(0x44_56_30_31).unwrap();
+ framed.write_u32::<LittleEndian>(3).unwrap();
+ framed.write_u8(7 | 0x80).unwrap();
+ framed.write_u32::<LittleEndian>(3).unwrap();
+ framed.extend_from_slice(b"odd");
+ let payload = zstd::encode_all(&b"nope"[..], 1).unwrap();
+ framed
+ .write_u64::<LittleEndian>(payload.len() as u64)
+ .unwrap();
+ framed.extend_from_slice(&payload);
+ // `bytes` is magic + count + columns. Skip the 8-byte header and
+ // keep the two known columns.
+ framed.extend_from_slice(&bytes[8..]);
+ let decoded = decode_columns(&framed).unwrap();
+ assert!(!decoded.contains_key("odd"));
+ assert!(matches!(decoded.get("msg"), Some(Column::Presence(_))));
+ assert!(matches!(decoded.get("ax_file"), Some(Column::Keyword(_))));
+ }
}