Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Cold FTS top-k: walk on identities, hydrate only the page's winners (#2454)

xerj-org767c377e97be
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 649720a29..40390de23 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1968,6 +1968,149 @@ mod flush_publication_recovery_tests {
schema
}
+ /// #2422: a broad query (most rows of a segment match) on a cold
+ /// segment used to decode the whole stored section inline, because the
+ /// first segment a page meets has no score floor and every matching row
+ /// was a candidate. The walk must read only identities and hydrate the
+ /// page's winners, with or without a `_source` projection, and a walk
+ /// that rejects thousands of rows (here: the best ones are deleted) must
+ /// still see every row, so every answer equals the warm answer.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn cold_broad_topk_reads_identities_and_hydrates_only_winners() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine
+ .create_index("broad-cold", topk_projection_schema())
+ .unwrap();
+ let idx = engine.get_index("broad-cold").unwrap();
+ idx.abort_background_tasks();
+ for i in 0..4096u64 {
+ let body = if i % 2 == 0 {
+ format!("needle stack report {i} needle")
+ } else {
+ format!("hay stack {i}")
+ };
+ idx.index_document(
+ Some(format!("doc-{i}")),
+ json!({"title": format!("title {i}"), "body": body, "blob": "x"}),
+ )
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+ // Delete the best-scoring needle rows, more than the row window
+ // holds, so the walk rejects every row in the window and must read
+ // the live needle rows past it.
+ for i in (0..4000u64).step_by(2) {
+ assert!(idx.delete_document(&format!("doc-{i}")).await.unwrap());
+ }
+ idx.flush().await.unwrap();
+ drop(idx);
+ drop(engine);
+
+ let reopen = || {
+ let mut c = Config::default();
+ c.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(c).unwrap();
+ let idx = engine.get_index("broad-cold").unwrap();
+ idx.abort_background_tasks();
+ // Hold the background warm back so each query is observed alone.
+ let snapshot = idx.store.snapshot();
+ let segs: Vec<_> = snapshot.segments.clone();
+ drop(snapshot);
+ assert!(
+ segs.iter().any(|s| s.doc_count > 1024),
+ "precondition: a segment large enough to exceed the row window"
+ );
+ for s in &segs {
+ idx.cold_stored_warms.insert(s.id.to_string(), ());
+ }
+ (engine, idx, segs)
+ };
+ let all_cold = |idx: &Index, segs: &[xerj_storage::segment::SegmentMeta]| {
+ segs.iter().all(|s| {
+ !idx.stored_slices_cache.contains_key(&s.id)
+ && !idx.decoded_stored_cache.contains_key(&s.id)
+ })
+ };
+ let key = |r: &xerj_query::executor::SearchResult| -> Vec<(String, u32, Value)> {
+ r.hits
+ .iter()
+ .map(|h| (h.id.clone(), h.score.to_bits(), h.source.clone()))
+ .collect()
+ };
+ let request = |q: &str, projected: bool| {
+ let mut body = json!({
+ "query": {"match": {"body": q}},
+ "size": 5,
+ });
+ if projected {
+ body["_source"] = json!(["title"]);
+ }
+ xerj_query::parse_request(&body).unwrap()
+ };
+ let mut requests = Vec::new();
+ for projected in [true, false] {
+ // Every document matches; the live hay rows score highest.
+ requests.push(request("hay stack", projected));
+ // Matches the needle rows only. The best ~2000 of them are
+ // deleted, so the walk rejects thousands of rows first.
+ requests.push(request("needle report", projected));
+ }
+
+ let (engine, idx, segs) = reopen();
+ let mut cold = Vec::new();
+ for r in &requests {
+ cold.push(idx.search(r).await.unwrap());
+ assert!(
+ all_cold(&idx, &segs),
+ "a 5-hit query must not decode the whole stored section"
+ );
+ }
+ assert!(idx.cold_row_hydrations.load(Ordering::Relaxed) > 0);
+ drop(idx);
+ drop(engine);
+
+ let (engine, idx, segs) = reopen();
+ // Warm every segment that can be warmed; the large one (the walk
+ // under test) must be, so the reference takes the full-decode path.
+ for s in &segs {
+ let warmed = idx.stored_slices_for(&s.id, s.doc_count).is_some();
+ assert!(warmed || s.doc_count <= 1024, "segment {} must warm", s.id);
+ }
+ for (r, cold) in requests.iter().zip(&cold) {
+ let warm = idx.search(r).await.unwrap();
+ assert_eq!(cold.hits.len(), 5);
+ assert_eq!(key(cold), key(&warm));
+ assert_eq!(cold.total.value, warm.total.value);
+ assert_eq!(
+ cold.hits
+ .iter()
+ .filter(|h| h.source["title"].is_string())
+ .count(),
+ 5
+ );
+ }
+ for (i, h) in [&cold[1], &cold[3]]
+ .iter()
+ .flat_map(|r| r.hits.iter())
+ .enumerate()
+ {
+ let n: u64 = h.id.trim_start_matches("doc-").parse().unwrap();
+ assert!(
+ n >= 4000 && n.is_multiple_of(2),
+ "live needle rows only: {i} {}",
+ h.id
+ );
+ }
+ // Unprojected hits carry the whole source.
+ assert!(cold[2].hits.iter().all(|h| h.source["body"].is_string()));
+ drop(idx);
+ drop(engine);
+ }
+
/// #2422/#2423: on a cold segment (after a restart, or evicted under the
/// cache budget) a top-k query that names its `_source` fields must read
/// only the winning rows and only the needed columns. Before the fix the
@@ -17401,12 +17544,14 @@ impl Index {
keep: Option<&[String]>,
) -> Option<HashMap<u32, Value>> {
use xerj_storage::stored_codec::{
- decode_stored_v2_rows, decode_stored_v2_rows_projected, stored_v2_source_column_names,
+ decode_stored_v2_rows_projected_parallel, stored_v2_source_column_names,
StoredV2RowHydrationResult,
};
let reader = self.store.open_segment_arc(segment_id).ok()?;
let bytes = reader.section(SectionType::Stored).ok()??;
let ordinals: Vec<usize> = positions.iter().map(|&p| p as usize).collect();
+ // Columns decode concurrently: a handful of rows still streams every
+ // wanted column, so the serial sum was the cost of a cold page.
let hydrated = match keep {
Some(keep) => {
let names = stored_v2_source_column_names(bytes).ok()??;
@@ -17415,9 +17560,9 @@ impl Index {
.map(String::as_str)
.filter(|column| row_projection_keeps_column(keep, column))
.collect();
- decode_stored_v2_rows_projected(bytes, &ordinals, &include).ok()?
+ decode_stored_v2_rows_projected_parallel(bytes, &ordinals, Some(&include)).ok()?
}
- None => decode_stored_v2_rows(bytes, &ordinals).ok()?,
+ None => decode_stored_v2_rows_projected_parallel(bytes, &ordinals, None).ok()?,
};
let StoredV2RowHydrationResult::Hydrated { rows, .. } = hydrated else {
return None;
@@ -17439,6 +17584,138 @@ impl Index {
Some(out)
}
+ /// `_id` per row for `positions` of a cold segment, decoded from the
+ /// identity columns alone (no `_source` column is read). `None` (legacy
+ /// layout, open/decode error, a missing row) sends the caller back to
+ /// the full decode.
+ fn cold_fts_row_ids(
+ &self,
+ segment_id: &str,
+ positions: &[u32],
+ ) -> Option<HashMap<u32, String>> {
+ use xerj_storage::stored_codec::{
+ decode_stored_v2_rows_projected, StoredV2RowHydrationResult,
+ };
+ let reader = self.store.open_segment_arc(segment_id).ok()?;
+ let bytes = reader.section(SectionType::Stored).ok()??;
+ let ordinals: Vec<usize> = positions.iter().map(|&p| p as usize).collect();
+ let StoredV2RowHydrationResult::Hydrated { rows, .. } =
+ decode_stored_v2_rows_projected(bytes, &ordinals, &[]).ok()?
+ else {
+ return None;
+ };
+ let mut out: HashMap<u32, String> = HashMap::with_capacity(rows.len());
+ for row in rows {
+ out.insert(
+ u32::try_from(row.ordinal).ok()?,
+ row.id.as_str()?.to_string(),
+ );
+ }
+ if positions.iter().any(|p| !out.contains_key(p)) {
+ return None;
+ }
+ Some(out)
+ }
+
+ /// Fill the `_source` of the cold FTS hits the segment loop kept with
+ /// identity only (`deferred`: `_id` -> (segment, row, doc count)). Only
+ /// the page's winners are read, row-selectively and per segment in
+ /// parallel. A segment whose rows cannot be hydrated that way falls back
+ /// to the full decode, as the walk itself used to; a hit whose document
+ /// cannot be read at all is dropped, as the walk skipped an unreadable
+ /// row. Projected hits are recorded in `row_projected` so the TF-IDF
+ /// fallback can restore their whole source.
+ fn hydrate_deferred_cold_sources(
+ &self,
+ hits: Vec<Hit>,
+ deferred: &HashMap<String, (String, u32, u64)>,
+ keep: Option<&[String]>,
+ row_projected: &mut HashMap<String, (String, u32)>,
+ ) -> Vec<Hit> {
+ use rayon::prelude::*;
+ let mut by_segment: HashMap<&str, (Vec<u32>, u64)> = HashMap::new();
+ for hit in &hits {
+ if let Some((seg, pos, doc_count)) = deferred.get(&hit.id) {
+ by_segment
+ .entry(seg.as_str())
+ .or_insert_with(|| (Vec::new(), *doc_count))
+ .0
+ .push(*pos);
+ }
+ }
+ if by_segment.is_empty() {
+ return hits;
+ }
+ let segments: Vec<(&str, Vec<u32>, u64)> = by_segment
+ .into_iter()
+ .map(|(seg, (positions, doc_count))| (seg, positions, doc_count))
+ .collect();
+ // (segment, docs by row, projected?)
+ let loaded: Vec<(&str, HashMap<u32, Value>, bool)> = segments
+ .par_iter()
+ .map(|(seg, positions, doc_count)| {
+ if let Some(rows) = self.hydrate_fts_rows_cold(seg, positions, keep) {
+ return (*seg, rows, keep.is_some());
+ }
+ let mut docs: HashMap<u32, Value> = HashMap::with_capacity(positions.len());
+ if let Some(slices) = self.stored_slices_for(seg, *doc_count) {
+ for &pos in positions {
+ let doc = slices
+ .offsets
+ .get(pos as usize)
+ .and_then(|&(start, end)| {
+ slices.bytes.get(start as usize..end as usize)
+ })
+ .and_then(|slice| serde_json::from_slice::<Value>(slice).ok());
+ if let Some(doc) = doc {
+ docs.insert(pos, doc);
+ }
+ }
+ } else if let Some(all) = self.store.open_segment_arc(seg).ok().and_then(|reader| {
+ match reader.section(SectionType::Stored) {
+ Ok(Some(raw)) => xerj_storage::stored_codec::decode_stored(raw)
+ .ok()
+ .and_then(|b| serde_json::from_slice::<Vec<Value>>(&b).ok()),
+ _ => None,
+ }
+ }) {
+ for &pos in positions {
+ if let Some(doc) = all.get(pos as usize) {
+ docs.insert(pos, doc.clone());
+ }
+ }
+ }
+ (*seg, docs, false)
+ })
+ .collect();
+ let mut docs_by_segment: HashMap<&str, (HashMap<u32, Value>, bool)> = loaded
+ .into_iter()
+ .map(|(seg, docs, projected)| (seg, (docs, projected)))
+ .collect();
+ let mut out = Vec::with_capacity(hits.len());
+ for mut hit in hits {
+ let Some((seg, pos, _)) = deferred.get(&hit.id) else {
+ out.push(hit);
+ continue;
+ };
+ let Some((docs, projected)) = docs_by_segment.get_mut(seg.as_str()) else {
+ continue;
+ };
+ let Some(mut doc) = docs.remove(pos) else {
+ continue;
+ };
+ hit.source = doc
+ .get_mut("_source")
+ .map(Value::take)
+ .unwrap_or(Value::Null);
+ if *projected {
+ row_projected.insert(hit.id.clone(), (seg.clone(), *pos));
+ }
+ out.push(hit);
+ }
+ out
+ }
+
#[allow(clippy::too_many_arguments)]
async fn run_knn_brute_force_with_deadline(
&self,
@@ -24184,6 +24461,9 @@ impl Index {
// position), so the one post-page reader that looks at the WHOLE
// document (the TF-IDF fallback below) can restore it.
let mut row_projected: HashMap<String, (String, u32)> = HashMap::new();
+ // Cold score-ordered FTS hits whose `_source` is read only once the
+ // segment loop has settled the page: `_id` -> (segment, row).
+ let mut deferred_sources: HashMap<String, (String, u32, u64)> = HashMap::new();
let materialisation_limit: usize = if mem_doc_count == 0
&& !deletes_present
&& size > 0
@@ -25159,18 +25439,35 @@ impl Index {
// threshold only tightens, so this is a
// superset) — and only the columns the
// request can observe.
+ // #2422: set when the cold row window
+ // below was cut short; the walk then
+ // falls back to the full decode for any
+ // row past it instead of skipping it.
+ let mut cold_window_truncated = false;
+ // Cold score-ordered top-k: the walk
+ // reads only each candidate's `_id`
+ // (version-map check, dedup, page key),
+ // so decode the identity columns for
+ // every candidate and defer `_source`
+ // until the segment loop has picked the
+ // page's winners (see the hydration
+ // after the loop). The first segment a
+ // page meets has no score floor, so
+ // without this a broad query decoded the
+ // whole stored section inline to show
+ // five hits (~1.45 s on a 131 MB
+ // section).
+ let mut cold_ids: Option<HashMap<u32, String>> = None;
let cold_rows: Option<HashMap<u32, Value>> = if self
.stored_slices_cache
.contains_key(seg_id.as_str())
|| self.decoded_stored_cache.contains_key(seg_id.as_str())
{
None
- } else {
- let entry_worst = if sort_topk.is_none() {
- page_worst.as_ref().map(|worst| worst.0)
- } else {
- None
- };
+ } else if sort_topk.is_none() {
+ let entry_worst = page_worst.as_ref().map(|worst| worst.0);
+ // A superset of the rows the walk can
+ // visit: the threshold only tightens.
let candidates: Vec<u32> = seg_hits
.iter()
.take_while(|sh| {
@@ -25178,8 +25475,33 @@ impl Index {
})
.map(|sh| sh.doc_id)
.collect();
- let bounded = candidates.len()
- <= materialisation_limit.saturating_mul(4).max(256);
+ if !candidates.is_empty() {
+ cold_ids =
+ self.cold_fts_row_ids(seg_id.as_str(), &candidates);
+ }
+ if cold_ids.is_some() {
+ self.cold_row_hydrations
+ .fetch_add(1, Ordering::Relaxed);
+ self.schedule_cold_stored_warm(
+ seg_id.as_str(),
+ meta.doc_count,
+ );
+ }
+ None
+ } else {
+ // A field sort offers every match to
+ // the top-N heap, so its candidates
+ // are all matches: decode a bounded
+ // score-ordered window and read any
+ // row past it from the full decode.
+ let mut candidates: Vec<u32> =
+ seg_hits.iter().map(|sh| sh.doc_id).collect();
+ let window =
+ materialisation_limit.saturating_mul(4).max(256);
+ if candidates.len() > window {
+ candidates.truncate(window);
+ cold_window_truncated = true;
+ }
// Only with a column projection: the
// stored columns are whole-stream
// zstd, so an unprojected row decode
@@ -25192,7 +25514,7 @@ impl Index {
// 2.3 s -> 6.6 s). The full decode
// below caches for the next query.
let worth_it = fts_row_keep.is_some();
- if bounded && worth_it && !candidates.is_empty() {
+ if worth_it && !candidates.is_empty() {
let rows = self.hydrate_fts_rows_cold(
seg_id.as_str(),
&candidates,
@@ -25211,7 +25533,7 @@ impl Index {
None
}
};
- let slices_opt = if cold_rows.is_some() {
+ let slices_opt = if cold_rows.is_some() || cold_ids.is_some() {
None
} else {
self.stored_slices_for(seg_id.as_str(), meta.doc_count)
@@ -25227,6 +25549,7 @@ impl Index {
// real corruption.
let whole_fallback: Option<Vec<Value>> = if slices_opt.is_some()
|| cold_rows.is_some()
+ || cold_ids.is_some()
{
None
} else {
@@ -25250,6 +25573,7 @@ impl Index {
}
};
if cold_rows.is_some()
+ || cold_ids.is_some()
|| slices_opt.is_some()
|| whole_fallback.is_some()
{
@@ -25258,9 +25582,43 @@ impl Index {
// slice arm parses one object on demand;
// the fallback clones from the fully
// decoded Vec.
+ let past_window: std::cell::OnceCell<
+ Option<Resident<StoredSlices>>,
+ > = std::cell::OnceCell::new();
let fetch_doc = |pos: u32| -> Option<Value> {
+ if let Some(ids) = cold_ids.as_ref() {
+ // Identity only: `_source` is
+ // hydrated for the winners
+ // after the segment loop.
+ let id = ids.get(&pos)?;
+ let mut doc = serde_json::Map::with_capacity(1);
+ doc.insert(
+ "_id".to_string(),
+ Value::String(id.clone()),
+ );
+ return Some(Value::Object(doc));
+ }
if let Some(rows) = cold_rows.as_ref() {
- rows.get(&pos).cloned()
+ if let Some(doc) = rows.get(&pos) {
+ return Some(doc.clone());
+ }
+ if !cold_window_truncated {
+ return None;
+ }
+ let slices = past_window
+ .get_or_init(|| {
+ self.stored_slices_for(
+ seg_id.as_str(),
+ meta.doc_count,
+ )
+ })
+ .as_deref()?;
+ let &(start, end) =
+ slices.offsets.get(pos as usize)?;
+ let slice = slices
+ .bytes
+ .get(start as usize..end as usize)?;
+ serde_json::from_slice::<Value>(slice).ok()
} else if let Some(slices) = slices_opt.as_deref() {
let &(start, end) =
slices.offsets.get(pos as usize)?;
@@ -25399,7 +25757,13 @@ impl Index {
let source =
doc.get("_source").cloned().unwrap_or(Value::Null);
seen_ids.insert(id.clone());
- if cold_rows.is_some() && fts_row_keep.is_some() {
+ if cold_ids.is_some() {
+ deferred_sources.insert(
+ id.clone(),
+ (seg_id.to_string(), sh.doc_id, meta.doc_count),
+ );
+ } else if cold_rows.is_some() && fts_row_keep.is_some()
+ {
row_projected.insert(
id.clone(),
(seg_id.to_string(), sh.doc_id),
@@ -26119,6 +26483,14 @@ impl Index {
}
}
}
+ if !deferred_sources.is_empty() {
+ all_hits = self.hydrate_deferred_cold_sources(
+ all_hits,
+ &deferred_sources,
+ fts_row_keep.as_deref(),
+ &mut row_projected,
+ );
+ }
drop(snap);
// F1: did the bounded collector fill to the cap? If so the stored-doc
@@ -33841,6 +34213,16 @@ impl Index {
))
}
+ /// `_id` of every row of a segment's columnar stored section, read from
+ /// its identity columns only. `None` (legacy layout, read error, a row
+ /// count other than `expect_docs`) sends the caller to the full decode.
+ fn cold_segment_ids(&self, seg_id: &str, expect_docs: u64) -> Option<Vec<Value>> {
+ let reader = self.store.open_segment_arc(seg_id).ok()?;
+ let bytes = reader.section(SectionType::Stored).ok()??;
+ let ids = xerj_storage::stored_codec::decode_stored_v2_ids(bytes).ok()??;
+ (ids.len() as u64 == expect_docs).then_some(ids)
+ }
+
/// Per-segment GHOST-position bitmap (b7 DEFECT 1c): bit `pos` set ⇔
/// the stored doc at `pos` is tombstoned or superseded — its live
/// version resides in another segment or the memtable per the version
@@ -33861,21 +34243,39 @@ impl Index {
return Some(Arc::clone(&entry.value().1));
}
}
- let slices = self.stored_slices_for(seg_id, expect_docs)?;
- let mut bm = vec![0u64; slices.offsets.len().div_ceil(64)];
- for (pos, &(start, end)) in slices.offsets.iter().enumerate() {
- let slice = slices.bytes.get(start as usize..end as usize)?;
- let doc: Value = serde_json::from_slice(slice).ok()?;
- let id = doc.get("_id").and_then(Value::as_str).unwrap_or("");
- let ghost = match self.store.version_map.get(id) {
- Some(ver) => ver.deleted || ver.segment_id.as_ref() != seg_id,
- // Not in the version map → nothing has ever superseded it.
- None => false,
- };
- if ghost {
- bm[pos / 64] |= 1u64 << (pos % 64);
+ let is_ghost = |id: &str| match self.store.version_map.get(id) {
+ Some(ver) => ver.deleted || ver.segment_id.as_ref() != seg_id,
+ // Not in the version map → nothing has ever superseded it.
+ None => false,
+ };
+ // A cold segment reads its `_id` column alone: decoding (and
+ // caching) the whole stored section here made the first query
+ // under deletes pay for every row's `_source`.
+ let cold = !self.stored_slices_cache.contains_key(seg_id)
+ && !self.decoded_stored_cache.contains_key(seg_id);
+ let cold_ids = cold
+ .then(|| self.cold_segment_ids(seg_id, expect_docs))
+ .flatten();
+ let bm = if let Some(ids) = cold_ids {
+ let mut bm = vec![0u64; ids.len().div_ceil(64)];
+ for (pos, id) in ids.iter().enumerate() {
+ if is_ghost(id.as_str().unwrap_or("")) {
+ bm[pos / 64] |= 1u64 << (pos % 64);
+ }
}
- }
+ bm
+ } else {
+ let slices = self.stored_slices_for(seg_id, expect_docs)?;
+ let mut bm = vec![0u64; slices.offsets.len().div_ceil(64)];
+ for (pos, &(start, end)) in slices.offsets.iter().enumerate() {
+ let slice = slices.bytes.get(start as usize..end as usize)?;
+ let doc: Value = serde_json::from_slice(slice).ok()?;
+ if is_ghost(doc.get("_id").and_then(Value::as_str).unwrap_or("")) {
+ bm[pos / 64] |= 1u64 << (pos % 64);
+ }
+ }
+ bm
+ };
let bm = Arc::new(bm);
// Bounded like `shortcut_count_cache`: wholesale clear — entries
// are per-segment (not query-shaped) so the cap is generous.
diff --git a/engine/crates/xerj-storage/examples/zz_cold_probe.rs b/engine/crates/xerj-storage/examples/zz_cold_probe.rs
new file mode 100644
index 000000000..d671c912d
--- /dev/null
+++ b/engine/crates/xerj-storage/examples/zz_cold_probe.rs
@@ -0,0 +1,56 @@
+use std::time::Instant;
+use xerj_storage::segment::{SectionType, SegmentReader};
+use xerj_storage::stored_codec::*;
+fn main() {
+ let path = std::env::args().nth(1).unwrap();
+ let r = SegmentReader::open(&path).unwrap();
+ let raw = r.section(SectionType::Stored).unwrap().unwrap();
+ println!("stored section {} bytes", raw.len());
+ let t = Instant::now();
+ let full = decode_stored(raw).unwrap();
+ println!("decode_stored {:?} -> {} bytes", t.elapsed(), full.len());
+ let n: usize = std::env::args()
+ .nth(2)
+ .map(|s| s.parse().unwrap())
+ .unwrap_or(30000);
+ let all: Vec<usize> = (0..n).collect();
+ let t = Instant::now();
+ let res = decode_stored_v2_rows_projected(raw, &all, &[]);
+ match res {
+ Ok(StoredV2RowHydrationResult::Hydrated { rows, .. }) => {
+ println!("identity rows {} {:?}", rows.len(), t.elapsed())
+ }
+ Ok(_) => println!("other"),
+ Err(e) => println!("err {e}"),
+ }
+ let five: Vec<usize> = vec![1, 100, 2000, 7000, n - 1];
+ let t = Instant::now();
+ let res = decode_stored_v2_rows(raw, &five);
+ match res {
+ Ok(StoredV2RowHydrationResult::Hydrated { rows, .. }) => {
+ println!("5 full rows {} {:?}", rows.len(), t.elapsed())
+ }
+ Ok(_) => println!("other"),
+ Err(e) => println!("err {e}"),
+ }
+ let t = Instant::now();
+ let res = decode_stored_v2_rows_projected_parallel(raw, &five, None);
+ match res {
+ Ok(StoredV2RowHydrationResult::Hydrated { rows, .. }) => {
+ println!("5 full rows PAR {} {:?}", rows.len(), t.elapsed())
+ }
+ Ok(_) => println!("other"),
+ Err(e) => println!("err {e}"),
+ }
+ if let Ok(Some(names)) = stored_v2_source_column_names(raw) {
+ for nm in &names {
+ let t = Instant::now();
+ let _ = decode_stored_v2_rows_projected(raw, &five, &[nm.as_str()]);
+ let el = t.elapsed();
+ if el.as_millis() > 5 {
+ println!("col {nm} {:?}", el);
+ }
+ }
+ println!("ncols {}", names.len());
+ }
+}
diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rs
index 677d40eab..225926429 100644
--- a/engine/crates/xerj-storage/src/stored_codec.rs
+++ b/engine/crates/xerj-storage/src/stored_codec.rs
@@ -86,8 +86,9 @@ use std::io::{Cursor, Read, Write};
mod row_hydration;
pub use row_hydration::{
- decode_stored_v2_rows, decode_stored_v2_rows_controlled, decode_stored_v2_rows_projected,
- decode_stored_v2_rows_projected_controlled, StoredV2HydratedRow, StoredV2RowHydrationResult,
+ decode_stored_v2_ids, decode_stored_v2_rows, decode_stored_v2_rows_controlled,
+ decode_stored_v2_rows_projected, decode_stored_v2_rows_projected_controlled,
+ decode_stored_v2_rows_projected_parallel, StoredV2HydratedRow, StoredV2RowHydrationResult,
StoredV2RowHydrationStats,
};
diff --git a/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs b/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs
index d52a1a316..1118d3c4e 100644
--- a/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs
+++ b/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs
@@ -80,6 +80,7 @@ pub(super) fn decode_v2_identity_columns_controlled(
&mut stats,
&mut dict_ids,
checkpoint,
+ false,
)?;
if result.is_ok()
&& decode_cancelled(
@@ -110,6 +111,28 @@ pub(super) fn decode_v2_identity_columns_controlled(
))))
}
+/// `_id` of every row of a columnar (ZBS2/ZBS3) stored section, in row
+/// order, decoded from the identity columns alone: no `_source` column is
+/// read. `Ok(None)` for a non-columnar section or an identity shape that
+/// cannot be read selectively; the caller decodes the whole section instead.
+pub fn decode_stored_v2_ids(bytes: &[u8]) -> Result<Option<Vec<serde_json::Value>>> {
+ let bytes = if bytes.len() >= 4 && &bytes[..4] == STORED_V3_MAGIC {
+ split_stored_v3(bytes)?.1
+ } else {
+ bytes
+ };
+ if !is_columnar_stored_magic(bytes) {
+ return Ok(None);
+ }
+ let mut no_cancel = |_| std::ops::ControlFlow::Continue(());
+ match decode_v2_identity_columns_controlled(bytes, &mut no_cancel)? {
+ StoredDecodeRun::Complete(Some((num_docs, ids, _))) if ids.len() == num_docs => {
+ Ok(Some(ids))
+ }
+ StoredDecodeRun::Complete(_) | StoredDecodeRun::Cancelled => Ok(None),
+ }
+}
+
struct SelectedRowsVisitor<'a> {
selected: &'a [usize],
expected_rows: usize,
@@ -195,6 +218,38 @@ fn select_json_rows<R: std::io::Read>(
Ok(Ok(values))
}
+/// [`select_json_rows`] over a column already decompressed in memory. The
+/// slice parser skips unselected rows several times faster than the
+/// byte-at-a-time reader parser; validation is the same.
+fn select_json_rows_slice(
+ bytes: &[u8],
+ selected: &[usize],
+ expected_rows: usize,
+ context: &str,
+ column_index: usize,
+ checkpoint: &mut dyn FnMut(StoredDecodeCheckpoint) -> std::ops::ControlFlow<()>,
+) -> Result<std::result::Result<Vec<serde_json::Value>, ()>> {
+ let cancelled = Cell::new(false);
+ let mut deserializer = serde_json::Deserializer::from_slice(bytes);
+ let values = match deserializer.deserialize_seq(SelectedRowsVisitor {
+ selected,
+ expected_rows,
+ column_index,
+ checkpoint,
+ cancelled: &cancelled,
+ }) {
+ Ok(values) => values,
+ Err(_) if cancelled.get() => return Ok(Err(())),
+ Err(error) => {
+ return Err(StorageError::Other(anyhow::anyhow!("{context}: {error}")));
+ }
+ };
+ deserializer
+ .end()
+ .map_err(|error| StorageError::Other(anyhow::anyhow!("{context}: {error}")))?;
+ Ok(Ok(values))
+}
+
fn unpack_id(packed: &[u8], bit_width: u8, ordinal: usize) -> Option<u32> {
let width = bit_width as usize;
if !(1..=32).contains(&width) {
@@ -489,6 +544,11 @@ fn repeat_constant_rows_controlled(
Ok(Ok(values))
}
+/// `in_memory`: decompress a JSON column whole and parse it as a slice
+/// (faster) instead of streaming it. Only the non-cancellable parallel
+/// hydration asks for it, so cancellation still never waits on a whole
+/// column's decompression.
+#[allow(clippy::too_many_arguments)]
fn select_column_rows(
column_index: usize,
directory: &V2Directory<'_>,
@@ -496,9 +556,28 @@ fn select_column_rows(
stats: &mut StoredV2RowHydrationStats,
dict_ids: &mut HashMap<usize, Vec<u32>>,
checkpoint: &mut dyn FnMut(StoredDecodeCheckpoint) -> std::ops::ControlFlow<()>,
+ in_memory: bool,
) -> Result<std::result::Result<Vec<serde_json::Value>, Option<String>>> {
let column = &directory.columns[column_index];
let values = match column.codec {
+ ColCodec::RawJson if in_memory => {
+ stats.encoded_rows_visited += directory.num_docs;
+ let frame = decode_raw_payload(column.payload)?;
+ let raw = zstd::stream::decode_all(frame)
+ .map_err(|error| StorageError::Other(anyhow::anyhow!("raw zstd: {error}")))?;
+ stats.decompressed_buffer_bytes += raw.len();
+ match select_json_rows_slice(
+ &raw,
+ selected,
+ directory.num_docs,
+ "raw json",
+ column_index,
+ checkpoint,
+ )? {
+ Ok(values) => values,
+ Err(()) => return Ok(Err(None)),
+ }
+ }
ColCodec::RawJson => {
stats.encoded_rows_visited += directory.num_docs;
let frame = decode_raw_payload(column.payload)?;
@@ -590,6 +669,7 @@ fn select_column_rows(
stats,
dict_ids,
checkpoint,
+ in_memory,
);
}
ColCodec::CrossDep => {
@@ -699,6 +779,38 @@ pub fn decode_stored_v2_rows_projected_controlled<F>(
bytes: &[u8],
ordinals: &[usize],
source_fields: Option<&[&str]>,
+ checkpoint: F,
+) -> Result<StoredDecodeRun<StoredV2RowHydrationResult>>
+where
+ F: FnMut(StoredDecodeCheckpoint) -> std::ops::ControlFlow<()>,
+{
+ hydrate_v2_rows(bytes, ordinals, source_fields, false, checkpoint)
+}
+
+/// Like [`decode_stored_v2_rows_projected`] (`None` = every field), but
+/// decodes the wanted columns concurrently on the rayon pool. Each column is
+/// an independent stream, so a cold top-k hydration that must walk every
+/// column of a segment for a handful of rows costs roughly its largest
+/// column instead of the sum of all of them. Output is identical to the
+/// serial decoder.
+pub fn decode_stored_v2_rows_projected_parallel(
+ bytes: &[u8],
+ ordinals: &[usize],
+ source_fields: Option<&[&str]>,
+) -> Result<StoredV2RowHydrationResult> {
+ match hydrate_v2_rows(bytes, ordinals, source_fields, true, |_| {
+ std::ops::ControlFlow::Continue(())
+ })? {
+ StoredDecodeRun::Complete(result) => Ok(result),
+ StoredDecodeRun::Cancelled => unreachable!("non-cancelling parallel wrapper"),
+ }
+}
+
+fn hydrate_v2_rows<F>(
+ bytes: &[u8],
+ ordinals: &[usize],
+ source_fields: Option<&[&str]>,
+ parallel: bool,
mut checkpoint: F,
) -> Result<StoredDecodeRun<StoredV2RowHydrationResult>>
where
@@ -753,14 +865,57 @@ where
let mut columns: Vec<Option<Vec<serde_json::Value>>> =
(0..directory.columns.len()).map(|_| None).collect();
let mut dict_ids = HashMap::new();
- for index in 0..directory.columns.len() {
- let column = &directory.columns[index];
- let wanted = index == id_index
- || index == seq_index
- || requested
- .as_ref()
- .is_none_or(|fields| fields.contains(column.name));
- if !wanted {
+ let wanted_columns: Vec<usize> = (0..directory.columns.len())
+ .filter(|&index| {
+ index == id_index
+ || index == seq_index
+ || requested
+ .as_ref()
+ .is_none_or(|fields| fields.contains(directory.columns[index].name))
+ })
+ .collect();
+ if parallel && wanted_columns.len() > 1 {
+ use rayon::prelude::*;
+ // Each task gets its own stats and dictionary-id scratch: a CROSS_DEP
+ // column re-derives its DICT source ids locally, which is the same
+ // work the serial path does once and shares.
+ let decoded: Vec<_> = wanted_columns
+ .par_iter()
+ .map(|&index| {
+ let mut local_stats = StoredV2RowHydrationStats::default();
+ let mut local_dict_ids = HashMap::new();
+ let mut noop = |_: StoredDecodeCheckpoint| std::ops::ControlFlow::Continue(());
+ select_column_rows(
+ index,
+ &directory,
+ &selected,
+ &mut local_stats,
+ &mut local_dict_ids,
+ &mut noop,
+ true,
+ )
+ .map(|values| (index, values, local_stats))
+ })
+ .collect();
+ for result in decoded {
+ let (index, values, local_stats) = result?;
+ stats.encoded_rows_visited += local_stats.encoded_rows_visited;
+ stats.selected_json_values += local_stats.selected_json_values;
+ stats.selected_dictionary_entries += local_stats.selected_dictionary_entries;
+ stats.decompressed_buffer_bytes += local_stats.decompressed_buffer_bytes;
+ match values {
+ Ok(values) => columns[index] = Some(values),
+ Err(Some(column)) => {
+ return Ok(StoredDecodeRun::Complete(
+ StoredV2RowHydrationResult::UnsupportedDependencyShape { column },
+ ));
+ }
+ Err(None) => return Ok(StoredDecodeRun::Cancelled),
+ }
+ }
+ }
+ for index in wanted_columns {
+ if columns[index].is_some() {
continue;
}
if decode_cancelled(&mut checkpoint, StoredDecodePhase::BeforeColumn, index, 0) {
@@ -773,6 +928,7 @@ where
&mut stats,
&mut dict_ids,
&mut checkpoint,
+ false,
)? {
Ok(values) => columns[index] = Some(values),
Err(Some(column)) => {
@@ -954,6 +1110,76 @@ mod tests {
assert!(decode_stored_v2_rows(&bad, &[0]).is_err());
}
+ /// The parallel hydration parses each JSON column as an in-memory slice;
+ /// its rows must equal the streaming path's, and it must still reject a
+ /// column with trailing JSON or a row count other than the directory's.
+ #[test]
+ fn parallel_hydration_matches_streaming_and_validates_columns() {
+ for codec in [ColCodec::RawJson, ColCodec::Lz4Json] {
+ let encoded = fixture(300, codec);
+ let ordinals = [299, 0, 7, 7, 128];
+ for fields in [None, Some(&["body"][..]), Some(&[][..])] {
+ let serial = match fields {
+ Some(fields) => decode_stored_v2_rows_projected(&encoded, &ordinals, fields),
+ None => decode_stored_v2_rows(&encoded, &ordinals),
+ }
+ .unwrap();
+ let parallel =
+ decode_stored_v2_rows_projected_parallel(&encoded, &ordinals, fields).unwrap();
+ let (
+ StoredV2RowHydrationResult::Hydrated { rows: a, .. },
+ StoredV2RowHydrationResult::Hydrated { rows: b, .. },
+ ) = (serial, parallel)
+ else {
+ panic!("expected hydration")
+ };
+ assert_eq!(a.len(), 4);
+ assert_eq!(a.len(), b.len());
+ for (a, b) in a.iter().zip(&b) {
+ assert_eq!(
+ (a.ordinal, &a.id, &a.seq_no, &a.source),
+ (b.ordinal, &b.id, &b.seq_no, &b.source)
+ );
+ }
+ }
+ }
+ for tail in [&b"[]"[..], &b""[..]] {
+ let mut values = serde_json::to_vec(&vec![json!("x"); 4]).unwrap();
+ values.extend_from_slice(tail);
+ let rows = if tail.is_empty() { 3 } else { 4 };
+ let bad = assemble(
+ rows,
+ &[
+ ("__id", ColCodec::RawJson, raw(&vec![json!("x"); rows])),
+ ("__seq_no", ColCodec::RawJson, raw(&vec![json!(0); rows])),
+ (
+ "bad",
+ ColCodec::RawJson,
+ zstd::encode_all(Cursor::new(values), 3).unwrap(),
+ ),
+ ("other", ColCodec::RawJson, raw(&vec![json!(1); rows])),
+ ],
+ );
+ assert!(decode_stored_v2_rows_projected_parallel(&bad, &[0], None).is_err());
+ }
+ }
+
+ /// The identity-only id list equals the `_id`s of the full decode.
+ #[test]
+ fn identity_ids_match_the_full_decode() {
+ for codec in [ColCodec::RawJson, ColCodec::Lz4Json] {
+ let encoded = fixture(200, codec);
+ let ids = decode_stored_v2_ids(&encoded).unwrap().unwrap();
+ let full: Vec<serde_json::Value> =
+ serde_json::from_slice(&super::super::decode_stored(&encoded).unwrap()).unwrap();
+ assert_eq!(ids.len(), full.len());
+ for (id, doc) in ids.iter().zip(&full) {
+ assert_eq!(id, &doc["_id"]);
+ }
+ }
+ assert!(decode_stored_v2_ids(b"not columnar").unwrap().is_none());
+ }
+
#[test]
fn dictionary_nulls_boundaries_and_corruption_are_validated() {
let values = vec![json!("a"), json!("b"), json!("c")];