AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Cold FTS top-k: walk on identities, hydrate only the page's winners (#2454)
xerj-org767c377e97be
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 649720a29..40390de23 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -1968,6 +1968,149 @@ mod flush_publication_recovery_tests {schema}+ /// #2422: a broad query (most rows of a segment match) on a cold+ /// segment used to decode the whole stored section inline, because the+ /// first segment a page meets has no score floor and every matching row+ /// was a candidate. The walk must read only identities and hydrate the+ /// page's winners, with or without a `_source` projection, and a walk+ /// that rejects thousands of rows (here: the best ones are deleted) must+ /// still see every row, so every answer equals the warm answer.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn cold_broad_topk_reads_identities_and_hydrates_only_winners() {+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ engine+ .create_index("broad-cold", topk_projection_schema())+ .unwrap();+ let idx = engine.get_index("broad-cold").unwrap();+ idx.abort_background_tasks();+ for i in 0..4096u64 {+ let body = if i % 2 == 0 {+ format!("needle stack report {i} needle")+ } else {+ format!("hay stack {i}")+ };+ idx.index_document(+ Some(format!("doc-{i}")),+ json!({"title": format!("title {i}"), "body": body, "blob": "x"}),+ )+ .await+ .unwrap();+ }+ idx.flush().await.unwrap();+ // Delete the best-scoring needle rows, more than the row window+ // holds, so the walk rejects every row in the window and must read+ // the live needle rows past it.+ for i in (0..4000u64).step_by(2) {+ assert!(idx.delete_document(&format!("doc-{i}")).await.unwrap());+ }+ idx.flush().await.unwrap();+ drop(idx);+ drop(engine);++ let reopen = || {+ let mut c = Config::default();+ c.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(c).unwrap();+ let idx = engine.get_index("broad-cold").unwrap();+ idx.abort_background_tasks();+ // Hold the background warm back so each query is observed alone.+ let snapshot = idx.store.snapshot();+ let segs: Vec<_> = snapshot.segments.clone();+ drop(snapshot);+ assert!(+ segs.iter().any(|s| s.doc_count > 1024),+ "precondition: a segment large enough to exceed the row window"+ );+ for s in &segs {+ idx.cold_stored_warms.insert(s.id.to_string(), ());+ }+ (engine, idx, segs)+ };+ let all_cold = |idx: &Index, segs: &[xerj_storage::segment::SegmentMeta]| {+ segs.iter().all(|s| {+ !idx.stored_slices_cache.contains_key(&s.id)+ && !idx.decoded_stored_cache.contains_key(&s.id)+ })+ };+ let key = |r: &xerj_query::executor::SearchResult| -> Vec<(String, u32, Value)> {+ r.hits+ .iter()+ .map(|h| (h.id.clone(), h.score.to_bits(), h.source.clone()))+ .collect()+ };+ let request = |q: &str, projected: bool| {+ let mut body = json!({+ "query": {"match": {"body": q}},+ "size": 5,+ });+ if projected {+ body["_source"] = json!(["title"]);+ }+ xerj_query::parse_request(&body).unwrap()+ };+ let mut requests = Vec::new();+ for projected in [true, false] {+ // Every document matches; the live hay rows score highest.+ requests.push(request("hay stack", projected));+ // Matches the needle rows only. The best ~2000 of them are+ // deleted, so the walk rejects thousands of rows first.+ requests.push(request("needle report", projected));+ }++ let (engine, idx, segs) = reopen();+ let mut cold = Vec::new();+ for r in &requests {+ cold.push(idx.search(r).await.unwrap());+ assert!(+ all_cold(&idx, &segs),+ "a 5-hit query must not decode the whole stored section"+ );+ }+ assert!(idx.cold_row_hydrations.load(Ordering::Relaxed) > 0);+ drop(idx);+ drop(engine);++ let (engine, idx, segs) = reopen();+ // Warm every segment that can be warmed; the large one (the walk+ // under test) must be, so the reference takes the full-decode path.+ for s in &segs {+ let warmed = idx.stored_slices_for(&s.id, s.doc_count).is_some();+ assert!(warmed || s.doc_count <= 1024, "segment {} must warm", s.id);+ }+ for (r, cold) in requests.iter().zip(&cold) {+ let warm = idx.search(r).await.unwrap();+ assert_eq!(cold.hits.len(), 5);+ assert_eq!(key(cold), key(&warm));+ assert_eq!(cold.total.value, warm.total.value);+ assert_eq!(+ cold.hits+ .iter()+ .filter(|h| h.source["title"].is_string())+ .count(),+ 5+ );+ }+ for (i, h) in [&cold[1], &cold[3]]+ .iter()+ .flat_map(|r| r.hits.iter())+ .enumerate()+ {+ let n: u64 = h.id.trim_start_matches("doc-").parse().unwrap();+ assert!(+ n >= 4000 && n.is_multiple_of(2),+ "live needle rows only: {i} {}",+ h.id+ );+ }+ // Unprojected hits carry the whole source.+ assert!(cold[2].hits.iter().all(|h| h.source["body"].is_string()));+ drop(idx);+ drop(engine);+ }+/// #2422/#2423: on a cold segment (after a restart, or evicted under the/// cache budget) a top-k query that names its `_source` fields must read/// only the winning rows and only the needed columns. Before the fix the@@ -17401,12 +17544,14 @@ impl Index {keep: Option<&[String]>,) -> Option<HashMap<u32, Value>> {use xerj_storage::stored_codec::{- decode_stored_v2_rows, decode_stored_v2_rows_projected, stored_v2_source_column_names,+ decode_stored_v2_rows_projected_parallel, stored_v2_source_column_names,StoredV2RowHydrationResult,};let reader = self.store.open_segment_arc(segment_id).ok()?;let bytes = reader.section(SectionType::Stored).ok()??;let ordinals: Vec<usize> = positions.iter().map(|&p| p as usize).collect();+ // Columns decode concurrently: a handful of rows still streams every+ // wanted column, so the serial sum was the cost of a cold page.let hydrated = match keep {Some(keep) => {let names = stored_v2_source_column_names(bytes).ok()??;@@ -17415,9 +17560,9 @@ impl Index {.map(String::as_str).filter(|column| row_projection_keeps_column(keep, column)).collect();- decode_stored_v2_rows_projected(bytes, &ordinals, &include).ok()?+ decode_stored_v2_rows_projected_parallel(bytes, &ordinals, Some(&include)).ok()?}- None => decode_stored_v2_rows(bytes, &ordinals).ok()?,+ None => decode_stored_v2_rows_projected_parallel(bytes, &ordinals, None).ok()?,};let StoredV2RowHydrationResult::Hydrated { rows, .. } = hydrated else {return None;@@ -17439,6 +17584,138 @@ impl Index {Some(out)}+ /// `_id` per row for `positions` of a cold segment, decoded from the+ /// identity columns alone (no `_source` column is read). `None` (legacy+ /// layout, open/decode error, a missing row) sends the caller back to+ /// the full decode.+ fn cold_fts_row_ids(+ &self,+ segment_id: &str,+ positions: &[u32],+ ) -> Option<HashMap<u32, String>> {+ use xerj_storage::stored_codec::{+ decode_stored_v2_rows_projected, StoredV2RowHydrationResult,+ };+ let reader = self.store.open_segment_arc(segment_id).ok()?;+ let bytes = reader.section(SectionType::Stored).ok()??;+ let ordinals: Vec<usize> = positions.iter().map(|&p| p as usize).collect();+ let StoredV2RowHydrationResult::Hydrated { rows, .. } =+ decode_stored_v2_rows_projected(bytes, &ordinals, &[]).ok()?+ else {+ return None;+ };+ let mut out: HashMap<u32, String> = HashMap::with_capacity(rows.len());+ for row in rows {+ out.insert(+ u32::try_from(row.ordinal).ok()?,+ row.id.as_str()?.to_string(),+ );+ }+ if positions.iter().any(|p| !out.contains_key(p)) {+ return None;+ }+ Some(out)+ }++ /// Fill the `_source` of the cold FTS hits the segment loop kept with+ /// identity only (`deferred`: `_id` -> (segment, row, doc count)). Only+ /// the page's winners are read, row-selectively and per segment in+ /// parallel. A segment whose rows cannot be hydrated that way falls back+ /// to the full decode, as the walk itself used to; a hit whose document+ /// cannot be read at all is dropped, as the walk skipped an unreadable+ /// row. Projected hits are recorded in `row_projected` so the TF-IDF+ /// fallback can restore their whole source.+ fn hydrate_deferred_cold_sources(+ &self,+ hits: Vec<Hit>,+ deferred: &HashMap<String, (String, u32, u64)>,+ keep: Option<&[String]>,+ row_projected: &mut HashMap<String, (String, u32)>,+ ) -> Vec<Hit> {+ use rayon::prelude::*;+ let mut by_segment: HashMap<&str, (Vec<u32>, u64)> = HashMap::new();+ for hit in &hits {+ if let Some((seg, pos, doc_count)) = deferred.get(&hit.id) {+ by_segment+ .entry(seg.as_str())+ .or_insert_with(|| (Vec::new(), *doc_count))+ .0+ .push(*pos);+ }+ }+ if by_segment.is_empty() {+ return hits;+ }+ let segments: Vec<(&str, Vec<u32>, u64)> = by_segment+ .into_iter()+ .map(|(seg, (positions, doc_count))| (seg, positions, doc_count))+ .collect();+ // (segment, docs by row, projected?)+ let loaded: Vec<(&str, HashMap<u32, Value>, bool)> = segments+ .par_iter()+ .map(|(seg, positions, doc_count)| {+ if let Some(rows) = self.hydrate_fts_rows_cold(seg, positions, keep) {+ return (*seg, rows, keep.is_some());+ }+ let mut docs: HashMap<u32, Value> = HashMap::with_capacity(positions.len());+ if let Some(slices) = self.stored_slices_for(seg, *doc_count) {+ for &pos in positions {+ let doc = slices+ .offsets+ .get(pos as usize)+ .and_then(|&(start, end)| {+ slices.bytes.get(start as usize..end as usize)+ })+ .and_then(|slice| serde_json::from_slice::<Value>(slice).ok());+ if let Some(doc) = doc {+ docs.insert(pos, doc);+ }+ }+ } else if let Some(all) = self.store.open_segment_arc(seg).ok().and_then(|reader| {+ match reader.section(SectionType::Stored) {+ Ok(Some(raw)) => xerj_storage::stored_codec::decode_stored(raw)+ .ok()+ .and_then(|b| serde_json::from_slice::<Vec<Value>>(&b).ok()),+ _ => None,+ }+ }) {+ for &pos in positions {+ if let Some(doc) = all.get(pos as usize) {+ docs.insert(pos, doc.clone());+ }+ }+ }+ (*seg, docs, false)+ })+ .collect();+ let mut docs_by_segment: HashMap<&str, (HashMap<u32, Value>, bool)> = loaded+ .into_iter()+ .map(|(seg, docs, projected)| (seg, (docs, projected)))+ .collect();+ let mut out = Vec::with_capacity(hits.len());+ for mut hit in hits {+ let Some((seg, pos, _)) = deferred.get(&hit.id) else {+ out.push(hit);+ continue;+ };+ let Some((docs, projected)) = docs_by_segment.get_mut(seg.as_str()) else {+ continue;+ };+ let Some(mut doc) = docs.remove(pos) else {+ continue;+ };+ hit.source = doc+ .get_mut("_source")+ .map(Value::take)+ .unwrap_or(Value::Null);+ if *projected {+ row_projected.insert(hit.id.clone(), (seg.clone(), *pos));+ }+ out.push(hit);+ }+ out+ }+#[allow(clippy::too_many_arguments)]async fn run_knn_brute_force_with_deadline(&self,@@ -24184,6 +24461,9 @@ impl Index {// position), so the one post-page reader that looks at the WHOLE// document (the TF-IDF fallback below) can restore it.let mut row_projected: HashMap<String, (String, u32)> = HashMap::new();+ // Cold score-ordered FTS hits whose `_source` is read only once the+ // segment loop has settled the page: `_id` -> (segment, row).+ let mut deferred_sources: HashMap<String, (String, u32, u64)> = HashMap::new();let materialisation_limit: usize = if mem_doc_count == 0&& !deletes_present&& size > 0@@ -25159,18 +25439,35 @@ impl Index {// threshold only tightens, so this is a// superset) — and only the columns the// request can observe.+ // #2422: set when the cold row window+ // below was cut short; the walk then+ // falls back to the full decode for any+ // row past it instead of skipping it.+ let mut cold_window_truncated = false;+ // Cold score-ordered top-k: the walk+ // reads only each candidate's `_id`+ // (version-map check, dedup, page key),+ // so decode the identity columns for+ // every candidate and defer `_source`+ // until the segment loop has picked the+ // page's winners (see the hydration+ // after the loop). The first segment a+ // page meets has no score floor, so+ // without this a broad query decoded the+ // whole stored section inline to show+ // five hits (~1.45 s on a 131 MB+ // section).+ let mut cold_ids: Option<HashMap<u32, String>> = None;let cold_rows: Option<HashMap<u32, Value>> = if self.stored_slices_cache.contains_key(seg_id.as_str())|| self.decoded_stored_cache.contains_key(seg_id.as_str()){None- } else {- let entry_worst = if sort_topk.is_none() {- page_worst.as_ref().map(|worst| worst.0)- } else {- None- };+ } else if sort_topk.is_none() {+ let entry_worst = page_worst.as_ref().map(|worst| worst.0);+ // A superset of the rows the walk can+ // visit: the threshold only tightens.let candidates: Vec<u32> = seg_hits.iter().take_while(|sh| {@@ -25178,8 +25475,33 @@ impl Index {}).map(|sh| sh.doc_id).collect();- let bounded = candidates.len()- <= materialisation_limit.saturating_mul(4).max(256);+ if !candidates.is_empty() {+ cold_ids =+ self.cold_fts_row_ids(seg_id.as_str(), &candidates);+ }+ if cold_ids.is_some() {+ self.cold_row_hydrations+ .fetch_add(1, Ordering::Relaxed);+ self.schedule_cold_stored_warm(+ seg_id.as_str(),+ meta.doc_count,+ );+ }+ None+ } else {+ // A field sort offers every match to+ // the top-N heap, so its candidates+ // are all matches: decode a bounded+ // score-ordered window and read any+ // row past it from the full decode.+ let mut candidates: Vec<u32> =+ seg_hits.iter().map(|sh| sh.doc_id).collect();+ let window =+ materialisation_limit.saturating_mul(4).max(256);+ if candidates.len() > window {+ candidates.truncate(window);+ cold_window_truncated = true;+ }// Only with a column projection: the// stored columns are whole-stream// zstd, so an unprojected row decode@@ -25192,7 +25514,7 @@ impl Index {// 2.3 s -> 6.6 s). The full decode// below caches for the next query.let worth_it = fts_row_keep.is_some();- if bounded && worth_it && !candidates.is_empty() {+ if worth_it && !candidates.is_empty() {let rows = self.hydrate_fts_rows_cold(seg_id.as_str(),&candidates,@@ -25211,7 +25533,7 @@ impl Index {None}};- let slices_opt = if cold_rows.is_some() {+ let slices_opt = if cold_rows.is_some() || cold_ids.is_some() {None} else {self.stored_slices_for(seg_id.as_str(), meta.doc_count)@@ -25227,6 +25549,7 @@ impl Index {// real corruption.let whole_fallback: Option<Vec<Value>> = if slices_opt.is_some()|| cold_rows.is_some()+ || cold_ids.is_some(){None} else {@@ -25250,6 +25573,7 @@ impl Index {}};if cold_rows.is_some()+ || cold_ids.is_some()|| slices_opt.is_some()|| whole_fallback.is_some(){@@ -25258,9 +25582,43 @@ impl Index {// slice arm parses one object on demand;// the fallback clones from the fully// decoded Vec.+ let past_window: std::cell::OnceCell<+ Option<Resident<StoredSlices>>,+ > = std::cell::OnceCell::new();let fetch_doc = |pos: u32| -> Option<Value> {+ if let Some(ids) = cold_ids.as_ref() {+ // Identity only: `_source` is+ // hydrated for the winners+ // after the segment loop.+ let id = ids.get(&pos)?;+ let mut doc = serde_json::Map::with_capacity(1);+ doc.insert(+ "_id".to_string(),+ Value::String(id.clone()),+ );+ return Some(Value::Object(doc));+ }if let Some(rows) = cold_rows.as_ref() {- rows.get(&pos).cloned()+ if let Some(doc) = rows.get(&pos) {+ return Some(doc.clone());+ }+ if !cold_window_truncated {+ return None;+ }+ let slices = past_window+ .get_or_init(|| {+ self.stored_slices_for(+ seg_id.as_str(),+ meta.doc_count,+ )+ })+ .as_deref()?;+ let &(start, end) =+ slices.offsets.get(pos as usize)?;+ let slice = slices+ .bytes+ .get(start as usize..end as usize)?;+ serde_json::from_slice::<Value>(slice).ok()} else if let Some(slices) = slices_opt.as_deref() {let &(start, end) =slices.offsets.get(pos as usize)?;@@ -25399,7 +25757,13 @@ impl Index {let source =doc.get("_source").cloned().unwrap_or(Value::Null);seen_ids.insert(id.clone());- if cold_rows.is_some() && fts_row_keep.is_some() {+ if cold_ids.is_some() {+ deferred_sources.insert(+ id.clone(),+ (seg_id.to_string(), sh.doc_id, meta.doc_count),+ );+ } else if cold_rows.is_some() && fts_row_keep.is_some()+ {row_projected.insert(id.clone(),(seg_id.to_string(), sh.doc_id),@@ -26119,6 +26483,14 @@ impl Index {}}}+ if !deferred_sources.is_empty() {+ all_hits = self.hydrate_deferred_cold_sources(+ all_hits,+ &deferred_sources,+ fts_row_keep.as_deref(),+ &mut row_projected,+ );+ }drop(snap);// F1: did the bounded collector fill to the cap? If so the stored-doc@@ -33841,6 +34213,16 @@ impl Index {))}+ /// `_id` of every row of a segment's columnar stored section, read from+ /// its identity columns only. `None` (legacy layout, read error, a row+ /// count other than `expect_docs`) sends the caller to the full decode.+ fn cold_segment_ids(&self, seg_id: &str, expect_docs: u64) -> Option<Vec<Value>> {+ let reader = self.store.open_segment_arc(seg_id).ok()?;+ let bytes = reader.section(SectionType::Stored).ok()??;+ let ids = xerj_storage::stored_codec::decode_stored_v2_ids(bytes).ok()??;+ (ids.len() as u64 == expect_docs).then_some(ids)+ }+/// Per-segment GHOST-position bitmap (b7 DEFECT 1c): bit `pos` set ⇔/// the stored doc at `pos` is tombstoned or superseded — its live/// version resides in another segment or the memtable per the version@@ -33861,21 +34243,39 @@ impl Index {return Some(Arc::clone(&entry.value().1));}}- let slices = self.stored_slices_for(seg_id, expect_docs)?;- let mut bm = vec![0u64; slices.offsets.len().div_ceil(64)];- for (pos, &(start, end)) in slices.offsets.iter().enumerate() {- let slice = slices.bytes.get(start as usize..end as usize)?;- let doc: Value = serde_json::from_slice(slice).ok()?;- let id = doc.get("_id").and_then(Value::as_str).unwrap_or("");- let ghost = match self.store.version_map.get(id) {- Some(ver) => ver.deleted || ver.segment_id.as_ref() != seg_id,- // Not in the version map → nothing has ever superseded it.- None => false,- };- if ghost {- bm[pos / 64] |= 1u64 << (pos % 64);+ let is_ghost = |id: &str| match self.store.version_map.get(id) {+ Some(ver) => ver.deleted || ver.segment_id.as_ref() != seg_id,+ // Not in the version map → nothing has ever superseded it.+ None => false,+ };+ // A cold segment reads its `_id` column alone: decoding (and+ // caching) the whole stored section here made the first query+ // under deletes pay for every row's `_source`.+ let cold = !self.stored_slices_cache.contains_key(seg_id)+ && !self.decoded_stored_cache.contains_key(seg_id);+ let cold_ids = cold+ .then(|| self.cold_segment_ids(seg_id, expect_docs))+ .flatten();+ let bm = if let Some(ids) = cold_ids {+ let mut bm = vec![0u64; ids.len().div_ceil(64)];+ for (pos, id) in ids.iter().enumerate() {+ if is_ghost(id.as_str().unwrap_or("")) {+ bm[pos / 64] |= 1u64 << (pos % 64);+ }}- }+ bm+ } else {+ let slices = self.stored_slices_for(seg_id, expect_docs)?;+ let mut bm = vec![0u64; slices.offsets.len().div_ceil(64)];+ for (pos, &(start, end)) in slices.offsets.iter().enumerate() {+ let slice = slices.bytes.get(start as usize..end as usize)?;+ let doc: Value = serde_json::from_slice(slice).ok()?;+ if is_ghost(doc.get("_id").and_then(Value::as_str).unwrap_or("")) {+ bm[pos / 64] |= 1u64 << (pos % 64);+ }+ }+ bm+ };let bm = Arc::new(bm);// Bounded like `shortcut_count_cache`: wholesale clear — entries// are per-segment (not query-shaped) so the cap is generous.diff --git a/engine/crates/xerj-storage/examples/zz_cold_probe.rs b/engine/crates/xerj-storage/examples/zz_cold_probe.rsnew file mode 100644index 000000000..d671c912d--- /dev/null+++ b/engine/crates/xerj-storage/examples/zz_cold_probe.rs@@ -0,0 +1,56 @@+use std::time::Instant;+use xerj_storage::segment::{SectionType, SegmentReader};+use xerj_storage::stored_codec::*;+fn main() {+ let path = std::env::args().nth(1).unwrap();+ let r = SegmentReader::open(&path).unwrap();+ let raw = r.section(SectionType::Stored).unwrap().unwrap();+ println!("stored section {} bytes", raw.len());+ let t = Instant::now();+ let full = decode_stored(raw).unwrap();+ println!("decode_stored {:?} -> {} bytes", t.elapsed(), full.len());+ let n: usize = std::env::args()+ .nth(2)+ .map(|s| s.parse().unwrap())+ .unwrap_or(30000);+ let all: Vec<usize> = (0..n).collect();+ let t = Instant::now();+ let res = decode_stored_v2_rows_projected(raw, &all, &[]);+ match res {+ Ok(StoredV2RowHydrationResult::Hydrated { rows, .. }) => {+ println!("identity rows {} {:?}", rows.len(), t.elapsed())+ }+ Ok(_) => println!("other"),+ Err(e) => println!("err {e}"),+ }+ let five: Vec<usize> = vec![1, 100, 2000, 7000, n - 1];+ let t = Instant::now();+ let res = decode_stored_v2_rows(raw, &five);+ match res {+ Ok(StoredV2RowHydrationResult::Hydrated { rows, .. }) => {+ println!("5 full rows {} {:?}", rows.len(), t.elapsed())+ }+ Ok(_) => println!("other"),+ Err(e) => println!("err {e}"),+ }+ let t = Instant::now();+ let res = decode_stored_v2_rows_projected_parallel(raw, &five, None);+ match res {+ Ok(StoredV2RowHydrationResult::Hydrated { rows, .. }) => {+ println!("5 full rows PAR {} {:?}", rows.len(), t.elapsed())+ }+ Ok(_) => println!("other"),+ Err(e) => println!("err {e}"),+ }+ if let Ok(Some(names)) = stored_v2_source_column_names(raw) {+ for nm in &names {+ let t = Instant::now();+ let _ = decode_stored_v2_rows_projected(raw, &five, &[nm.as_str()]);+ let el = t.elapsed();+ if el.as_millis() > 5 {+ println!("col {nm} {:?}", el);+ }+ }+ println!("ncols {}", names.len());+ }+}diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rsindex 677d40eab..225926429 100644--- a/engine/crates/xerj-storage/src/stored_codec.rs+++ b/engine/crates/xerj-storage/src/stored_codec.rs@@ -86,8 +86,9 @@ use std::io::{Cursor, Read, Write};mod row_hydration;pub use row_hydration::{- decode_stored_v2_rows, decode_stored_v2_rows_controlled, decode_stored_v2_rows_projected,- decode_stored_v2_rows_projected_controlled, StoredV2HydratedRow, StoredV2RowHydrationResult,+ decode_stored_v2_ids, decode_stored_v2_rows, decode_stored_v2_rows_controlled,+ decode_stored_v2_rows_projected, decode_stored_v2_rows_projected_controlled,+ decode_stored_v2_rows_projected_parallel, StoredV2HydratedRow, StoredV2RowHydrationResult,StoredV2RowHydrationStats,};diff --git a/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs b/engine/crates/xerj-storage/src/stored_codec/row_hydration.rsindex d52a1a316..1118d3c4e 100644--- a/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs+++ b/engine/crates/xerj-storage/src/stored_codec/row_hydration.rs@@ -80,6 +80,7 @@ pub(super) fn decode_v2_identity_columns_controlled(&mut stats,&mut dict_ids,checkpoint,+ false,)?;if result.is_ok()&& decode_cancelled(@@ -110,6 +111,28 @@ pub(super) fn decode_v2_identity_columns_controlled())))}+/// `_id` of every row of a columnar (ZBS2/ZBS3) stored section, in row+/// order, decoded from the identity columns alone: no `_source` column is+/// read. `Ok(None)` for a non-columnar section or an identity shape that+/// cannot be read selectively; the caller decodes the whole section instead.+pub fn decode_stored_v2_ids(bytes: &[u8]) -> Result<Option<Vec<serde_json::Value>>> {+ let bytes = if bytes.len() >= 4 && &bytes[..4] == STORED_V3_MAGIC {+ split_stored_v3(bytes)?.1+ } else {+ bytes+ };+ if !is_columnar_stored_magic(bytes) {+ return Ok(None);+ }+ let mut no_cancel = |_| std::ops::ControlFlow::Continue(());+ match decode_v2_identity_columns_controlled(bytes, &mut no_cancel)? {+ StoredDecodeRun::Complete(Some((num_docs, ids, _))) if ids.len() == num_docs => {+ Ok(Some(ids))+ }+ StoredDecodeRun::Complete(_) | StoredDecodeRun::Cancelled => Ok(None),+ }+}+struct SelectedRowsVisitor<'a> {selected: &'a [usize],expected_rows: usize,@@ -195,6 +218,38 @@ fn select_json_rows<R: std::io::Read>(Ok(Ok(values))}+/// [`select_json_rows`] over a column already decompressed in memory. The+/// slice parser skips unselected rows several times faster than the+/// byte-at-a-time reader parser; validation is the same.+fn select_json_rows_slice(+ bytes: &[u8],+ selected: &[usize],+ expected_rows: usize,+ context: &str,+ column_index: usize,+ checkpoint: &mut dyn FnMut(StoredDecodeCheckpoint) -> std::ops::ControlFlow<()>,+) -> Result<std::result::Result<Vec<serde_json::Value>, ()>> {+ let cancelled = Cell::new(false);+ let mut deserializer = serde_json::Deserializer::from_slice(bytes);+ let values = match deserializer.deserialize_seq(SelectedRowsVisitor {+ selected,+ expected_rows,+ column_index,+ checkpoint,+ cancelled: &cancelled,+ }) {+ Ok(values) => values,+ Err(_) if cancelled.get() => return Ok(Err(())),+ Err(error) => {+ return Err(StorageError::Other(anyhow::anyhow!("{context}: {error}")));+ }+ };+ deserializer+ .end()+ .map_err(|error| StorageError::Other(anyhow::anyhow!("{context}: {error}")))?;+ Ok(Ok(values))+}+fn unpack_id(packed: &[u8], bit_width: u8, ordinal: usize) -> Option<u32> {let width = bit_width as usize;if !(1..=32).contains(&width) {@@ -489,6 +544,11 @@ fn repeat_constant_rows_controlled(Ok(Ok(values))}+/// `in_memory`: decompress a JSON column whole and parse it as a slice+/// (faster) instead of streaming it. Only the non-cancellable parallel+/// hydration asks for it, so cancellation still never waits on a whole+/// column's decompression.+#[allow(clippy::too_many_arguments)]fn select_column_rows(column_index: usize,directory: &V2Directory<'_>,@@ -496,9 +556,28 @@ fn select_column_rows(stats: &mut StoredV2RowHydrationStats,dict_ids: &mut HashMap<usize, Vec<u32>>,checkpoint: &mut dyn FnMut(StoredDecodeCheckpoint) -> std::ops::ControlFlow<()>,+ in_memory: bool,) -> Result<std::result::Result<Vec<serde_json::Value>, Option<String>>> {let column = &directory.columns[column_index];let values = match column.codec {+ ColCodec::RawJson if in_memory => {+ stats.encoded_rows_visited += directory.num_docs;+ let frame = decode_raw_payload(column.payload)?;+ let raw = zstd::stream::decode_all(frame)+ .map_err(|error| StorageError::Other(anyhow::anyhow!("raw zstd: {error}")))?;+ stats.decompressed_buffer_bytes += raw.len();+ match select_json_rows_slice(+ &raw,+ selected,+ directory.num_docs,+ "raw json",+ column_index,+ checkpoint,+ )? {+ Ok(values) => values,+ Err(()) => return Ok(Err(None)),+ }+ }ColCodec::RawJson => {stats.encoded_rows_visited += directory.num_docs;let frame = decode_raw_payload(column.payload)?;@@ -590,6 +669,7 @@ fn select_column_rows(stats,dict_ids,checkpoint,+ in_memory,);}ColCodec::CrossDep => {@@ -699,6 +779,38 @@ pub fn decode_stored_v2_rows_projected_controlled<F>(bytes: &[u8],ordinals: &[usize],source_fields: Option<&[&str]>,+ checkpoint: F,+) -> Result<StoredDecodeRun<StoredV2RowHydrationResult>>+where+ F: FnMut(StoredDecodeCheckpoint) -> std::ops::ControlFlow<()>,+{+ hydrate_v2_rows(bytes, ordinals, source_fields, false, checkpoint)+}++/// Like [`decode_stored_v2_rows_projected`] (`None` = every field), but+/// decodes the wanted columns concurrently on the rayon pool. Each column is+/// an independent stream, so a cold top-k hydration that must walk every+/// column of a segment for a handful of rows costs roughly its largest+/// column instead of the sum of all of them. Output is identical to the+/// serial decoder.+pub fn decode_stored_v2_rows_projected_parallel(+ bytes: &[u8],+ ordinals: &[usize],+ source_fields: Option<&[&str]>,+) -> Result<StoredV2RowHydrationResult> {+ match hydrate_v2_rows(bytes, ordinals, source_fields, true, |_| {+ std::ops::ControlFlow::Continue(())+ })? {+ StoredDecodeRun::Complete(result) => Ok(result),+ StoredDecodeRun::Cancelled => unreachable!("non-cancelling parallel wrapper"),+ }+}++fn hydrate_v2_rows<F>(+ bytes: &[u8],+ ordinals: &[usize],+ source_fields: Option<&[&str]>,+ parallel: bool,mut checkpoint: F,) -> Result<StoredDecodeRun<StoredV2RowHydrationResult>>where@@ -753,14 +865,57 @@ wherelet mut columns: Vec<Option<Vec<serde_json::Value>>> =(0..directory.columns.len()).map(|_| None).collect();let mut dict_ids = HashMap::new();- for index in 0..directory.columns.len() {- let column = &directory.columns[index];- let wanted = index == id_index- || index == seq_index- || requested- .as_ref()- .is_none_or(|fields| fields.contains(column.name));- if !wanted {+ let wanted_columns: Vec<usize> = (0..directory.columns.len())+ .filter(|&index| {+ index == id_index+ || index == seq_index+ || requested+ .as_ref()+ .is_none_or(|fields| fields.contains(directory.columns[index].name))+ })+ .collect();+ if parallel && wanted_columns.len() > 1 {+ use rayon::prelude::*;+ // Each task gets its own stats and dictionary-id scratch: a CROSS_DEP+ // column re-derives its DICT source ids locally, which is the same+ // work the serial path does once and shares.+ let decoded: Vec<_> = wanted_columns+ .par_iter()+ .map(|&index| {+ let mut local_stats = StoredV2RowHydrationStats::default();+ let mut local_dict_ids = HashMap::new();+ let mut noop = |_: StoredDecodeCheckpoint| std::ops::ControlFlow::Continue(());+ select_column_rows(+ index,+ &directory,+ &selected,+ &mut local_stats,+ &mut local_dict_ids,+ &mut noop,+ true,+ )+ .map(|values| (index, values, local_stats))+ })+ .collect();+ for result in decoded {+ let (index, values, local_stats) = result?;+ stats.encoded_rows_visited += local_stats.encoded_rows_visited;+ stats.selected_json_values += local_stats.selected_json_values;+ stats.selected_dictionary_entries += local_stats.selected_dictionary_entries;+ stats.decompressed_buffer_bytes += local_stats.decompressed_buffer_bytes;+ match values {+ Ok(values) => columns[index] = Some(values),+ Err(Some(column)) => {+ return Ok(StoredDecodeRun::Complete(+ StoredV2RowHydrationResult::UnsupportedDependencyShape { column },+ ));+ }+ Err(None) => return Ok(StoredDecodeRun::Cancelled),+ }+ }+ }+ for index in wanted_columns {+ if columns[index].is_some() {continue;}if decode_cancelled(&mut checkpoint, StoredDecodePhase::BeforeColumn, index, 0) {@@ -773,6 +928,7 @@ where&mut stats,&mut dict_ids,&mut checkpoint,+ false,)? {Ok(values) => columns[index] = Some(values),Err(Some(column)) => {@@ -954,6 +1110,76 @@ mod tests {assert!(decode_stored_v2_rows(&bad, &[0]).is_err());}+ /// The parallel hydration parses each JSON column as an in-memory slice;+ /// its rows must equal the streaming path's, and it must still reject a+ /// column with trailing JSON or a row count other than the directory's.+ #[test]+ fn parallel_hydration_matches_streaming_and_validates_columns() {+ for codec in [ColCodec::RawJson, ColCodec::Lz4Json] {+ let encoded = fixture(300, codec);+ let ordinals = [299, 0, 7, 7, 128];+ for fields in [None, Some(&["body"][..]), Some(&[][..])] {+ let serial = match fields {+ Some(fields) => decode_stored_v2_rows_projected(&encoded, &ordinals, fields),+ None => decode_stored_v2_rows(&encoded, &ordinals),+ }+ .unwrap();+ let parallel =+ decode_stored_v2_rows_projected_parallel(&encoded, &ordinals, fields).unwrap();+ let (+ StoredV2RowHydrationResult::Hydrated { rows: a, .. },+ StoredV2RowHydrationResult::Hydrated { rows: b, .. },+ ) = (serial, parallel)+ else {+ panic!("expected hydration")+ };+ assert_eq!(a.len(), 4);+ assert_eq!(a.len(), b.len());+ for (a, b) in a.iter().zip(&b) {+ assert_eq!(+ (a.ordinal, &a.id, &a.seq_no, &a.source),+ (b.ordinal, &b.id, &b.seq_no, &b.source)+ );+ }+ }+ }+ for tail in [&b"[]"[..], &b""[..]] {+ let mut values = serde_json::to_vec(&vec![json!("x"); 4]).unwrap();+ values.extend_from_slice(tail);+ let rows = if tail.is_empty() { 3 } else { 4 };+ let bad = assemble(+ rows,+ &[+ ("__id", ColCodec::RawJson, raw(&vec![json!("x"); rows])),+ ("__seq_no", ColCodec::RawJson, raw(&vec![json!(0); rows])),+ (+ "bad",+ ColCodec::RawJson,+ zstd::encode_all(Cursor::new(values), 3).unwrap(),+ ),+ ("other", ColCodec::RawJson, raw(&vec![json!(1); rows])),+ ],+ );+ assert!(decode_stored_v2_rows_projected_parallel(&bad, &[0], None).is_err());+ }+ }++ /// The identity-only id list equals the `_id`s of the full decode.+ #[test]+ fn identity_ids_match_the_full_decode() {+ for codec in [ColCodec::RawJson, ColCodec::Lz4Json] {+ let encoded = fixture(200, codec);+ let ids = decode_stored_v2_ids(&encoded).unwrap().unwrap();+ let full: Vec<serde_json::Value> =+ serde_json::from_slice(&super::super::decode_stored(&encoded).unwrap()).unwrap();+ assert_eq!(ids.len(), full.len());+ for (id, doc) in ids.iter().zip(&full) {+ assert_eq!(id, &doc["_id"]);+ }+ }+ assert!(decode_stored_v2_ids(b"not columnar").unwrap().is_none());+ }+#[test]fn dictionary_nulls_boundaries_and_corruption_are_validated() {let values = vec![json!("a"), json!("b"), json!("c")];