AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(storage): route ZBS3 through the columnar retained-size estimator (#538)
xerj-orgf34ad8d729bd
diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rsindex 0111ef55e..5d2e9c55f 100644--- a/engine/crates/xerj-storage/src/stored_codec.rs+++ b/engine/crates/xerj-storage/src/stored_codec.rs@@ -742,6 +742,17 @@ pub fn stored_slices_retained_upper_bound(bytes: &[u8], expected_docs: u64) -> R.and_then(|size| size.checked_add(STORED_SLICES_STRUCT_OVERHEAD)).ok_or_else(|| StorageError::Other(anyhow::anyhow!("stored retained bound overflow")))?;+ // ZBS3 (#538): a present-null sidecar wrapping an unchanged ZBS2 blob.+ // Route it through the columnar estimator on the inner ZBS2 blob rather than+ // the compressed-bytes fallback below (which would badly UNDER-estimate the+ // re-inflated JSON). The inner ZBS2 bound already over-counts every row's+ // key name (`rows × (name+2)`) and a >=4-byte value per row, so the sparse+ // `"<name>":null` cells that ZBS3 decode re-inserts are already within it.+ if bytes.len() >= 4 && &bytes[..4] == STORED_V3_MAGIC {+ let (_present_nulls, inner) = split_stored_v3(bytes)?;+ return stored_slices_retained_upper_bound(inner, expected_docs);+ }+let (decoded_bound, offsets_bound) = if bytes.len() >= 4 && &bytes[..4] == STORED_LZ4_MAGIC {let payload = bytes.get(4..8)@@ -2752,6 +2763,51 @@ mod tests {assert!(got[1]["_source"].get("z").is_none());}+ #[test]+ fn v3_retained_bound_routes_through_inner_zbs2_estimator() {+ // #538: a null-bearing segment is ZBS3. Its retained upper bound must be+ // computed via the columnar estimator on the inner ZBS2 blob — NOT the+ // compressed-bytes fallback (which would badly under-estimate the+ // re-inflated JSON). Assert the ZBS3 bound equals the inner ZBS2 bound;+ // this fails-before with the pre-#538 code, where a ZBS3 blob hit the+ // else branch and returned ~compressed bytes.len(). Robust whether the+ // estimator is finite or fails closed (Some==Some or None==None).+ let docs: Vec<serde_json::Value> = (0..400u64)+ .map(|i| {+ if i % 5 == 0 {+ json!({ "_id": i.to_string(), "_seq_no": i, "_source": { "a": "x", "nullable_field": null } })+ } else {+ json!({ "_id": i.to_string(), "_seq_no": i, "_source": { "a": "x" } })+ }+ })+ .collect();+ let encoded = encode_stored_v2(&serde_json::to_vec(&docs).unwrap());+ assert_eq!(+ &encoded[..4],+ STORED_V3_MAGIC,+ "must exercise the ZBS3 path"+ );++ let (_present_nulls, inner) = split_stored_v3(&encoded).unwrap();+ let zbs3_bound = stored_slices_retained_upper_bound(&encoded, docs.len() as u64).unwrap();+ let inner_bound = stored_slices_retained_upper_bound(inner, docs.len() as u64).unwrap();+ assert_eq!(+ zbs3_bound, inner_bound,+ "ZBS3 retained bound must route through the inner ZBS2 estimator (#538)"+ );++ // When the estimator is finite it must remain a true UPPER bound over the+ // re-inflated JSON that re-inserts the present-null keys.+ if let Some(bound) = zbs3_bound {+ let decoded = decode_stored(&encoded).unwrap();+ assert!(+ bound >= decoded.len() as u64,+ "retained bound {bound} must be >= actual decoded length {} (#538)",+ decoded.len()+ );+ }+ }+fn projection_fixture() -> (Vec<serde_json::Value>, Vec<u8>) {let docs: Vec<_> = (0..256).map(|i| {