Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(ingest): P2.2 no-reparse flush + ahash memtable + gated ignore_malformed rewrite — 1M bulk 68.5k -> 110k docs/s

xerj-org0cdf07835e8c
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 86b403105..654f05ba2 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -9,6 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
+ "getrandom 0.3.4",
"once_cell",
"version_check",
"zerocopy",
@@ -3736,6 +3737,7 @@ dependencies = [
name = "xerj-engine"
version = "1.0.0-rc.1"
dependencies = [
+ "ahash",
"anyhow",
"arc-swap",
"bytes",
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 6dea20aa4..feb74d30e 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -9299,7 +9299,19 @@ async fn process_bulk_body(
let base_text: &str = ts_rewritten.as_deref().unwrap_or(text);
let rewritten_body;
- let text_ref: &str = if state.engine.index_mappings.is_empty() {
+ // P2.2 — the ignore_malformed rewrite parses + re-serialises EVERY
+ // source line serially (client-blocking). `apply_ignore_malformed`
+ // can only modify a doc when some field in the target mapping carries
+ // `ignore_malformed: true` or `ignore_above`; when no known mapping
+ // declares either directive the whole pass is a guaranteed no-op, so
+ // skip it instead of round-tripping the body through serde.
+ let text_ref: &str = if state.engine.index_mappings.is_empty()
+ || !state
+ .engine
+ .index_mappings
+ .iter()
+ .any(|e| mapping_has_ignore_directives(e.value()))
+ {
base_text
} else {
rewritten_body = rewrite_bulk_ignore_malformed(state, default_index, base_text);
@@ -19296,6 +19308,29 @@ pub(crate) fn rewrite_bulk_time_series_ids(
}
}
+/// True when a stored index mapping contains ANY directive that can make
+/// `apply_ignore_malformed` modify a document: a truthy `ignore_malformed`
+/// or an `ignore_above` anywhere in the (possibly nested) properties tree.
+/// Conservative on false positives (e.g. a user field literally named
+/// `ignore_above` still returns true and takes the slow rewrite path);
+/// never falsely negative, because `validate_doc_fields` only mutates a
+/// doc under one of these two directives.
+pub(crate) fn mapping_has_ignore_directives(mapping: &Value) -> bool {
+ match mapping {
+ Value::Object(m) => {
+ if m.get("ignore_malformed").and_then(Value::as_bool).unwrap_or(false) {
+ return true;
+ }
+ if m.contains_key("ignore_above") {
+ return true;
+ }
+ m.values().any(mapping_has_ignore_directives)
+ }
+ Value::Array(arr) => arr.iter().any(mapping_has_ignore_directives),
+ _ => false,
+ }
+}
+
pub(crate) fn rewrite_bulk_ignore_malformed(
state: &AppState,
default_index: Option<&str>,
diff --git a/engine/crates/xerj-engine/Cargo.toml b/engine/crates/xerj-engine/Cargo.toml
index e339c0feb..e108fc4d1 100644
--- a/engine/crates/xerj-engine/Cargo.toml
+++ b/engine/crates/xerj-engine/Cargo.toml
@@ -40,6 +40,12 @@ regex = "1"
# Fast non-cryptographic hash — used to shard the engine FTS memtable
# by doc_id so concurrent bulk ingest batches target different shards.
xxhash-rust = { workspace = true }
+# Fast string hasher for the FTS memtable's private postings/lookup maps.
+# perf(P2.2): SipHash over token/doc_id String keys was ~21% of ALL CPU
+# samples during 1M-doc bulk ingest. These maps are drained and discarded
+# at flush time and never serialized, so the hasher swap is invisible on
+# disk.
+ahash = "0.8"
[dev-dependencies]
tempfile = { workspace = true }
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 2fa900488..3544d567f 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1289,12 +1289,17 @@ impl Index {
// `fetch_add`; (c) response order matches request order.
use rayon::prelude::*;
- // 1. Parse in parallel — each doc exactly once.
- let sources: Vec<Value> = docs
+ // 1. Parse in parallel — each doc exactly once. Wrapped in `Arc`
+ // right here so the memtable insert below shares the SAME parsed
+ // tree (P2.2: pre-fix `insert` deep-cloned every Value under the
+ // shard lock, ~1.7s CPU/1M docs).
+ let sources: Vec<Arc<Value>> = docs
.par_iter()
.map(|(_, bytes)| {
- serde_json::from_slice::<Value>(bytes)
- .unwrap_or_else(|_| Value::Object(serde_json::Map::new()))
+ Arc::new(
+ serde_json::from_slice::<Value>(bytes)
+ .unwrap_or_else(|_| Value::Object(serde_json::Map::new())),
+ )
})
.collect();
@@ -1327,7 +1332,16 @@ impl Index {
// the batch leaves exactly one live entry
// (mirrors index_document).
m.remove(id);
- m.insert(id.clone(), &sources[i], schema, seq_nos[i]);
+ // Share the parsed Value via Arc (no deep clone)
+ // and pass the raw NDJSON length for size
+ // accounting (no `to_string()` re-serialisation).
+ m.insert_arc(
+ id.clone(),
+ Arc::clone(&sources[i]),
+ docs[i].1.len(),
+ schema,
+ seq_nos[i],
+ );
}
});
});
@@ -6773,7 +6787,7 @@ async fn do_flush_shard(
Vec<(
String,
std::collections::HashMap<String, String>,
- serde_json::Value,
+ std::sync::Arc<serde_json::Value>,
)>,
xerj_storage::index_store::DrainedMemtable,
)> = {
@@ -6798,19 +6812,24 @@ async fn do_flush_shard(
// CPU per flush — irrelevant against the 50-100 ms segment
// write. Always build the drained_fts payload so the FTS
// sidecar carries the full doc set.
- let drained_fts: Vec<(String, std::collections::HashMap<String, String>, serde_json::Value)> =
+ let drained_fts: Vec<(String, std::collections::HashMap<String, String>, std::sync::Arc<serde_json::Value>)> =
raw.iter()
.map(|(_seq, doc_id, arc, bytes)| {
// Raw-bytes path stores Value::Null in `arc` and
// the actual JSON in `bytes` (skip-parse drain).
// Re-parse just-in-time for FTS field extraction.
- let val: serde_json::Value = if !arc.is_null() {
- (**arc).clone()
+ // P2.2: the parsed path SHARES the memtable's Arc
+ // instead of deep-cloning every Value (~3.7s CPU
+ // per 1M docs on the background flush thread).
+ let val: std::sync::Arc<serde_json::Value> = if !arc.is_null() {
+ std::sync::Arc::clone(arc)
} else if !bytes.is_empty() {
- serde_json::from_slice::<serde_json::Value>(bytes)
- .unwrap_or(serde_json::Value::Null)
+ std::sync::Arc::new(
+ serde_json::from_slice::<serde_json::Value>(bytes)
+ .unwrap_or(serde_json::Value::Null),
+ )
} else {
- serde_json::Value::Null
+ std::sync::Arc::new(serde_json::Value::Null)
};
let fields = crate::memtable::extract_text_fields_from(&val);
(doc_id.clone(), fields, val)
@@ -6873,7 +6892,7 @@ async fn do_flush_shard(
// ── Doc-values side-car (always built) ────────────────────
{
let columns = build_doc_value_columns(
- drained_fts.iter().map(|(_id, _fields, src)| Some(src)),
+ drained_fts.iter().map(|(_id, _fields, src)| Some(src.as_ref())),
);
if !columns.is_empty() {
if let Err(e) = write_doc_values_sidecar(
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 4cdfec329..7e20b008b 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -80,8 +80,17 @@ pub fn extract_text_fields_from(source: &Value) -> HashMap<String, String> {
out
}
+/// Fast-hash map used for the memtable's PRIVATE hot-path structures
+/// (inverted index, field lengths, doc_id lookup). These maps are hit
+/// once-or-more per token on the bulk-ingest hot path; the default
+/// SipHash over String keys measured ~21% of all CPU samples during a
+/// 1M-doc bulk run. They are drained and rebuilt from scratch at flush
+/// time and never serialized, so swapping the hasher has no on-disk or
+/// wire-visible effect.
+type FastMap<K, V> = HashMap<K, V, ahash::RandomState>;
+
/// Posting list entry: doc_id → term frequency.
-type PostingList = HashMap<String, u32>; // doc_id → tf
+type PostingList = FastMap<String, u32>; // doc_id → tf
// ── DocValues ─────────────────────────────────────────────────────────────────
@@ -877,12 +886,12 @@ impl ShardedFtsMemtable {
let drained: Vec<MemEntry> = {
let mut g = self.shards[shard_idx].write();
let d: Vec<MemEntry> = g.docs.drain(..).collect();
- g.index = HashMap::new();
+ g.index = FastMap::default();
g.doc_values = DocValues::default();
g.total_bytes = 0;
- g.field_lengths = HashMap::new();
- g.avg_field_lengths = HashMap::new();
- g.doc_id_index = HashMap::new();
+ g.field_lengths = FastMap::default();
+ g.avg_field_lengths = FastMap::default();
+ g.doc_id_index = FastMap::default();
// Flush == merge: purge delete-aware ghost collection stats.
g.ghost_docs = 0;
g.ghost_field_len = HashMap::new();
@@ -961,12 +970,12 @@ impl FtsMemtable {
(seq, (e.doc_id, fields, val))
})
.collect();
- self.index = HashMap::new();
+ self.index = FastMap::default();
self.doc_values = DocValues::default();
self.total_bytes = 0;
- self.field_lengths = HashMap::new();
- self.avg_field_lengths = HashMap::new();
- self.doc_id_index = HashMap::new();
+ self.field_lengths = FastMap::default();
+ self.avg_field_lengths = FastMap::default();
+ self.doc_id_index = FastMap::default();
// Flush is the equivalent of a Lucene merge: tombstone contributions
// are purged, so delete-aware ghost statistics reset to empty.
self.ghost_docs = 0;
@@ -987,12 +996,12 @@ impl FtsMemtable {
(seq, (e.doc_id, fields))
})
.collect();
- self.index = HashMap::new();
+ self.index = FastMap::default();
self.doc_values = DocValues::default();
self.total_bytes = 0;
- self.field_lengths = HashMap::new();
- self.avg_field_lengths = HashMap::new();
- self.doc_id_index = HashMap::new();
+ self.field_lengths = FastMap::default();
+ self.avg_field_lengths = FastMap::default();
+ self.doc_id_index = FastMap::default();
// Flush is the equivalent of a Lucene merge: tombstone contributions
// are purged, so delete-aware ghost statistics reset to empty.
self.ghost_docs = 0;
@@ -1017,7 +1026,7 @@ pub struct FtsMemtable {
/// Documents in insertion order.
docs: Vec<MemEntry>,
/// Inverted index: field → term → posting list (doc_id → tf).
- index: HashMap<String, HashMap<String, PostingList>>,
+ index: FastMap<String, FastMap<String, PostingList>>,
/// Columnar doc-values store for fast term/range/agg queries.
pub doc_values: DocValues,
/// Total accumulated byte size.
@@ -1025,11 +1034,11 @@ pub struct FtsMemtable {
/// Analyzer registry.
registry: Arc<AnalyzerRegistry>,
/// Precomputed field lengths for BM25 scoring: field → {doc_id → token_count}
- field_lengths: HashMap<String, HashMap<String, u32>>,
+ field_lengths: FastMap<String, FastMap<String, u32>>,
/// Running average field length per field: field → (total_tokens, doc_count)
- avg_field_lengths: HashMap<String, (f64, u64)>,
+ avg_field_lengths: FastMap<String, (f64, u64)>,
/// doc_id → position in self.docs for O(1) lookup
- doc_id_index: HashMap<String, usize>,
+ doc_id_index: FastMap<String, usize>,
/// Delete-aware BM25 collection statistics (Lucene/ES parity).
///
/// When a document is superseded by an update (remove + re-insert) or
@@ -1054,13 +1063,13 @@ impl FtsMemtable {
pub fn new() -> Self {
Self {
docs: Vec::new(),
- index: HashMap::new(),
+ index: FastMap::default(),
doc_values: DocValues::default(),
total_bytes: 0,
registry: Arc::new(AnalyzerRegistry::default()),
- field_lengths: HashMap::new(),
- avg_field_lengths: HashMap::new(),
- doc_id_index: HashMap::new(),
+ field_lengths: FastMap::default(),
+ avg_field_lengths: FastMap::default(),
+ doc_id_index: FastMap::default(),
ghost_docs: 0,
ghost_field_len: HashMap::new(),
ghost_doc_freq: HashMap::new(),
@@ -1075,13 +1084,13 @@ impl FtsMemtable {
pub fn with_registry(registry: Arc<AnalyzerRegistry>) -> Self {
Self {
docs: Vec::new(),
- index: HashMap::new(),
+ index: FastMap::default(),
doc_values: DocValues::default(),
total_bytes: 0,
registry,
- field_lengths: HashMap::new(),
- avg_field_lengths: HashMap::new(),
- doc_id_index: HashMap::new(),
+ field_lengths: FastMap::default(),
+ avg_field_lengths: FastMap::default(),
+ doc_id_index: FastMap::default(),
ghost_docs: 0,
ghost_field_len: HashMap::new(),
ghost_doc_freq: HashMap::new(),
@@ -1089,7 +1098,31 @@ impl FtsMemtable {
}
/// Insert a document into the memtable, indexing all text fields.
+ ///
+ /// Compatibility wrapper around [`insert_arc`] for callers that hold a
+ /// borrowed `Value` (single-doc PUT, WAL replay). The bulk hot path
+ /// uses `insert_arc` directly to skip the deep `source.clone()` and
+ /// the `to_string()` re-serialisation that only fed size accounting.
pub fn insert(&mut self, doc_id: String, source: &Value, schema: &Schema, seq_no: u64) {
+ let raw_len = source.to_string().len();
+ self.insert_arc(doc_id, Arc::new(source.clone()), raw_len, schema, seq_no);
+ }
+
+ /// Zero-copy insert: takes an `Arc<Value>` shared with the caller (no
+ /// deep clone of the JSON tree) plus the raw serialized length of the
+ /// document (`raw_len`, e.g. the NDJSON line length) for byte-size
+ /// accounting. P2.2: pre-fix the hot bulk path paid one full
+ /// `source.to_string()` (whose ONLY consumer was `.len()`) and one
+ /// deep `Arc::new(source.clone())` per document — together ~3.2s of
+ /// client-blocking CPU per 1M docs.
+ pub fn insert_arc(
+ &mut self,
+ doc_id: String,
+ source: Arc<Value>,
+ raw_len: usize,
+ schema: &Schema,
+ seq_no: u64,
+ ) {
let mut text_fields: HashMap<String, String> = HashMap::new();
// Index fields that are defined as Text in the schema.
@@ -1150,7 +1183,7 @@ impl FtsMemtable {
}
}
if let Some(_obj) = source.as_object() {
- collect_text_fields(source, "", &mut text_fields);
+ collect_text_fields(source.as_ref(), "", &mut text_fields);
}
// Build the inverted index entries.
@@ -1184,15 +1217,18 @@ impl FtsMemtable {
}
}
- // See `insert_pretokenized` for sizing rationale.
- let raw_size = source.to_string().len() + doc_id.len();
+ // See `insert_pretokenized` for sizing rationale. `raw_len` is the
+ // caller-supplied serialized length (NDJSON line length on the bulk
+ // path) — same ballpark as the old `source.to_string().len()`
+ // without re-serialising the document just to measure it.
+ let raw_size = raw_len + doc_id.len();
let size = raw_size * 3 + 64;
self.total_bytes += size;
// Populate the columnar DocValues store BEFORE pushing to docs so that
// the doc_index equals the current length (i.e. the slot we're about to fill).
let doc_index = self.docs.len();
- self.doc_values.push(source, doc_index);
+ self.doc_values.push(&source, doc_index);
// Track doc_id → index for O(1) lookup.
self.doc_id_index.insert(doc_id.clone(), doc_index);
@@ -1200,7 +1236,8 @@ impl FtsMemtable {
self.docs.push(MemEntry {
seq_no,
doc_id,
- source: Arc::new(source.clone()),
+ // Arc clone from the caller — refcount bump, no deep copy.
+ source,
source_bytes: Arc::from(&[][..]),
size_bytes: size,
});
@@ -1702,12 +1739,12 @@ impl FtsMemtable {
(e.doc_id, fields)
})
.collect();
- self.index = HashMap::new();
+ self.index = FastMap::default();
self.doc_values = DocValues::default();
self.total_bytes = 0;
- self.field_lengths = HashMap::new();
- self.avg_field_lengths = HashMap::new();
- self.doc_id_index = HashMap::new();
+ self.field_lengths = FastMap::default();
+ self.avg_field_lengths = FastMap::default();
+ self.doc_id_index = FastMap::default();
// Flush is the equivalent of a Lucene merge: tombstone contributions
// are purged, so delete-aware ghost statistics reset to empty.
self.ghost_docs = 0;
@@ -1751,12 +1788,12 @@ impl FtsMemtable {
// See `drain()` for why we must reassign these (not `.clear()`):
// `HashMap::clear` retains internal bucket capacity and the RSS
// never shrinks between flushes.
- self.index = HashMap::new();
+ self.index = FastMap::default();
self.doc_values = DocValues::default();
self.total_bytes = 0;
- self.field_lengths = HashMap::new();
- self.avg_field_lengths = HashMap::new();
- self.doc_id_index = HashMap::new();
+ self.field_lengths = FastMap::default();
+ self.avg_field_lengths = FastMap::default();
+ self.doc_id_index = FastMap::default();
// Flush is the equivalent of a Lucene merge: tombstone contributions
// are purged, so delete-aware ghost statistics reset to empty.
self.ghost_docs = 0;
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index d8d3b09f4..b5f4a3cfd 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -521,7 +521,11 @@ impl FtsIndexWriter {
/// - Doc ordinals are assigned by position in the input `docs` vec,
/// matching the row index the caller used with
/// `add_document(ordinal, ...)`.
- pub fn add_documents_parallel(&mut self, docs: &[(String, HashMap<String, String>, serde_json::Value)]) {
+ ///
+ /// Generic over the third tuple element (a source payload the caller
+ /// keeps alongside for its own use — `serde_json::Value`,
+ /// `Arc<serde_json::Value>`, …): this method never reads it.
+ pub fn add_documents_parallel<S: Sync>(&mut self, docs: &[(String, HashMap<String, String>, S)]) {
use rayon::prelude::*;
use std::collections::HashMap as StdHashMap;
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 4780b68ba..4c8143e51 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -859,7 +859,49 @@ impl IndexStore {
// segments. The encoder picks v1 or v2 internally based on
// which is smaller on this segment, so we never write a
// larger file than the pre-M4.6 format.
- let encoded = crate::stored_codec::encode_stored_v2(&stored_bytes);
+ //
+ // P2.2 — when every live entry carries a parsed `source`
+ // (the HTTP `_bulk` turbo path: engine memtable drained
+ // parsed Values, `source_bytes` empty), feed the encoder the
+ // Values directly instead of letting it re-parse the JSON
+ // array we just serialised (~10s background CPU per 1M
+ // docs). Byte-identical output by contract — see
+ // `encode_stored_v2_from_values`; assert it live with
+ // `XERJ_FLUSH_PARITY=1`.
+ let all_parsed = live_entries
+ .iter()
+ .all(|e| e.source_bytes.is_empty() && e.source.is_some());
+ let encoded = if all_parsed {
+ let doc_refs: Vec<(&str, u64, &serde_json::Value)> = live_entries
+ .iter()
+ .map(|e| {
+ (
+ e.doc_id.as_str(),
+ e.seq_no,
+ e.source.as_deref().expect("all_parsed checked source.is_some()"),
+ )
+ })
+ .collect();
+ let enc =
+ crate::stored_codec::encode_stored_v2_from_values(&stored_bytes, &doc_refs);
+ if std::env::var("XERJ_FLUSH_PARITY").map(|v| v == "1").unwrap_or(false) {
+ let legacy = crate::stored_codec::encode_stored_v2(&stored_bytes);
+ assert_eq!(
+ legacy, enc,
+ "XERJ_FLUSH_PARITY: encode_stored_v2_from_values diverged from \
+ encode_stored_v2 ({} live docs)",
+ doc_refs.len()
+ );
+ tracing::info!(
+ docs = doc_refs.len(),
+ bytes = enc.len(),
+ "XERJ_FLUSH_PARITY: stored-section bytes identical"
+ );
+ }
+ enc
+ } else {
+ crate::stored_codec::encode_stored_v2(&stored_bytes)
+ };
writer.add_section(SectionType::Stored, &encoded)?;
}
diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rs
index ea302fb83..287f14f46 100644
--- a/engine/crates/xerj-storage/src/stored_codec.rs
+++ b/engine/crates/xerj-storage/src/stored_codec.rs
@@ -176,29 +176,102 @@ pub fn encode_stored_v2(stored_docs_json: &[u8]) -> Vec<u8> {
}
}
- // Second pass: materialise each column as Vec<&serde_json::Value>.
+ // Second pass: materialise each column as Vec<&serde_json::Value>
+ // (borrowed out of the parsed docs — no per-field clones).
// `__id` and `__seq_no` are hydrated from the top-level doc object.
- let mut columns: Vec<Vec<serde_json::Value>> =
+ static NULL: serde_json::Value = serde_json::Value::Null;
+ let mut columns: Vec<Vec<&serde_json::Value>> =
vec![Vec::with_capacity(num_docs); col_order.len()];
- let null = serde_json::Value::Null;
for doc in &docs {
// __id and __seq_no
- let id = doc.get("_id").cloned().unwrap_or(serde_json::Value::Null);
- let seq = doc.get("_seq_no").cloned().unwrap_or(serde_json::Value::Null);
- columns[0].push(id);
- columns[1].push(seq);
+ columns[0].push(doc.get("_id").unwrap_or(&NULL));
+ columns[1].push(doc.get("_seq_no").unwrap_or(&NULL));
let src_obj = doc.get("_source").and_then(|s| s.as_object());
for (cix, cname) in col_order.iter().enumerate().skip(2) {
- let val = src_obj
- .and_then(|m| m.get(cname))
- .cloned()
- .unwrap_or_else(|| null.clone());
+ let val = src_obj.and_then(|m| m.get(cname)).unwrap_or(&NULL);
columns[cix].push(val);
}
}
+ encode_v2_columns(num_docs, &col_order, &columns, stored_docs_json)
+}
+
+/// P2.2 — columnar V2 encoder fed directly from already-parsed values.
+///
+/// `encode_stored_v2` re-parses the (potentially multi-MB) JSON array the
+/// flush path just serialised and clones every field into columns — ~10s
+/// of background-flush CPU per 1M docs. When the flush already holds the
+/// parsed `_source` trees (the HTTP `_bulk` turbo path), this entry point
+/// builds the columns straight from them.
+///
+/// Contract: `stored_docs_json` MUST be the canonical serialisation of
+/// `docs` (`[{"_id":…,"_seq_no":…,"_source":…}, …]`, as built by
+/// `IndexStore::finalize_flush_with_publisher`). It is used for the v1
+/// (LZ4) fallback on tiny segments and the final "never make things
+/// worse" size comparison, so the output is byte-identical to
+/// `encode_stored_v2(stored_docs_json)` — asserted by unit tests here and
+/// by the `XERJ_FLUSH_PARITY=1` runtime gate in the flush path.
+pub fn encode_stored_v2_from_values(
+ stored_docs_json: &[u8],
+ docs: &[(&str, u64, &serde_json::Value)],
+) -> Vec<u8> {
+ if docs.len() < V2_MIN_DOCS {
+ return encode_stored_lz4(stored_docs_json);
+ }
+ let num_docs = docs.len();
+
+ // Synthetic __id / __seq_no columns need owned Values to borrow from.
+ let ids: Vec<serde_json::Value> = docs
+ .iter()
+ .map(|(id, _, _)| serde_json::Value::String((*id).to_string()))
+ .collect();
+ let seqs: Vec<serde_json::Value> = docs
+ .iter()
+ .map(|(_, seq, _)| serde_json::Value::from(*seq))
+ .collect();
+
+ let mut col_order: Vec<String> = vec!["__id".into(), "__seq_no".into()];
+ let mut col_seen: HashMap<String, usize> = HashMap::new();
+ col_seen.insert("__id".into(), 0);
+ col_seen.insert("__seq_no".into(), 1);
+
+ for (_, _, src) in docs {
+ if let Some(obj) = src.as_object() {
+ for key in obj.keys() {
+ if !col_seen.contains_key(key) {
+ col_seen.insert(key.clone(), col_order.len());
+ col_order.push(key.clone());
+ }
+ }
+ }
+ }
+
+ static NULL: serde_json::Value = serde_json::Value::Null;
+ let mut columns: Vec<Vec<&serde_json::Value>> =
+ vec![Vec::with_capacity(num_docs); col_order.len()];
+ for (i, (_, _, src)) in docs.iter().enumerate() {
+ columns[0].push(&ids[i]);
+ columns[1].push(&seqs[i]);
+ let src_obj = src.as_object();
+ for (cix, cname) in col_order.iter().enumerate().skip(2) {
+ let val = src_obj.and_then(|m| m.get(cname)).unwrap_or(&NULL);
+ columns[cix].push(val);
+ }
+ }
+
+ encode_v2_columns(num_docs, &col_order, &columns, stored_docs_json)
+}
+
+/// Shared tail of the two V2 entry points: per-column codec selection +
+/// payload assembly + the v1-LZ4 "never make things worse" size net.
+fn encode_v2_columns(
+ num_docs: usize,
+ col_order: &[String],
+ columns: &[Vec<&serde_json::Value>],
+ stored_docs_json: &[u8],
+) -> Vec<u8> {
// Build dict-encoded form for each column as a stable side representation.
// `dict_encode(col)` returns Some((dict_entries, ids)) where ids[i] is the
// 1-based id (0 reserved for null, we add 1 here). Returns None when
@@ -415,9 +488,10 @@ fn decode_stored_v2(body: &[u8]) -> Result<Vec<u8>> {
/// `_source`. A column that contains even one non-integer float must skip
/// the numeric optimization entirely and fall back to the lossless dict /
/// raw-JSON path, which preserves the exact `serde_json::Value`.
-fn col_is_all_integer(col: &[serde_json::Value]) -> bool {
+fn col_is_all_integer<B: std::borrow::Borrow<serde_json::Value>>(col: &[B]) -> bool {
let mut saw_num = false;
for v in col {
+ let v = v.borrow();
if v.is_null() { continue; }
// Only true JSON integers qualify; floats (even integer-valued
// ones like `10.0`) are left for the lossless path so their exact
@@ -438,11 +512,14 @@ fn all_scalar_dict_entries(entries: &[serde_json::Value]) -> bool {
///
/// Returns `None` if the column contains any non-scalar (object/array) value
/// or if the unique-count explodes beyond the cap.
-fn dict_encode_column(col: &[serde_json::Value]) -> Option<(Vec<serde_json::Value>, Vec<u32>)> {
+fn dict_encode_column<B: std::borrow::Borrow<serde_json::Value>>(
+ col: &[B],
+) -> Option<(Vec<serde_json::Value>, Vec<u32>)> {
let mut map: HashMap<String, u32> = HashMap::new();
let mut entries: Vec<serde_json::Value> = Vec::new();
let mut ids: Vec<u32> = Vec::with_capacity(col.len());
for v in col {
+ let v = v.borrow();
if v.is_null() {
ids.push(u32::MAX); // resolved to "null id" later
continue;
@@ -474,10 +551,10 @@ fn dict_encode_column(col: &[serde_json::Value]) -> Option<(Vec<serde_json::Valu
/// For a numeric target column, find the index of an earlier dict-encoded
/// column (if any) whose dict ids deterministically predict the target at
/// ≥ `CROSS_DEP_MIN_DETERMINISM` fraction of rows.
-fn best_cross_dep_source(
+fn best_cross_dep_source<B: std::borrow::Borrow<serde_json::Value>>(
dict_encoded: &[Option<(Vec<serde_json::Value>, Vec<u32>)>],
target_ix: usize,
- target_col: &[serde_json::Value],
+ target_col: &[B],
) -> Option<usize> {
for (src_ix, de) in dict_encoded.iter().enumerate() {
if src_ix == target_ix { continue; }
@@ -489,6 +566,7 @@ fn best_cross_dep_source(
// Build mode table: mode_values[src_id] = most frequent target int
let mut mode_tally: HashMap<u32, HashMap<i64, usize>> = HashMap::new();
for (row, t_val) in target_col.iter().enumerate() {
+ let t_val = t_val.borrow();
let Some(t) = t_val.as_i64().or_else(|| t_val.as_f64().map(|f| f as i64)) else { continue };
let sid = ids[row];
*mode_tally.entry(sid).or_default().entry(t).or_insert(0) += 1;
@@ -503,6 +581,7 @@ fn best_cross_dep_source(
let mut hits = 0usize;
let mut total_numeric = 0usize;
for (row, t_val) in target_col.iter().enumerate() {
+ let t_val = t_val.borrow();
let Some(t) = t_val.as_i64().or_else(|| t_val.as_f64().map(|f| f as i64)) else { continue };
total_numeric += 1;
if mode_pick.get(&ids[row]) == Some(&t) { hits += 1; }
@@ -516,9 +595,11 @@ fn best_cross_dep_source(
None
}
-fn try_encode_constant(col: &[serde_json::Value]) -> Option<Vec<u8>> {
- let first = col.iter().find(|v| !v.is_null())?;
- if col.iter().all(|v| v == first || v.is_null()) && !col.iter().any(|v| v.is_null()) {
+fn try_encode_constant<B: std::borrow::Borrow<serde_json::Value>>(col: &[B]) -> Option<Vec<u8>> {
+ let first = col.iter().map(|v| v.borrow()).find(|v| !v.is_null())?;
+ if col.iter().all(|v| { let v = v.borrow(); v == first || v.is_null() })
+ && !col.iter().any(|v| v.borrow().is_null())
+ {
// Only encode as constant when there are no nulls (keep the codec simple).
return Some(serde_json::to_vec(first).ok()?);
}
@@ -607,8 +688,8 @@ fn decode_dict_bitpack(payload: &[u8], num_docs: usize) -> Result<Vec<serde_json
Ok(values)
}
-fn encode_cross_dep(
- target_col: &[serde_json::Value],
+fn encode_cross_dep<B: std::borrow::Borrow<serde_json::Value>>(
+ target_col: &[B],
src_ix: usize,
dict_encoded: &[Option<(Vec<serde_json::Value>, Vec<u32>)>],
) -> (Vec<u8>, bool) {
@@ -626,6 +707,7 @@ fn encode_cross_dep(
};
let mut tally: Vec<HashMap<i64, u32>> = vec![HashMap::new(); dict_count + 1];
for (row, t) in target_col.iter().enumerate() {
+ let t = t.borrow();
let Some(tv) = t.as_i64().or_else(|| t.as_f64().map(|f| f as i64)) else { continue };
let sid = src_ids[row] as usize;
*tally[sid.min(dict_count)].entry(tv).or_insert(0) += 1;
@@ -639,6 +721,7 @@ fn encode_cross_dep(
let mut exceptions: Vec<(u32, i64)> = Vec::new();
for (row, t) in target_col.iter().enumerate() {
+ let t = t.borrow();
let Some(tv) = t.as_i64().or_else(|| t.as_f64().map(|f| f as i64)) else {
// Null / non-numeric — emit as exception with sentinel i64::MIN+1 ? No,
// simpler: mark with i64::MIN via sentinel tuple, but we cannot
@@ -982,4 +1065,103 @@ mod tests {
let unpacked = bitunpack_u32(&packed, 3, ids.len());
assert_eq!(unpacked, ids);
}
+
+ /// Serialise (id, seq_no, source) triples with the EXACT byte
+ /// construction `IndexStore::finalize_flush_with_publisher` uses for
+ /// parsed-source entries, so the parity assertion below covers the
+ /// real flush input.
+ fn stored_json_like_flush(docs: &[(String, u64, serde_json::Value)]) -> Vec<u8> {
+ use std::io::Write;
+ let mut stored_bytes: Vec<u8> = Vec::with_capacity(docs.len() * 512);
+ stored_bytes.push(b'[');
+ let mut first = true;
+ for (id, seq, src) in docs {
+ if !first {
+ stored_bytes.push(b',');
+ }
+ first = false;
+ stored_bytes.extend_from_slice(br#"{"_id":"#);
+ serde_json::to_writer(&mut stored_bytes, id).unwrap();
+ stored_bytes.extend_from_slice(br#","_seq_no":"#);
+ write!(stored_bytes, "{}", seq).unwrap();
+ stored_bytes.extend_from_slice(br#","_source":"#);
+ serde_json::to_writer(&mut stored_bytes, src).unwrap();
+ stored_bytes.push(b'}');
+ }
+ stored_bytes.push(b']');
+ stored_bytes
+ }
+
+ /// P2.2 parity gate — `encode_stored_v2_from_values` must be
+ /// byte-identical to the legacy parse-based `encode_stored_v2` for
+ /// the same flush input, across mixed field types (strings, ints,
+ /// floats, dates, bools, explicit nulls, nested objects, arrays)
+ /// and ragged shapes (missing fields).
+ #[test]
+ fn v2_from_values_byte_identical_to_legacy() {
+ let mut docs: Vec<(String, u64, serde_json::Value)> = Vec::new();
+ for i in 0..2000usize {
+ let mut src = json!({
+ "name": format!("doc-{}", i % 37),
+ "count": i as i64,
+ "big": u64::MAX - (i as u64 % 3),
+ "neg": -(i as i64) * 7,
+ "score": (i as f64) * 0.25 + 0.010127,
+ "when": format!("2026-07-{:02}T12:00:{:02}Z", (i % 28) + 1, i % 60),
+ "flag": i % 2 == 0,
+ "maybe": if i % 5 == 0 { serde_json::Value::Null } else { json!("x") },
+ "nested": { "a": i % 3, "b": [1, "two", serde_json::Value::Null, 3.5] },
+ "tags": ["alpha", "beta"],
+ });
+ if i % 7 == 0 {
+ src.as_object_mut().unwrap().remove("count");
+ }
+ if i % 11 == 0 {
+ src.as_object_mut()
+ .unwrap()
+ .insert("rare".into(), json!({"deep": {"x": i}}));
+ }
+ docs.push((format!("id-{}", i), 1_000 + i as u64, src));
+ }
+ let stored = stored_json_like_flush(&docs);
+ let refs: Vec<(&str, u64, &serde_json::Value)> = docs
+ .iter()
+ .map(|(id, seq, src)| (id.as_str(), *seq, src))
+ .collect();
+
+ let legacy = encode_stored_v2(&stored);
+ let from_values = encode_stored_v2_from_values(&stored, &refs);
+ assert_eq!(
+ legacy, from_values,
+ "from_values encoder diverged from legacy parse-based encoder"
+ );
+
+ // Decode round-trip sanity: every doc materialises with its id/seq.
+ let decoded = decode_stored(&from_values).unwrap();
+ let round: Vec<serde_json::Value> = serde_json::from_slice(&decoded).unwrap();
+ assert_eq!(round.len(), docs.len());
+ for i in [0usize, 1, 6, 42, 1024, 1999] {
+ assert_eq!(round[i]["_id"].as_str().unwrap(), format!("id-{}", i));
+ assert_eq!(round[i]["_seq_no"].as_u64().unwrap(), 1_000 + i as u64);
+ assert_eq!(round[i]["_source"]["score"], docs[i].2["score"]);
+ }
+ }
+
+ /// Below `V2_MIN_DOCS` both entry points must produce the identical
+ /// v1 LZ4 fallback bytes.
+ #[test]
+ fn v2_from_values_tiny_input_matches_legacy_fallback() {
+ let docs: Vec<(String, u64, serde_json::Value)> = (0..10usize)
+ .map(|i| (format!("t-{}", i), i as u64, json!({"m": i, "s": "x"})))
+ .collect();
+ let stored = stored_json_like_flush(&docs);
+ let refs: Vec<(&str, u64, &serde_json::Value)> = docs
+ .iter()
+ .map(|(id, seq, src)| (id.as_str(), *seq, src))
+ .collect();
+ let legacy = encode_stored_v2(&stored);
+ let from_values = encode_stored_v2_from_values(&stored, &refs);
+ assert_eq!(legacy, from_values);
+ assert_eq!(&from_values[..4], STORED_LZ4_MAGIC);
+ }
}