Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): pack semantic embeddings so JSONL autoindex RSS stays bounded (#2240)

xerj-orge15abb780f25
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index f07cc500e..996c94598 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -17044,6 +17044,11 @@ async fn search_impl(
} else {
raw
};
+ // #1940 stores the embedding as one `xerj:f32:` string.
+ // `fields` reports the vector the caller named, which
+ // is the number array — including `"_source": false`,
+ // where the engine keeps the packed source on purpose.
+ let raw = raw.map(expand_packed_embedding_value);
let arr_raw: Vec<Value> = match raw {
Some(Value::Array(a)) => a,
Some(Value::Null) | None => continue,
@@ -17515,6 +17520,7 @@ async fn search_impl(
other => {
if is_stored(other) {
if let Some(raw) = get_source_value_by_path(&h.source, other) {
+ let raw = expand_packed_embedding_value(raw);
let arr = match raw {
Value::Array(a) => a,
Value::Null => continue,
@@ -17561,6 +17567,7 @@ async fn search_impl(
.or_else(|| target_via_alias.and_then(|t| get_source_value_by_path(&h.source, t)))
.or_else(|| get_source_value_by_path(&h.source, &response_key));
if let Some(raw) = raw {
+ let raw = expand_packed_embedding_value(raw);
let arr = match raw {
Value::Array(a) => a,
Value::Null => vec![],
@@ -42517,6 +42524,9 @@ pub(crate) fn build_docvalue_fields(
let base = field_name.strip_suffix(".keyword")?;
get_source_value_by_path(source, base)
});
+ // Same packed-embedding expansion as `fields` (#1940). Doc values
+ // skip the string, so this path is the one that still sees it.
+ let raw = raw.map(expand_packed_embedding_value);
let arr: Vec<Value> = match raw {
Some(Value::Array(a)) => a,
Some(Value::Null) | None => vec![],
@@ -42676,6 +42686,14 @@ pub(crate) fn get_field_value_via_walk(source: &Value, segs: &[&str]) -> Option<
walk(source, segs)
}
+/// Expand a packed embedding (`xerj:f32:` hex) into the number array
+/// `fields` and `docvalue_fields` have always returned. Leaves every other
+/// value unchanged, including a vector that is already a number array.
+fn expand_packed_embedding_value(mut value: Value) -> Value {
+ xerj_common::packed_f32::expand_packed_f32_values(&mut value);
+ value
+}
+
pub(crate) fn get_source_value_by_path(source: &Value, path: &str) -> Option<Value> {
if let Some(v) = source.get(path) {
return Some(v.clone());
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index 5aa7aee58..04afcfd34 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -58,6 +58,7 @@ pub mod hub;
pub mod localauth;
pub mod metrics;
pub mod net;
+pub mod packed_f32;
pub mod resource;
pub mod schema;
pub mod thread_spawn;
diff --git a/engine/crates/xerj-common/src/packed_f32.rs b/engine/crates/xerj-common/src/packed_f32.rs
new file mode 100644
index 000000000..c30278a74
--- /dev/null
+++ b/engine/crates/xerj-common/src/packed_f32.rs
@@ -0,0 +1,201 @@
+//! Compact stand-in for a JSON array of finite `f32`s.
+//!
+//! A 384-d embedding stored as `serde_json::Value::Number`s is one `Value`
+//! per component. On the lexical embedder that is what `semantic_text`
+//! writes into every autoindexed record, and both memtables retain that
+//! tree until flush. Measured on the #1940 corpus (5,000 records, ~195
+//! bytes each) the parsed tree was ~43 KB/doc in the stored-values cache
+//! and ~78 KB/doc of RSS growth during `index`, against ~1.5 KB of actual
+//! `f32` payload.
+//!
+//! The packed form is a single string, `xerj:f32:` plus lowercase hex of
+//! the little-endian `f32` bits. It round-trips bit-exactly, survives the
+//! stored-section JSON codec, and expands back to a number array only when
+//! a response actually returns the field.
+
+use serde_json::Value;
+
+pub const PACKED_F32_PREFIX: &str = "xerj:f32:";
+
+const HEX: &[u8; 16] = b"0123456789abcdef";
+
+/// Pack `values` when every element is finite and the slice is non-empty.
+pub fn pack_f32_vector(values: &[f32]) -> Option<String> {
+ if values.is_empty() || values.iter().any(|value| !value.is_finite()) {
+ return None;
+ }
+ let mut out = String::with_capacity(PACKED_F32_PREFIX.len() + values.len() * 8);
+ out.push_str(PACKED_F32_PREFIX);
+ for value in values {
+ for byte in value.to_le_bytes() {
+ out.push(HEX[(byte >> 4) as usize] as char);
+ out.push(HEX[(byte & 0x0f) as usize] as char);
+ }
+ }
+ Some(out)
+}
+
+pub fn is_packed_f32(value: &str) -> bool {
+ value.starts_with(PACKED_F32_PREFIX)
+ && value.len() > PACKED_F32_PREFIX.len()
+ && (value.len() - PACKED_F32_PREFIX.len()).is_multiple_of(8)
+}
+
+/// Decode a packed vector. `None` for any other string, a bad nibble, a
+/// non-multiple-of-8 payload, or a non-finite element.
+pub fn unpack_f32_vector(value: &str) -> Option<Vec<f32>> {
+ let rest = value.strip_prefix(PACKED_F32_PREFIX)?;
+ if rest.is_empty() || !rest.len().is_multiple_of(8) {
+ return None;
+ }
+ let mut out = Vec::with_capacity(rest.len() / 8);
+ let bytes = rest.as_bytes();
+ for chunk in bytes.chunks_exact(8) {
+ let mut le = [0u8; 4];
+ for (i, pair) in chunk.chunks_exact(2).enumerate() {
+ le[i] = (hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?;
+ }
+ let decoded = f32::from_le_bytes(le);
+ if !decoded.is_finite() {
+ return None;
+ }
+ out.push(decoded);
+ }
+ Some(out)
+}
+
+fn hex_nibble(byte: u8) -> Option<u8> {
+ match byte {
+ b'0'..=b'9' => Some(byte - b'0'),
+ b'a'..=b'f' => Some(byte - b'a' + 10),
+ b'A'..=b'F' => Some(byte - b'A' + 10),
+ _ => None,
+ }
+}
+
+/// A JSON number array, or a packed string, as finite `f32`s.
+///
+/// Number arrays use the same `as_f64() as f32` conversion the exact kNN
+/// scan has always used, which is bit-exact for every finite `f32`.
+pub fn json_to_f32_vector(value: &Value) -> Option<Vec<f32>> {
+ match value {
+ Value::String(text) => unpack_f32_vector(text),
+ // An empty array is a present-but-empty vector. The kNN projection
+ // treats that as supported (`Some(vec![])`), distinct from null and
+ // from a shape it cannot score. Packing still refuses it.
+ Value::Array(items) => {
+ let mut out = Vec::with_capacity(items.len());
+ for item in items {
+ let decoded = item.as_f64()? as f32;
+ if !decoded.is_finite() {
+ return None;
+ }
+ out.push(decoded);
+ }
+ Some(out)
+ }
+ _ => None,
+ }
+}
+
+/// Replace a number array (or an array of number arrays) with the packed
+/// string form, in place. Leaves every other shape untouched, including a
+/// value that is already packed and a vector containing a non-finite
+/// element.
+pub fn compact_vector_value(value: &mut Value) {
+ let Some(packed) = try_compact(value) else {
+ return;
+ };
+ *value = packed;
+}
+
+fn try_compact(value: &Value) -> Option<Value> {
+ let Value::Array(items) = value else {
+ return None;
+ };
+ if items.is_empty() {
+ return None;
+ }
+ if items.iter().all(Value::is_number) {
+ return pack_f32_vector(&json_to_f32_vector(value)?).map(Value::String);
+ }
+ if items.iter().all(|item| {
+ item.as_array()
+ .is_some_and(|inner| !inner.is_empty() && inner.iter().all(Value::is_number))
+ }) {
+ let mut chunks = Vec::with_capacity(items.len());
+ for item in items {
+ chunks.push(Value::String(pack_f32_vector(&json_to_f32_vector(item)?)?));
+ }
+ return Some(Value::Array(chunks));
+ }
+ None
+}
+
+/// Expand packed strings back into JSON number arrays, recursively.
+///
+/// Response paths call this on the page (or the one document) they are
+/// about to return. The memtable and the stored-values cache keep the
+/// packed form.
+pub fn expand_packed_f32_values(value: &mut Value) {
+ if let Value::String(text) = value {
+ if let Some(floats) = unpack_f32_vector(text) {
+ *value = Value::Array(floats.into_iter().map(|f| Value::from(f as f64)).collect());
+ }
+ return;
+ }
+ match value {
+ Value::Array(items) => {
+ for item in items {
+ expand_packed_f32_values(item);
+ }
+ }
+ Value::Object(map) => {
+ for child in map.values_mut() {
+ expand_packed_f32_values(child);
+ }
+ }
+ _ => {}
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn round_trip_preserves_f32_bits() {
+ let values = [0.0_f32, -0.0, 1.0, -1.5, 1.0e-20, f32::MIN, f32::MAX];
+ let packed = pack_f32_vector(&values).unwrap();
+ assert!(packed.starts_with(PACKED_F32_PREFIX));
+ assert_eq!(unpack_f32_vector(&packed).unwrap(), values);
+ }
+
+ #[test]
+ fn rejects_non_finite_and_foreign_strings() {
+ assert!(pack_f32_vector(&[]).is_none());
+ assert!(pack_f32_vector(&[f32::NAN]).is_none());
+ assert!(pack_f32_vector(&[f32::INFINITY]).is_none());
+ assert!(unpack_f32_vector("xerj:f32:zz").is_none());
+ assert!(unpack_f32_vector("hello").is_none());
+ assert!(unpack_f32_vector("xerj:f32:0000000").is_none());
+ }
+
+ #[test]
+ fn compact_then_expand_matches_the_number_array() {
+ let mut pooled = json!([0.25, -2.0, 4.0]);
+ compact_vector_value(&mut pooled);
+ assert!(pooled.as_str().unwrap().starts_with(PACKED_F32_PREFIX));
+ expand_packed_f32_values(&mut pooled);
+ let back = json_to_f32_vector(&pooled).unwrap();
+ assert_eq!(back, vec![0.25, -2.0, 4.0]);
+
+ let mut chunks = json!([[1.0, 0.0], [0.0, 1.0]]);
+ compact_vector_value(&mut chunks);
+ assert!(chunks.as_array().unwrap()[0].is_string());
+ expand_packed_f32_values(&mut chunks);
+ assert_eq!(chunks[0].as_array().unwrap().len(), 2);
+ assert_eq!(chunks[1].as_array().unwrap().len(), 2);
+ }
+}
diff --git a/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs b/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs
index f134843b5..b318bd200 100644
--- a/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs
+++ b/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs
@@ -239,3 +239,91 @@ async fn overwrite_flush_and_merge_churn_returns_to_a_bounded_baseline() {
assert_eq!(index.memtable_bytes(), 0);
assert_eq!(index.live_doc_count(), 1);
}
+
+/// #1940: a small semantic_text record must not retain one `serde_json::Value`
+/// per embedding component. 32 docs at 384-d is enough to prove the per-doc
+/// bound; the old tree was `384 * size_of::<Value>()` before the rest of the
+/// object. Exact kNN after flush checks the packed form still scores, and
+/// `_source: true` / GET still return a number array.
+#[tokio::test]
+async fn semantic_log_records_retain_packed_vectors_not_value_arrays() {
+ let name = "packed-semantic-rss";
+ let dir = TempDir::new().expect("temp data directory");
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = Engine::new(config).expect("engine");
+ let mut schema = Schema::empty();
+ let mut content = FieldConfig::new("content", FieldType::Text);
+ content.options.dimensions = Some(384);
+ content.embedding = Some(EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: Some("content_vector".into()),
+ });
+ schema.fields.push(content);
+ engine.create_index(name, schema).expect("create index");
+ engine
+ .put_index_mapping(
+ name,
+ json!({"properties": {"content": {"type": "semantic_text"}}}),
+ )
+ .unwrap();
+
+ let index = engine.get_index(name).unwrap();
+ const N: usize = 32;
+ for i in 0..N {
+ index
+ .index_document(
+ Some(format!("d{i}")),
+ json!({"content": format!("segment queue header field flush buffer {i}")}),
+ )
+ .await
+ .unwrap();
+ }
+
+ let heaps = index.memtable_source_heaps_for_test();
+ assert_eq!(heaps.len(), N);
+ let expanded_floor = 384 * std::mem::size_of::<Value>();
+ let worst = heaps.iter().copied().max().unwrap_or(0);
+ assert!(
+ worst < expanded_floor,
+ "retained source heap {worst} bytes is at least one expanded 384-d Value array ({expanded_floor})"
+ );
+ // Packed hex of 384 f32s is 9 + 3072 bytes, plus the short log object.
+ assert!(
+ worst < 8 * 1024,
+ "retained source heap {worst} bytes exceeds 8 KB"
+ );
+
+ index.flush().await.unwrap();
+ assert_eq!(index.memtable_bytes(), 0);
+
+ let query_vector =
+ xerj_ai::local::local_embed("segment queue header field flush buffer 0", 384);
+ let request = xerj_query::parse_request(&json!({
+ "query": {
+ "knn": {
+ "field": "content_vector",
+ "query_vector": query_vector,
+ "k": 1,
+ "num_candidates": N,
+ "boost": 1.0
+ }
+ },
+ "size": 1,
+ "_source": true
+ }))
+ .unwrap();
+ let result = index.search(&request).await.unwrap();
+ assert_eq!(result.hits.len(), 1, "exact kNN must still answer");
+ assert_eq!(result.hits[0].id, "d0");
+ let returned = result.hits[0]
+ .source
+ .get("content_vector")
+ .and_then(Value::as_array)
+ .expect("search _source returns the embedding as a number array");
+ assert_eq!(returned.len(), 384);
+
+ let fetched = index.get_document("d0").await.unwrap().unwrap();
+ assert_eq!(fetched["content_vector"].as_array().unwrap().len(), 384);
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 7c897dc3f..e2f2989a9 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -11599,17 +11599,21 @@ impl Index {
}
// Vector indexing is only meaningful when at least one document
- // in the batch carries an array of numbers. Scan until the first
- // candidate instead of inspecting only processed[0]: mixed batches
- // may legitimately begin with an ordinary text/log document.
+ // in the batch carries a vector. Scan until the first candidate
+ // instead of inspecting only processed[0]: mixed batches may
+ // legitimately begin with an ordinary text/log document. A packed
+ // `xerj:f32:` string is that vector (#1940); treating only number
+ // arrays as candidates skipped HNSW for every semantic bulk batch.
if processed.iter().any(|r| {
r.source
.as_object()
.map(|obj| {
obj.values().any(|v| {
- v.as_array()
- .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))
- .unwrap_or(false)
+ v.as_str()
+ .is_some_and(xerj_common::packed_f32::is_packed_f32)
+ || v.as_array()
+ .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))
+ .unwrap_or(false)
})
})
.unwrap_or(false)
@@ -14976,9 +14980,11 @@ impl Index {
// (see the load gate in `Index::open`), so a live graph always has
// a pinned dense_vector field and takes the fast path above.)
let has_candidate = obj.values().any(|v| {
- v.as_array()
- .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))
- .unwrap_or(false)
+ v.as_str()
+ .is_some_and(xerj_common::packed_f32::is_packed_f32)
+ || v.as_array()
+ .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))
+ .unwrap_or(false)
});
if !has_candidate {
return;
@@ -15966,13 +15972,12 @@ impl Index {
if get_field_value(&src, &chunk_field).is_some() {
return None;
}
- let doc_vec: Vec<f32> = match get_field_value(&src, field) {
- Some(Value::Array(arr)) => arr
- .iter()
- .filter_map(|v| v.as_f64().map(|f| f as f32))
- .collect(),
- _ => return None,
- };
+ // Packed embeddings are the retained source form (#1940). A
+ // number array is still valid for segments written before that.
+ // `extract_numeric_vector` reads both. Any other shape, or a
+ // vector whose length is not the query's, is an extract anomaly:
+ // hand the whole request back to the exact scan.
+ let doc_vec = extract_numeric_vector(&src, field)?;
if doc_vec.len() != query_vec.len() {
return None;
}
@@ -17497,9 +17502,14 @@ impl Index {
/// Early-exits on the first hit; only invoked on an empty nested-knn result.
async fn index_has_any_nested_vector_at(&self, nested_path: &str, subfield: &str) -> bool {
fn elem_has_vector(elem: &Value, subfield: &str) -> bool {
- elem.get(subfield)
- .and_then(Value::as_array)
- .is_some_and(|a| !a.is_empty() && a.iter().all(Value::is_number))
+ match elem.get(subfield) {
+ Some(Value::String(text)) => xerj_common::packed_f32::unpack_f32_vector(text)
+ .is_some_and(|values| !values.is_empty()),
+ Some(value) => value
+ .as_array()
+ .is_some_and(|a| !a.is_empty() && a.iter().all(Value::is_number)),
+ None => false,
+ }
}
fn doc_has(source: &Value, nested_path: &str, subfield: &str) -> bool {
match get_field_value(source, nested_path) {
@@ -18165,8 +18175,20 @@ impl Index {
1 => chunk_vecs[0].clone(),
_ => mean_pool_normalize(&chunk_vecs),
};
- let arr: Vec<Value> = pooled.into_iter().map(|f| Value::from(f as f64)).collect();
- obj.insert(job.target.clone(), Value::Array(arr));
+ // #1940: a 384-d `Vec<Value>` is tens of KB of retained heap per
+ // record (two memtables plus any later stored-source parse) for
+ // ~1.5 KB of f32s. Keep the packed string in the source; readers
+ // unpack it, and response paths expand it back to a number array.
+ match xerj_common::packed_f32::pack_f32_vector(&pooled) {
+ Some(packed) => {
+ obj.insert(job.target.clone(), Value::String(packed));
+ }
+ None => {
+ let arr: Vec<Value> =
+ pooled.into_iter().map(|f| Value::from(f as f64)).collect();
+ obj.insert(job.target.clone(), Value::Array(arr));
+ }
+ }
// Compact, durable passage provenance. Text is deliberately NOT
// duplicated: the response reconstructs only the winning page
// slice from the original semantic field using these UTF-8 byte
@@ -18194,12 +18216,33 @@ impl Index {
if chunk_vecs.len() > 1 {
let chunks_json: Vec<Value> = chunk_vecs
.iter()
- .map(|cv| Value::Array(cv.iter().map(|f| Value::from(*f as f64)).collect()))
+ .map(|cv| match xerj_common::packed_f32::pack_f32_vector(cv) {
+ Some(packed) => Value::String(packed),
+ None => Value::Array(cv.iter().map(|f| Value::from(*f as f64)).collect()),
+ })
.collect();
obj.insert(format!("{}_chunks", job.target), Value::Array(chunks_json));
}
}
+ // Caller-supplied vectors skip the job loop above and would otherwise
+ // stay as number arrays — the same retained-heap shape. Pack those
+ // too, field by field, after every generated vector is in place.
+ for source in sources.iter_mut() {
+ let Some(obj) = source.as_object_mut() else {
+ continue;
+ };
+ for (_, target, _) in &specs {
+ if let Some(value) = obj.get_mut(target) {
+ xerj_common::packed_f32::compact_vector_value(value);
+ }
+ let chunks_name = format!("{target}_chunks");
+ if let Some(value) = obj.get_mut(&chunks_name) {
+ xerj_common::packed_f32::compact_vector_value(value);
+ }
+ }
+ }
+
sources
.into_iter()
.enumerate()
@@ -18801,6 +18844,9 @@ impl Index {
.iter()
.filter_map(|v| v.as_f64().map(|f| f as f32))
.collect(),
+ Value::String(text) => {
+ xerj_common::packed_f32::unpack_f32_vector(text).unwrap_or_default()
+ }
_ => continue,
};
if doc_vec.len() != query_vec.len() {
@@ -19407,6 +19453,9 @@ impl Index {
};
if let Ok(Some(source)) = &mut res {
strip_internal_passage_metadata(source);
+ // The stored source keeps embeddings packed (#1940). GET returns
+ // the number array callers already see.
+ xerj_common::packed_f32::expand_packed_f32_values(source);
}
match &res {
Ok(Some(_)) => {
@@ -26444,6 +26493,17 @@ impl Index {
self.memtable.size_bytes()
}
+ /// Retained JSON-heap estimate of every live memtable source. Test-only
+ /// so a regression can bound #1940 without sampling process RSS.
+ #[cfg(test)]
+ pub(crate) fn memtable_source_heaps_for_test(&self) -> Vec<usize> {
+ self.memtable
+ .all_docs_with_sources_arc()
+ .into_iter()
+ .map(|(_, source)| crate::ingest_memory::estimated_json_heap(&source))
+ .collect()
+ }
+
/// #948 — entries currently resident in the STORAGE memtable
/// (`IndexStore::memtable_shards`). Diagnostic twin of
/// [`Self::memtable_bytes`]: the FTS memtable drains on every flush,
@@ -27896,6 +27956,14 @@ fn build_doc_value_columns<'a>(
col.push(Some(v));
}
Value::String(s) => {
+ // A packed embedding is not a keyword term. Filing it
+ // would put one multi-kilobyte dictionary entry per
+ // document into the `.dv` sidecar (#1940). Absence of a
+ // column is the same outcome the number-array form
+ // already had: arrays are multi-valued and ship no column.
+ if xerj_common::packed_f32::is_packed_f32(s) {
+ continue;
+ }
if !keyword.contains_key(field) {
keyword.insert(field.clone(), Vec::new());
}
@@ -35650,8 +35718,20 @@ fn apply_source_filter_measured(
);
}
// The caller explicitly asked for the whole document — MORE than the
- // default projection, not less. Nothing was withheld.
- SourceFilter::Enabled(true) => return (hits, None),
+ // default projection, not less. Nothing was withheld. Expand packed
+ // embeddings here, on the page only, so the wire form stays a number
+ // array (#1940).
+ SourceFilter::Enabled(true) => {
+ return (
+ hits.into_iter()
+ .map(|mut hit| {
+ xerj_common::packed_f32::expand_packed_f32_values(&mut hit.source);
+ hit
+ })
+ .collect(),
+ None,
+ );
+ }
// `_source: false`: keep the raw source so the response layer can
// still resolve `fields` / `_ignored` / `highlight` against it —
// `_source` emission is suppressed in es_compat.rs.
@@ -35687,6 +35767,10 @@ fn apply_source_filter_measured(
.then(|| default_projection_len(&h.source, generated_companion_fields, mode));
let sentinels = take_collapse_sentinels(&h.source);
h.source = filter_object(&h.source, fields, &[]);
+ // Expand before the savings measurement so a caller who asked
+ // for the companion is billed for the number array on the wire,
+ // not the packed string it was stored as.
+ xerj_common::packed_f32::expand_packed_f32_values(&mut h.source);
if let Some(baseline) = baseline {
// Saturating: a caller who explicitly projects a generated
// companion gets MORE than the default, and more is not a
@@ -35704,6 +35788,7 @@ fn apply_source_filter_measured(
.then(|| default_projection_len(&h.source, generated_companion_fields, mode));
let sentinels = take_collapse_sentinels(&h.source);
h.source = filter_object(&h.source, includes, excludes);
+ xerj_common::packed_f32::expand_packed_f32_values(&mut h.source);
if let Some(baseline) = baseline {
acc.record(&h.id, baseline.saturating_sub(json_len(&h.source, mode)));
}
@@ -37866,6 +37951,15 @@ fn stored_chunk_vectors(source: &Value, vector_field: &str) -> Vec<Vec<f32>> {
}
fn json_f32_vector(value: &Value) -> Option<Vec<f32>> {
+ // #1940 stores each chunk as a `xerj:f32:` string. A number array is
+ // still valid for segments written before that. Non-finite components
+ // in a number array keep the previous `as_f64() as f32` read.
+ if value
+ .as_str()
+ .is_some_and(xerj_common::packed_f32::is_packed_f32)
+ {
+ return xerj_common::packed_f32::unpack_f32_vector(value.as_str()?);
+ }
let row = value.as_array()?;
let mut vector = Vec::with_capacity(row.len());
for component in row {
@@ -42461,14 +42555,20 @@ fn push_field_root_keys(field: &str, out: &mut Vec<String>) -> bool {
/// Returns `None` when the field is absent, not an array, empty, or holds
/// any non-number element.
fn extract_numeric_vector(source: &Value, field: &str) -> Option<Vec<f32>> {
- match get_field_value(source, field) {
- Some(Value::Array(arr)) if !arr.is_empty() && arr.iter().all(|v| v.is_number()) => Some(
- arr.iter()
- .filter_map(|v| v.as_f64().map(|f| f as f32))
- .collect(),
- ),
- _ => None,
+ let value = get_field_value(source, field)?;
+ // Packed strings are the retained form (#1940). Number arrays remain
+ // valid for segments written before that, and for a caller that raced
+ // a vector in before compaction.
+ if value
+ .as_str()
+ .is_some_and(xerj_common::packed_f32::is_packed_f32)
+ || value
+ .as_array()
+ .is_some_and(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))
+ {
+ return xerj_common::packed_f32::json_to_f32_vector(&value);
}
+ None
}
/// Collect every dense_vector-mapped field in schema order (dotted paths
@@ -62468,6 +62568,29 @@ mod exact_scan_hydration_tests {
}
}
+#[cfg(test)]
+mod packed_vector_doc_values_tests {
+ use super::build_doc_value_columns;
+ use serde_json::json;
+
+ #[test]
+ fn packed_embedding_is_not_filed_as_a_keyword() {
+ let src = json!({
+ "msg": "hello",
+ "msg_vector": xerj_common::packed_f32::pack_f32_vector(&[1.0, 0.0, -1.0]).unwrap(),
+ });
+ let cols = build_doc_value_columns(
+ std::iter::once(Some(&src)),
+ &std::collections::HashSet::new(),
+ );
+ assert!(cols.contains_key("msg"));
+ assert!(
+ !cols.contains_key("msg_vector"),
+ "a packed embedding must not become a per-doc keyword term"
+ );
+ }
+}
+
/// #1504: a logs index's omitted `match` requires every token. The same
/// query on an index without the setting stays ES's OR, and an explicit
/// `"operator": "or"` on the logs index stays OR too.
diff --git a/engine/crates/xerj-engine/src/ingest_memory.rs b/engine/crates/xerj-engine/src/ingest_memory.rs
index 1c923ab94..5fa110f76 100644
--- a/engine/crates/xerj-engine/src/ingest_memory.rs
+++ b/engine/crates/xerj-engine/src/ingest_memory.rs
@@ -730,12 +730,27 @@ pub fn vector_value_capacity(
_ => 0,
}
}
+ fn retained_vector_bytes(value: &serde_json::Value) -> usize {
+ match value {
+ // Packed embeddings retain one string, not one Value per
+ // component (#1940). Count it or the ledger reports the vector
+ // as free. Number arrays still go through `arrays`.
+ serde_json::Value::String(text) if xerj_common::packed_f32::is_packed_f32(text) => {
+ text.capacity()
+ }
+ serde_json::Value::Array(values) => {
+ values.capacity() * std::mem::size_of::<serde_json::Value>()
+ + values.iter().map(retained_vector_bytes).sum::<usize>()
+ }
+ other => arrays(other),
+ }
+ }
match value {
serde_json::Value::Object(values) => values
.iter()
.map(|(key, value)| {
if fields.contains(key) {
- arrays(value)
+ retained_vector_bytes(value)
} else {
vector_value_capacity(value, fields)
}
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 213595bd8..7e83671a2 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -5158,15 +5158,25 @@ pub(crate) fn looks_like_multi_vector(val: &serde_json::Value) -> bool {
return false;
}
for row in rows {
- let Some(components) = row.as_array() else {
- return false;
+ let width = if let Some(text) = row.as_str() {
+ // Packed chunk vectors (#1940) are one string per passage, not
+ // a number array. Same rectangular rule as the array form.
+ match xerj_common::packed_f32::unpack_f32_vector(text) {
+ Some(values) => values.len(),
+ None => return false,
+ }
+ } else {
+ let Some(components) = row.as_array() else {
+ return false;
+ };
+ if components.is_empty() || !components.iter().all(serde_json::Value::is_number) {
+ return false;
+ }
+ components.len()
};
- if components.is_empty() || !components.iter().all(serde_json::Value::is_number) {
- return false;
- }
match dims {
- None => dims = Some(components.len()),
- Some(d) if d == components.len() => {}
+ None => dims = Some(width),
+ Some(d) if d == width => {}
Some(_) => return false,
}
}
diff --git a/engine/crates/xerj-engine/src/vector_column.rs b/engine/crates/xerj-engine/src/vector_column.rs
index e7b3aea19..559ceeec0 100644
--- a/engine/crates/xerj-engine/src/vector_column.rs
+++ b/engine/crates/xerj-engine/src/vector_column.rs
@@ -342,15 +342,22 @@ impl ColumnBuilder {
fn push_chunks(&mut self, chunks: &[Value]) -> DocVectors {
let first = self.column.entries.len();
for (position, chunk) in chunks.iter().enumerate() {
- let Value::Array(elements) = chunk else {
- continue;
- };
// The ordinal is the passage's position in the ORIGINAL array, so
// a skipped passage never renumbers the ones after it.
let Ok(ordinal) = u32::try_from(position) else {
continue;
};
- self.push_vector(elements, ordinal);
+ match chunk {
+ Value::Array(elements) => {
+ self.push_vector(elements, ordinal);
+ }
+ Value::String(text) => {
+ if let Some(values) = xerj_common::packed_f32::unpack_f32_vector(text) {
+ self.push_f32_vector(&values, ordinal);
+ }
+ }
+ _ => {}
+ }
}
DocVectors::Chunks {
first,
@@ -361,6 +368,12 @@ impl ColumnBuilder {
fn push_pooled(&mut self, source: &Value) -> DocVectors {
match field_value_cow(source, &self.field).as_deref() {
Some(Value::Array(elements)) => DocVectors::Pooled(self.push_vector(elements, 0)),
+ // #1940 packed form. Same f32 bits `push_vector` would have
+ // produced from the number array.
+ Some(Value::String(text)) => match xerj_common::packed_f32::unpack_f32_vector(text) {
+ Some(values) => DocVectors::Pooled(self.push_f32_vector(&values, 0)),
+ None => DocVectors::Absent,
+ },
_ => DocVectors::Absent,
}
}
diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rs
index 997c42bc7..dc00b129e 100644
--- a/engine/crates/xerj-storage/src/stored_codec.rs
+++ b/engine/crates/xerj-storage/src/stored_codec.rs
@@ -1510,18 +1510,6 @@ fn decode_projected_column<'a>(
Ok(decoded[index].as_ref().unwrap())
}
-fn finite_f32_vector(values: Vec<f64>) -> Option<Vec<f32>> {
- let mut out = Vec::with_capacity(values.len());
- for value in values {
- let value = value as f32;
- if !value.is_finite() {
- return None;
- }
- out.push(value);
- }
- Some(out)
-}
-
struct TypedVectorRowsVisitor<'a> {
expected_rows: usize,
column_index: usize,
@@ -1548,32 +1536,19 @@ impl<'de> serde::de::Visitor<'de> for TypedVectorRowsVisitor<'_> {
let Some(value) = seq.next_element::<serde_json::Value>()? else {
return Err(serde::de::Error::custom("vector column ended early"));
};
- let typed = match value {
- serde_json::Value::Null => None,
- serde_json::Value::Array(values) => {
- let values = values
- .into_iter()
- .map(|value| value.as_f64())
- .collect::<Option<Vec<_>>>()
- .and_then(finite_f32_vector);
- let Some(values) = values else {
+ // Null is a supported absence. A packed `xerj:f32:` string is the
+ // retained embedding form (#1940); a number array is the form
+ // segments wrote before that. Anything else cannot be scored as
+ // a vector and drops the whole column to the stored-document scan.
+ let typed = if value.is_null() {
+ None
+ } else {
+ match xerj_common::packed_f32::json_to_f32_vector(&value) {
+ Some(values) => Some(values),
+ None => {
supported = false;
- out.push(None);
- if checkpoint_decoded_row(
- row,
- self.column_index,
- self.checkpoint,
- self.cancelled,
- ) {
- return Err(serde::de::Error::custom("stored decode cancelled"));
- }
- continue;
- };
- Some(values)
- }
- _ => {
- supported = false;
- None
+ None
+ }
}
};
out.push(typed);
@@ -1649,18 +1624,10 @@ fn decode_typed_vector_rows_controlled(
.map_err(|error| StorageError::Other(anyhow::anyhow!("constant decode: {error}")))?
{
serde_json::Value::Null => None,
- serde_json::Value::Array(values) => {
- let Some(values) = values
- .into_iter()
- .map(|value| value.as_f64())
- .collect::<Option<Vec<_>>>()
- .and_then(finite_f32_vector)
- else {
- return Ok(Ok(None));
- };
- Some(values)
- }
- _ => return Ok(Ok(None)),
+ other => match xerj_common::packed_f32::json_to_f32_vector(&other) {
+ Some(values) => Some(values),
+ None => return Ok(Ok(None)),
+ },
};
return repeat_constant_rows_controlled(row, num_docs, column_index, checkpoint)
.map(|result| result.map(Some));
@@ -1733,15 +1700,7 @@ fn decode_typed_vector_chunk_rows_controlled(
))
})?;
for chunk in chunks {
- let serde_json::Value::Array(values) = chunk else {
- return Ok(Ok(None));
- };
- let Some(vector) = values
- .into_iter()
- .map(|value| value.as_f64())
- .collect::<Option<Vec<_>>>()
- .and_then(finite_f32_vector)
- else {
+ let Some(vector) = xerj_common::packed_f32::json_to_f32_vector(&chunk) else {
return Ok(Ok(None));
};
typed.push(vector);
@@ -1805,16 +1764,8 @@ fn parse_typed_chunk_reader<R: std::io::Read>(
serde_json::Value::Array(chunks) => {
let mut result = Vec::with_capacity(chunks.len());
for chunk in chunks {
- let serde_json::Value::Array(values) = chunk else {
- supported = false;
- continue;
- };
- let vector = values
- .into_iter()
- .map(|value| value.as_f64())
- .collect::<Option<Vec<_>>>()
- .and_then(finite_f32_vector);
- let Some(vector) = vector else {
+ let Some(vector) = xerj_common::packed_f32::json_to_f32_vector(&chunk)
+ else {
supported = false;
continue;
};