AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): pack semantic embeddings so JSONL autoindex RSS stays bounded (#2240)
xerj-orge15abb780f25
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex f07cc500e..996c94598 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -17044,6 +17044,11 @@ async fn search_impl(} else {raw};+ // #1940 stores the embedding as one `xerj:f32:` string.+ // `fields` reports the vector the caller named, which+ // is the number array — including `"_source": false`,+ // where the engine keeps the packed source on purpose.+ let raw = raw.map(expand_packed_embedding_value);let arr_raw: Vec<Value> = match raw {Some(Value::Array(a)) => a,Some(Value::Null) | None => continue,@@ -17515,6 +17520,7 @@ async fn search_impl(other => {if is_stored(other) {if let Some(raw) = get_source_value_by_path(&h.source, other) {+ let raw = expand_packed_embedding_value(raw);let arr = match raw {Value::Array(a) => a,Value::Null => continue,@@ -17561,6 +17567,7 @@ async fn search_impl(.or_else(|| target_via_alias.and_then(|t| get_source_value_by_path(&h.source, t))).or_else(|| get_source_value_by_path(&h.source, &response_key));if let Some(raw) = raw {+ let raw = expand_packed_embedding_value(raw);let arr = match raw {Value::Array(a) => a,Value::Null => vec![],@@ -42517,6 +42524,9 @@ pub(crate) fn build_docvalue_fields(let base = field_name.strip_suffix(".keyword")?;get_source_value_by_path(source, base)});+ // Same packed-embedding expansion as `fields` (#1940). Doc values+ // skip the string, so this path is the one that still sees it.+ let raw = raw.map(expand_packed_embedding_value);let arr: Vec<Value> = match raw {Some(Value::Array(a)) => a,Some(Value::Null) | None => vec![],@@ -42676,6 +42686,14 @@ pub(crate) fn get_field_value_via_walk(source: &Value, segs: &[&str]) -> Option<walk(source, segs)}+/// Expand a packed embedding (`xerj:f32:` hex) into the number array+/// `fields` and `docvalue_fields` have always returned. Leaves every other+/// value unchanged, including a vector that is already a number array.+fn expand_packed_embedding_value(mut value: Value) -> Value {+ xerj_common::packed_f32::expand_packed_f32_values(&mut value);+ value+}+pub(crate) fn get_source_value_by_path(source: &Value, path: &str) -> Option<Value> {if let Some(v) = source.get(path) {return Some(v.clone());diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rsindex 5aa7aee58..04afcfd34 100644--- a/engine/crates/xerj-common/src/lib.rs+++ b/engine/crates/xerj-common/src/lib.rs@@ -58,6 +58,7 @@ pub mod hub;pub mod localauth;pub mod metrics;pub mod net;+pub mod packed_f32;pub mod resource;pub mod schema;pub mod thread_spawn;diff --git a/engine/crates/xerj-common/src/packed_f32.rs b/engine/crates/xerj-common/src/packed_f32.rsnew file mode 100644index 000000000..c30278a74--- /dev/null+++ b/engine/crates/xerj-common/src/packed_f32.rs@@ -0,0 +1,201 @@+//! Compact stand-in for a JSON array of finite `f32`s.+//!+//! A 384-d embedding stored as `serde_json::Value::Number`s is one `Value`+//! per component. On the lexical embedder that is what `semantic_text`+//! writes into every autoindexed record, and both memtables retain that+//! tree until flush. Measured on the #1940 corpus (5,000 records, ~195+//! bytes each) the parsed tree was ~43 KB/doc in the stored-values cache+//! and ~78 KB/doc of RSS growth during `index`, against ~1.5 KB of actual+//! `f32` payload.+//!+//! The packed form is a single string, `xerj:f32:` plus lowercase hex of+//! the little-endian `f32` bits. It round-trips bit-exactly, survives the+//! stored-section JSON codec, and expands back to a number array only when+//! a response actually returns the field.++use serde_json::Value;++pub const PACKED_F32_PREFIX: &str = "xerj:f32:";++const HEX: &[u8; 16] = b"0123456789abcdef";++/// Pack `values` when every element is finite and the slice is non-empty.+pub fn pack_f32_vector(values: &[f32]) -> Option<String> {+ if values.is_empty() || values.iter().any(|value| !value.is_finite()) {+ return None;+ }+ let mut out = String::with_capacity(PACKED_F32_PREFIX.len() + values.len() * 8);+ out.push_str(PACKED_F32_PREFIX);+ for value in values {+ for byte in value.to_le_bytes() {+ out.push(HEX[(byte >> 4) as usize] as char);+ out.push(HEX[(byte & 0x0f) as usize] as char);+ }+ }+ Some(out)+}++pub fn is_packed_f32(value: &str) -> bool {+ value.starts_with(PACKED_F32_PREFIX)+ && value.len() > PACKED_F32_PREFIX.len()+ && (value.len() - PACKED_F32_PREFIX.len()).is_multiple_of(8)+}++/// Decode a packed vector. `None` for any other string, a bad nibble, a+/// non-multiple-of-8 payload, or a non-finite element.+pub fn unpack_f32_vector(value: &str) -> Option<Vec<f32>> {+ let rest = value.strip_prefix(PACKED_F32_PREFIX)?;+ if rest.is_empty() || !rest.len().is_multiple_of(8) {+ return None;+ }+ let mut out = Vec::with_capacity(rest.len() / 8);+ let bytes = rest.as_bytes();+ for chunk in bytes.chunks_exact(8) {+ let mut le = [0u8; 4];+ for (i, pair) in chunk.chunks_exact(2).enumerate() {+ le[i] = (hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?;+ }+ let decoded = f32::from_le_bytes(le);+ if !decoded.is_finite() {+ return None;+ }+ out.push(decoded);+ }+ Some(out)+}++fn hex_nibble(byte: u8) -> Option<u8> {+ match byte {+ b'0'..=b'9' => Some(byte - b'0'),+ b'a'..=b'f' => Some(byte - b'a' + 10),+ b'A'..=b'F' => Some(byte - b'A' + 10),+ _ => None,+ }+}++/// A JSON number array, or a packed string, as finite `f32`s.+///+/// Number arrays use the same `as_f64() as f32` conversion the exact kNN+/// scan has always used, which is bit-exact for every finite `f32`.+pub fn json_to_f32_vector(value: &Value) -> Option<Vec<f32>> {+ match value {+ Value::String(text) => unpack_f32_vector(text),+ // An empty array is a present-but-empty vector. The kNN projection+ // treats that as supported (`Some(vec![])`), distinct from null and+ // from a shape it cannot score. Packing still refuses it.+ Value::Array(items) => {+ let mut out = Vec::with_capacity(items.len());+ for item in items {+ let decoded = item.as_f64()? as f32;+ if !decoded.is_finite() {+ return None;+ }+ out.push(decoded);+ }+ Some(out)+ }+ _ => None,+ }+}++/// Replace a number array (or an array of number arrays) with the packed+/// string form, in place. Leaves every other shape untouched, including a+/// value that is already packed and a vector containing a non-finite+/// element.+pub fn compact_vector_value(value: &mut Value) {+ let Some(packed) = try_compact(value) else {+ return;+ };+ *value = packed;+}++fn try_compact(value: &Value) -> Option<Value> {+ let Value::Array(items) = value else {+ return None;+ };+ if items.is_empty() {+ return None;+ }+ if items.iter().all(Value::is_number) {+ return pack_f32_vector(&json_to_f32_vector(value)?).map(Value::String);+ }+ if items.iter().all(|item| {+ item.as_array()+ .is_some_and(|inner| !inner.is_empty() && inner.iter().all(Value::is_number))+ }) {+ let mut chunks = Vec::with_capacity(items.len());+ for item in items {+ chunks.push(Value::String(pack_f32_vector(&json_to_f32_vector(item)?)?));+ }+ return Some(Value::Array(chunks));+ }+ None+}++/// Expand packed strings back into JSON number arrays, recursively.+///+/// Response paths call this on the page (or the one document) they are+/// about to return. The memtable and the stored-values cache keep the+/// packed form.+pub fn expand_packed_f32_values(value: &mut Value) {+ if let Value::String(text) = value {+ if let Some(floats) = unpack_f32_vector(text) {+ *value = Value::Array(floats.into_iter().map(|f| Value::from(f as f64)).collect());+ }+ return;+ }+ match value {+ Value::Array(items) => {+ for item in items {+ expand_packed_f32_values(item);+ }+ }+ Value::Object(map) => {+ for child in map.values_mut() {+ expand_packed_f32_values(child);+ }+ }+ _ => {}+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn round_trip_preserves_f32_bits() {+ let values = [0.0_f32, -0.0, 1.0, -1.5, 1.0e-20, f32::MIN, f32::MAX];+ let packed = pack_f32_vector(&values).unwrap();+ assert!(packed.starts_with(PACKED_F32_PREFIX));+ assert_eq!(unpack_f32_vector(&packed).unwrap(), values);+ }++ #[test]+ fn rejects_non_finite_and_foreign_strings() {+ assert!(pack_f32_vector(&[]).is_none());+ assert!(pack_f32_vector(&[f32::NAN]).is_none());+ assert!(pack_f32_vector(&[f32::INFINITY]).is_none());+ assert!(unpack_f32_vector("xerj:f32:zz").is_none());+ assert!(unpack_f32_vector("hello").is_none());+ assert!(unpack_f32_vector("xerj:f32:0000000").is_none());+ }++ #[test]+ fn compact_then_expand_matches_the_number_array() {+ let mut pooled = json!([0.25, -2.0, 4.0]);+ compact_vector_value(&mut pooled);+ assert!(pooled.as_str().unwrap().starts_with(PACKED_F32_PREFIX));+ expand_packed_f32_values(&mut pooled);+ let back = json_to_f32_vector(&pooled).unwrap();+ assert_eq!(back, vec![0.25, -2.0, 4.0]);++ let mut chunks = json!([[1.0, 0.0], [0.0, 1.0]]);+ compact_vector_value(&mut chunks);+ assert!(chunks.as_array().unwrap()[0].is_string());+ expand_packed_f32_values(&mut chunks);+ assert_eq!(chunks[0].as_array().unwrap().len(), 2);+ assert_eq!(chunks[1].as_array().unwrap().len(), 2);+ }+}diff --git a/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs b/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rsindex f134843b5..b318bd200 100644--- a/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs+++ b/engine/crates/xerj-engine/src/bounded_ingest_resources_tests.rs@@ -239,3 +239,91 @@ async fn overwrite_flush_and_merge_churn_returns_to_a_bounded_baseline() {assert_eq!(index.memtable_bytes(), 0);assert_eq!(index.live_doc_count(), 1);}++/// #1940: a small semantic_text record must not retain one `serde_json::Value`+/// per embedding component. 32 docs at 384-d is enough to prove the per-doc+/// bound; the old tree was `384 * size_of::<Value>()` before the rest of the+/// object. Exact kNN after flush checks the packed form still scores, and+/// `_source: true` / GET still return a number array.+#[tokio::test]+async fn semantic_log_records_retain_packed_vectors_not_value_arrays() {+ let name = "packed-semantic-rss";+ let dir = TempDir::new().expect("temp data directory");+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = Engine::new(config).expect("engine");+ let mut schema = Schema::empty();+ let mut content = FieldConfig::new("content", FieldType::Text);+ content.options.dimensions = Some(384);+ content.embedding = Some(EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("content_vector".into()),+ });+ schema.fields.push(content);+ engine.create_index(name, schema).expect("create index");+ engine+ .put_index_mapping(+ name,+ json!({"properties": {"content": {"type": "semantic_text"}}}),+ )+ .unwrap();++ let index = engine.get_index(name).unwrap();+ const N: usize = 32;+ for i in 0..N {+ index+ .index_document(+ Some(format!("d{i}")),+ json!({"content": format!("segment queue header field flush buffer {i}")}),+ )+ .await+ .unwrap();+ }++ let heaps = index.memtable_source_heaps_for_test();+ assert_eq!(heaps.len(), N);+ let expanded_floor = 384 * std::mem::size_of::<Value>();+ let worst = heaps.iter().copied().max().unwrap_or(0);+ assert!(+ worst < expanded_floor,+ "retained source heap {worst} bytes is at least one expanded 384-d Value array ({expanded_floor})"+ );+ // Packed hex of 384 f32s is 9 + 3072 bytes, plus the short log object.+ assert!(+ worst < 8 * 1024,+ "retained source heap {worst} bytes exceeds 8 KB"+ );++ index.flush().await.unwrap();+ assert_eq!(index.memtable_bytes(), 0);++ let query_vector =+ xerj_ai::local::local_embed("segment queue header field flush buffer 0", 384);+ let request = xerj_query::parse_request(&json!({+ "query": {+ "knn": {+ "field": "content_vector",+ "query_vector": query_vector,+ "k": 1,+ "num_candidates": N,+ "boost": 1.0+ }+ },+ "size": 1,+ "_source": true+ }))+ .unwrap();+ let result = index.search(&request).await.unwrap();+ assert_eq!(result.hits.len(), 1, "exact kNN must still answer");+ assert_eq!(result.hits[0].id, "d0");+ let returned = result.hits[0]+ .source+ .get("content_vector")+ .and_then(Value::as_array)+ .expect("search _source returns the embedding as a number array");+ assert_eq!(returned.len(), 384);++ let fetched = index.get_document("d0").await.unwrap().unwrap();+ assert_eq!(fetched["content_vector"].as_array().unwrap().len(), 384);+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 7c897dc3f..e2f2989a9 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -11599,17 +11599,21 @@ impl Index {}// Vector indexing is only meaningful when at least one document- // in the batch carries an array of numbers. Scan until the first- // candidate instead of inspecting only processed[0]: mixed batches- // may legitimately begin with an ordinary text/log document.+ // in the batch carries a vector. Scan until the first candidate+ // instead of inspecting only processed[0]: mixed batches may+ // legitimately begin with an ordinary text/log document. A packed+ // `xerj:f32:` string is that vector (#1940); treating only number+ // arrays as candidates skipped HNSW for every semantic bulk batch.if processed.iter().any(|r| {r.source.as_object().map(|obj| {obj.values().any(|v| {- v.as_array()- .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))- .unwrap_or(false)+ v.as_str()+ .is_some_and(xerj_common::packed_f32::is_packed_f32)+ || v.as_array()+ .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))+ .unwrap_or(false)})}).unwrap_or(false)@@ -14976,9 +14980,11 @@ impl Index {// (see the load gate in `Index::open`), so a live graph always has// a pinned dense_vector field and takes the fast path above.)let has_candidate = obj.values().any(|v| {- v.as_array()- .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))- .unwrap_or(false)+ v.as_str()+ .is_some_and(xerj_common::packed_f32::is_packed_f32)+ || v.as_array()+ .map(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))+ .unwrap_or(false)});if !has_candidate {return;@@ -15966,13 +15972,12 @@ impl Index {if get_field_value(&src, &chunk_field).is_some() {return None;}- let doc_vec: Vec<f32> = match get_field_value(&src, field) {- Some(Value::Array(arr)) => arr- .iter()- .filter_map(|v| v.as_f64().map(|f| f as f32))- .collect(),- _ => return None,- };+ // Packed embeddings are the retained source form (#1940). A+ // number array is still valid for segments written before that.+ // `extract_numeric_vector` reads both. Any other shape, or a+ // vector whose length is not the query's, is an extract anomaly:+ // hand the whole request back to the exact scan.+ let doc_vec = extract_numeric_vector(&src, field)?;if doc_vec.len() != query_vec.len() {return None;}@@ -17497,9 +17502,14 @@ impl Index {/// Early-exits on the first hit; only invoked on an empty nested-knn result.async fn index_has_any_nested_vector_at(&self, nested_path: &str, subfield: &str) -> bool {fn elem_has_vector(elem: &Value, subfield: &str) -> bool {- elem.get(subfield)- .and_then(Value::as_array)- .is_some_and(|a| !a.is_empty() && a.iter().all(Value::is_number))+ match elem.get(subfield) {+ Some(Value::String(text)) => xerj_common::packed_f32::unpack_f32_vector(text)+ .is_some_and(|values| !values.is_empty()),+ Some(value) => value+ .as_array()+ .is_some_and(|a| !a.is_empty() && a.iter().all(Value::is_number)),+ None => false,+ }}fn doc_has(source: &Value, nested_path: &str, subfield: &str) -> bool {match get_field_value(source, nested_path) {@@ -18165,8 +18175,20 @@ impl Index {1 => chunk_vecs[0].clone(),_ => mean_pool_normalize(&chunk_vecs),};- let arr: Vec<Value> = pooled.into_iter().map(|f| Value::from(f as f64)).collect();- obj.insert(job.target.clone(), Value::Array(arr));+ // #1940: a 384-d `Vec<Value>` is tens of KB of retained heap per+ // record (two memtables plus any later stored-source parse) for+ // ~1.5 KB of f32s. Keep the packed string in the source; readers+ // unpack it, and response paths expand it back to a number array.+ match xerj_common::packed_f32::pack_f32_vector(&pooled) {+ Some(packed) => {+ obj.insert(job.target.clone(), Value::String(packed));+ }+ None => {+ let arr: Vec<Value> =+ pooled.into_iter().map(|f| Value::from(f as f64)).collect();+ obj.insert(job.target.clone(), Value::Array(arr));+ }+ }// Compact, durable passage provenance. Text is deliberately NOT// duplicated: the response reconstructs only the winning page// slice from the original semantic field using these UTF-8 byte@@ -18194,12 +18216,33 @@ impl Index {if chunk_vecs.len() > 1 {let chunks_json: Vec<Value> = chunk_vecs.iter()- .map(|cv| Value::Array(cv.iter().map(|f| Value::from(*f as f64)).collect()))+ .map(|cv| match xerj_common::packed_f32::pack_f32_vector(cv) {+ Some(packed) => Value::String(packed),+ None => Value::Array(cv.iter().map(|f| Value::from(*f as f64)).collect()),+ }).collect();obj.insert(format!("{}_chunks", job.target), Value::Array(chunks_json));}}+ // Caller-supplied vectors skip the job loop above and would otherwise+ // stay as number arrays — the same retained-heap shape. Pack those+ // too, field by field, after every generated vector is in place.+ for source in sources.iter_mut() {+ let Some(obj) = source.as_object_mut() else {+ continue;+ };+ for (_, target, _) in &specs {+ if let Some(value) = obj.get_mut(target) {+ xerj_common::packed_f32::compact_vector_value(value);+ }+ let chunks_name = format!("{target}_chunks");+ if let Some(value) = obj.get_mut(&chunks_name) {+ xerj_common::packed_f32::compact_vector_value(value);+ }+ }+ }+sources.into_iter().enumerate()@@ -18801,6 +18844,9 @@ impl Index {.iter().filter_map(|v| v.as_f64().map(|f| f as f32)).collect(),+ Value::String(text) => {+ xerj_common::packed_f32::unpack_f32_vector(text).unwrap_or_default()+ }_ => continue,};if doc_vec.len() != query_vec.len() {@@ -19407,6 +19453,9 @@ impl Index {};if let Ok(Some(source)) = &mut res {strip_internal_passage_metadata(source);+ // The stored source keeps embeddings packed (#1940). GET returns+ // the number array callers already see.+ xerj_common::packed_f32::expand_packed_f32_values(source);}match &res {Ok(Some(_)) => {@@ -26444,6 +26493,17 @@ impl Index {self.memtable.size_bytes()}+ /// Retained JSON-heap estimate of every live memtable source. Test-only+ /// so a regression can bound #1940 without sampling process RSS.+ #[cfg(test)]+ pub(crate) fn memtable_source_heaps_for_test(&self) -> Vec<usize> {+ self.memtable+ .all_docs_with_sources_arc()+ .into_iter()+ .map(|(_, source)| crate::ingest_memory::estimated_json_heap(&source))+ .collect()+ }+/// #948 — entries currently resident in the STORAGE memtable/// (`IndexStore::memtable_shards`). Diagnostic twin of/// [`Self::memtable_bytes`]: the FTS memtable drains on every flush,@@ -27896,6 +27956,14 @@ fn build_doc_value_columns<'a>(col.push(Some(v));}Value::String(s) => {+ // A packed embedding is not a keyword term. Filing it+ // would put one multi-kilobyte dictionary entry per+ // document into the `.dv` sidecar (#1940). Absence of a+ // column is the same outcome the number-array form+ // already had: arrays are multi-valued and ship no column.+ if xerj_common::packed_f32::is_packed_f32(s) {+ continue;+ }if !keyword.contains_key(field) {keyword.insert(field.clone(), Vec::new());}@@ -35650,8 +35718,20 @@ fn apply_source_filter_measured();}// The caller explicitly asked for the whole document — MORE than the- // default projection, not less. Nothing was withheld.- SourceFilter::Enabled(true) => return (hits, None),+ // default projection, not less. Nothing was withheld. Expand packed+ // embeddings here, on the page only, so the wire form stays a number+ // array (#1940).+ SourceFilter::Enabled(true) => {+ return (+ hits.into_iter()+ .map(|mut hit| {+ xerj_common::packed_f32::expand_packed_f32_values(&mut hit.source);+ hit+ })+ .collect(),+ None,+ );+ }// `_source: false`: keep the raw source so the response layer can// still resolve `fields` / `_ignored` / `highlight` against it —// `_source` emission is suppressed in es_compat.rs.@@ -35687,6 +35767,10 @@ fn apply_source_filter_measured(.then(|| default_projection_len(&h.source, generated_companion_fields, mode));let sentinels = take_collapse_sentinels(&h.source);h.source = filter_object(&h.source, fields, &[]);+ // Expand before the savings measurement so a caller who asked+ // for the companion is billed for the number array on the wire,+ // not the packed string it was stored as.+ xerj_common::packed_f32::expand_packed_f32_values(&mut h.source);if let Some(baseline) = baseline {// Saturating: a caller who explicitly projects a generated// companion gets MORE than the default, and more is not a@@ -35704,6 +35788,7 @@ fn apply_source_filter_measured(.then(|| default_projection_len(&h.source, generated_companion_fields, mode));let sentinels = take_collapse_sentinels(&h.source);h.source = filter_object(&h.source, includes, excludes);+ xerj_common::packed_f32::expand_packed_f32_values(&mut h.source);if let Some(baseline) = baseline {acc.record(&h.id, baseline.saturating_sub(json_len(&h.source, mode)));}@@ -37866,6 +37951,15 @@ fn stored_chunk_vectors(source: &Value, vector_field: &str) -> Vec<Vec<f32>> {}fn json_f32_vector(value: &Value) -> Option<Vec<f32>> {+ // #1940 stores each chunk as a `xerj:f32:` string. A number array is+ // still valid for segments written before that. Non-finite components+ // in a number array keep the previous `as_f64() as f32` read.+ if value+ .as_str()+ .is_some_and(xerj_common::packed_f32::is_packed_f32)+ {+ return xerj_common::packed_f32::unpack_f32_vector(value.as_str()?);+ }let row = value.as_array()?;let mut vector = Vec::with_capacity(row.len());for component in row {@@ -42461,14 +42555,20 @@ fn push_field_root_keys(field: &str, out: &mut Vec<String>) -> bool {/// Returns `None` when the field is absent, not an array, empty, or holds/// any non-number element.fn extract_numeric_vector(source: &Value, field: &str) -> Option<Vec<f32>> {- match get_field_value(source, field) {- Some(Value::Array(arr)) if !arr.is_empty() && arr.iter().all(|v| v.is_number()) => Some(- arr.iter()- .filter_map(|v| v.as_f64().map(|f| f as f32))- .collect(),- ),- _ => None,+ let value = get_field_value(source, field)?;+ // Packed strings are the retained form (#1940). Number arrays remain+ // valid for segments written before that, and for a caller that raced+ // a vector in before compaction.+ if value+ .as_str()+ .is_some_and(xerj_common::packed_f32::is_packed_f32)+ || value+ .as_array()+ .is_some_and(|arr| !arr.is_empty() && arr.iter().all(Value::is_number))+ {+ return xerj_common::packed_f32::json_to_f32_vector(&value);}+ None}/// Collect every dense_vector-mapped field in schema order (dotted paths@@ -62468,6 +62568,29 @@ mod exact_scan_hydration_tests {}}+#[cfg(test)]+mod packed_vector_doc_values_tests {+ use super::build_doc_value_columns;+ use serde_json::json;++ #[test]+ fn packed_embedding_is_not_filed_as_a_keyword() {+ let src = json!({+ "msg": "hello",+ "msg_vector": xerj_common::packed_f32::pack_f32_vector(&[1.0, 0.0, -1.0]).unwrap(),+ });+ let cols = build_doc_value_columns(+ std::iter::once(Some(&src)),+ &std::collections::HashSet::new(),+ );+ assert!(cols.contains_key("msg"));+ assert!(+ !cols.contains_key("msg_vector"),+ "a packed embedding must not become a per-doc keyword term"+ );+ }+}+/// #1504: a logs index's omitted `match` requires every token. The same/// query on an index without the setting stays ES's OR, and an explicit/// `"operator": "or"` on the logs index stays OR too.diff --git a/engine/crates/xerj-engine/src/ingest_memory.rs b/engine/crates/xerj-engine/src/ingest_memory.rsindex 1c923ab94..5fa110f76 100644--- a/engine/crates/xerj-engine/src/ingest_memory.rs+++ b/engine/crates/xerj-engine/src/ingest_memory.rs@@ -730,12 +730,27 @@ pub fn vector_value_capacity(_ => 0,}}+ fn retained_vector_bytes(value: &serde_json::Value) -> usize {+ match value {+ // Packed embeddings retain one string, not one Value per+ // component (#1940). Count it or the ledger reports the vector+ // as free. Number arrays still go through `arrays`.+ serde_json::Value::String(text) if xerj_common::packed_f32::is_packed_f32(text) => {+ text.capacity()+ }+ serde_json::Value::Array(values) => {+ values.capacity() * std::mem::size_of::<serde_json::Value>()+ + values.iter().map(retained_vector_bytes).sum::<usize>()+ }+ other => arrays(other),+ }+ }match value {serde_json::Value::Object(values) => values.iter().map(|(key, value)| {if fields.contains(key) {- arrays(value)+ retained_vector_bytes(value)} else {vector_value_capacity(value, fields)}diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex 213595bd8..7e83671a2 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -5158,15 +5158,25 @@ pub(crate) fn looks_like_multi_vector(val: &serde_json::Value) -> bool {return false;}for row in rows {- let Some(components) = row.as_array() else {- return false;+ let width = if let Some(text) = row.as_str() {+ // Packed chunk vectors (#1940) are one string per passage, not+ // a number array. Same rectangular rule as the array form.+ match xerj_common::packed_f32::unpack_f32_vector(text) {+ Some(values) => values.len(),+ None => return false,+ }+ } else {+ let Some(components) = row.as_array() else {+ return false;+ };+ if components.is_empty() || !components.iter().all(serde_json::Value::is_number) {+ return false;+ }+ components.len()};- if components.is_empty() || !components.iter().all(serde_json::Value::is_number) {- return false;- }match dims {- None => dims = Some(components.len()),- Some(d) if d == components.len() => {}+ None => dims = Some(width),+ Some(d) if d == width => {}Some(_) => return false,}}diff --git a/engine/crates/xerj-engine/src/vector_column.rs b/engine/crates/xerj-engine/src/vector_column.rsindex e7b3aea19..559ceeec0 100644--- a/engine/crates/xerj-engine/src/vector_column.rs+++ b/engine/crates/xerj-engine/src/vector_column.rs@@ -342,15 +342,22 @@ impl ColumnBuilder {fn push_chunks(&mut self, chunks: &[Value]) -> DocVectors {let first = self.column.entries.len();for (position, chunk) in chunks.iter().enumerate() {- let Value::Array(elements) = chunk else {- continue;- };// The ordinal is the passage's position in the ORIGINAL array, so// a skipped passage never renumbers the ones after it.let Ok(ordinal) = u32::try_from(position) else {continue;};- self.push_vector(elements, ordinal);+ match chunk {+ Value::Array(elements) => {+ self.push_vector(elements, ordinal);+ }+ Value::String(text) => {+ if let Some(values) = xerj_common::packed_f32::unpack_f32_vector(text) {+ self.push_f32_vector(&values, ordinal);+ }+ }+ _ => {}+ }}DocVectors::Chunks {first,@@ -361,6 +368,12 @@ impl ColumnBuilder {fn push_pooled(&mut self, source: &Value) -> DocVectors {match field_value_cow(source, &self.field).as_deref() {Some(Value::Array(elements)) => DocVectors::Pooled(self.push_vector(elements, 0)),+ // #1940 packed form. Same f32 bits `push_vector` would have+ // produced from the number array.+ Some(Value::String(text)) => match xerj_common::packed_f32::unpack_f32_vector(text) {+ Some(values) => DocVectors::Pooled(self.push_f32_vector(&values, 0)),+ None => DocVectors::Absent,+ },_ => DocVectors::Absent,}}diff --git a/engine/crates/xerj-storage/src/stored_codec.rs b/engine/crates/xerj-storage/src/stored_codec.rsindex 997c42bc7..dc00b129e 100644--- a/engine/crates/xerj-storage/src/stored_codec.rs+++ b/engine/crates/xerj-storage/src/stored_codec.rs@@ -1510,18 +1510,6 @@ fn decode_projected_column<'a>(Ok(decoded[index].as_ref().unwrap())}-fn finite_f32_vector(values: Vec<f64>) -> Option<Vec<f32>> {- let mut out = Vec::with_capacity(values.len());- for value in values {- let value = value as f32;- if !value.is_finite() {- return None;- }- out.push(value);- }- Some(out)-}-struct TypedVectorRowsVisitor<'a> {expected_rows: usize,column_index: usize,@@ -1548,32 +1536,19 @@ impl<'de> serde::de::Visitor<'de> for TypedVectorRowsVisitor<'_> {let Some(value) = seq.next_element::<serde_json::Value>()? else {return Err(serde::de::Error::custom("vector column ended early"));};- let typed = match value {- serde_json::Value::Null => None,- serde_json::Value::Array(values) => {- let values = values- .into_iter()- .map(|value| value.as_f64())- .collect::<Option<Vec<_>>>()- .and_then(finite_f32_vector);- let Some(values) = values else {+ // Null is a supported absence. A packed `xerj:f32:` string is the+ // retained embedding form (#1940); a number array is the form+ // segments wrote before that. Anything else cannot be scored as+ // a vector and drops the whole column to the stored-document scan.+ let typed = if value.is_null() {+ None+ } else {+ match xerj_common::packed_f32::json_to_f32_vector(&value) {+ Some(values) => Some(values),+ None => {supported = false;- out.push(None);- if checkpoint_decoded_row(- row,- self.column_index,- self.checkpoint,- self.cancelled,- ) {- return Err(serde::de::Error::custom("stored decode cancelled"));- }- continue;- };- Some(values)- }- _ => {- supported = false;- None+ None+ }}};out.push(typed);@@ -1649,18 +1624,10 @@ fn decode_typed_vector_rows_controlled(.map_err(|error| StorageError::Other(anyhow::anyhow!("constant decode: {error}")))?{serde_json::Value::Null => None,- serde_json::Value::Array(values) => {- let Some(values) = values- .into_iter()- .map(|value| value.as_f64())- .collect::<Option<Vec<_>>>()- .and_then(finite_f32_vector)- else {- return Ok(Ok(None));- };- Some(values)- }- _ => return Ok(Ok(None)),+ other => match xerj_common::packed_f32::json_to_f32_vector(&other) {+ Some(values) => Some(values),+ None => return Ok(Ok(None)),+ },};return repeat_constant_rows_controlled(row, num_docs, column_index, checkpoint).map(|result| result.map(Some));@@ -1733,15 +1700,7 @@ fn decode_typed_vector_chunk_rows_controlled())})?;for chunk in chunks {- let serde_json::Value::Array(values) = chunk else {- return Ok(Ok(None));- };- let Some(vector) = values- .into_iter()- .map(|value| value.as_f64())- .collect::<Option<Vec<_>>>()- .and_then(finite_f32_vector)- else {+ let Some(vector) = xerj_common::packed_f32::json_to_f32_vector(&chunk) else {return Ok(Ok(None));};typed.push(vector);@@ -1805,16 +1764,8 @@ fn parse_typed_chunk_reader<R: std::io::Read>(serde_json::Value::Array(chunks) => {let mut result = Vec::with_capacity(chunks.len());for chunk in chunks {- let serde_json::Value::Array(values) = chunk else {- supported = false;- continue;- };- let vector = values- .into_iter()- .map(|value| value.as_f64())- .collect::<Option<Vec<_>>>()- .and_then(finite_f32_vector);- let Some(vector) = vector else {+ let Some(vector) = xerj_common::packed_f32::json_to_f32_vector(&chunk)+ else {supported = false;continue;};