Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(search): stop returning generated embedding vectors in the default _source

bugerbe0137d2c7ee
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0fbaee792..10c8946cc 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -10361,7 +10361,18 @@ impl Index {
candidates = scored.len(),
"knn served via HNSW (coverage gate passed)"
);
- Some(knn_result_from_scored(request, field, scored, k, started))
+ let generated_companion_fields = {
+ let schema = self.schema.read().await;
+ generated_embedding_companion_fields(&schema.schema)
+ };
+ Some(knn_result_from_scored(
+ request,
+ field,
+ scored,
+ k,
+ started,
+ &generated_companion_fields,
+ ))
}
/// Brute-force exact KNN against every doc's stored source.
@@ -10832,7 +10843,18 @@ impl Index {
// ── Rank, cap the candidate pool at k, then paginate ──────────
// (shared with the HNSW path so hits format / total semantics
// cannot drift between the exact and approximate executors)
- let mut result = knn_result_from_scored(request, field, scored, k, started);
+ let generated_companion_fields = {
+ let schema = self.schema.read().await;
+ generated_embedding_companion_fields(&schema.schema)
+ };
+ let mut result = knn_result_from_scored(
+ request,
+ field,
+ scored,
+ k,
+ started,
+ &generated_companion_fields,
+ );
result.timed_out = timed_out;
if timed_out {
result.total.relation = TotalHitsRelation::Gte;
@@ -10915,7 +10937,18 @@ impl Index {
// passage metadata. Child clauses retain ordinary ranking semantics;
// provenance is intentionally omitted until per-clause ownership is
// represented in the merged winner.
- let mut result = knn_result_from_scored(request, "", merged, k_sum, started);
+ let generated_companion_fields = {
+ let schema = self.schema.read().await;
+ generated_embedding_companion_fields(&schema.schema)
+ };
+ let mut result = knn_result_from_scored(
+ request,
+ "",
+ merged,
+ k_sum,
+ started,
+ &generated_companion_fields,
+ );
result.timed_out = any_timed_out;
if any_timed_out {
result.total.relation = TotalHitsRelation::Gte;
@@ -17379,7 +17412,11 @@ impl Index {
};
// --- Apply _source filtering ---
- let page = apply_source_filter(page, &request.source);
+ let generated_companion_fields = {
+ let schema = self.schema.read().await;
+ generated_embedding_companion_fields(&schema.schema)
+ };
+ let page = apply_source_filter(page, &request.source, &generated_companion_fields);
// --- Build profile data if requested ---
let profile = if request.profile {
@@ -24416,7 +24453,11 @@ fn es_format_to_epoch_ms(s: &str, fmt: &str) -> Option<i64> {
None
}
-fn apply_source_filter(hits: Vec<Hit>, filter: &SourceFilter) -> Vec<Hit> {
+fn apply_source_filter(
+ hits: Vec<Hit>,
+ filter: &SourceFilter,
+ generated_companion_fields: &HashSet<String>,
+) -> Vec<Hit> {
let hits: Vec<Hit> = hits
.into_iter()
.map(|mut hit| {
@@ -24424,7 +24465,16 @@ fn apply_source_filter(hits: Vec<Hit>, filter: &SourceFilter) -> Vec<Hit> {
hit
})
.collect();
+ let generated_companion_fields: Vec<String> =
+ generated_companion_fields.iter().cloned().collect();
match filter {
+ SourceFilter::Default => hits
+ .into_iter()
+ .map(|mut h| {
+ h.source = filter_object(&h.source, &[], &generated_companion_fields);
+ h
+ })
+ .collect(),
SourceFilter::Enabled(true) => hits,
// `_source: false`: keep the raw source so the response layer can
// still resolve `fields` / `_ignored` / `highlight` against it —
@@ -24447,6 +24497,27 @@ fn apply_source_filter(hits: Vec<Hit>, filter: &SourceFilter) -> Vec<Hit> {
}
}
+/// Return the engine-generated embedding companions for a schema.
+///
+/// This is deliberately derived from `FieldConfig.embedding` rather than from
+/// field-name suffixes: a user-owned field called `body_vector` is ordinary
+/// source data unless an embedding mapping designates it as a target.
+fn generated_embedding_companion_fields(schema: &Schema) -> HashSet<String> {
+ let mut companions = HashSet::new();
+ for field in &schema.fields {
+ let Some(embedding) = &field.embedding else {
+ continue;
+ };
+ let target = embedding
+ .target_field
+ .clone()
+ .unwrap_or_else(|| format!("{}_vector", field.name));
+ companions.insert(target.clone());
+ companions.insert(format!("{target}_chunks"));
+ }
+ companions
+}
+
/// Return a filtered copy of a JSON object keeping `includes` and removing `excludes`.
///
/// Supports dotted field paths: `include.field1` matches the nested
@@ -24576,6 +24647,142 @@ fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Va
collect_and_filter(source, "", includes, excludes)
}
+#[cfg(test)]
+mod source_filter_tests {
+ use super::*;
+ use serde_json::json;
+ use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType};
+
+ fn embedded_schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .add_field(
+ FieldConfig::new("body", FieldType::Text).with_embedding(EmbeddingConfig {
+ endpoint: None,
+ model: None,
+ target_field: None,
+ }),
+ )
+ .unwrap();
+ schema
+ }
+
+ fn hit(id: &str, score: f32, source: Value) -> Hit {
+ Hit {
+ id: id.to_string(),
+ score,
+ source,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ }
+ }
+
+ fn generated_source() -> Value {
+ json!({
+ "body": "the source text",
+ "body_vector": [0.1, 0.2, 0.3],
+ "body_vector_chunks": [[0.1, 0.2], [0.2, 0.3]],
+ "title": "a user field"
+ })
+ }
+
+ #[test]
+ fn omitted_source_excludes_mapping_derived_companions_only() {
+ let companions = generated_embedding_companion_fields(&embedded_schema());
+ let hits = vec![
+ hit("first", 0.9, generated_source()),
+ hit("second", 0.4, generated_source()),
+ ];
+
+ let filtered = apply_source_filter(hits, &SourceFilter::Default, &companions);
+
+ assert_eq!(
+ filtered
+ .iter()
+ .map(|hit| (hit.id.as_str(), hit.score))
+ .collect::<Vec<_>>(),
+ vec![("first", 0.9), ("second", 0.4)]
+ );
+ for hit in filtered {
+ assert_eq!(
+ hit.source,
+ json!({
+ "body": "the source text",
+ "title": "a user field"
+ })
+ );
+ }
+ }
+
+ #[test]
+ fn explicit_source_true_keeps_mapping_derived_companions() {
+ let companions = generated_embedding_companion_fields(&embedded_schema());
+ let original = generated_source();
+ let filtered = apply_source_filter(
+ vec![hit("doc", 0.9, original.clone())],
+ &SourceFilter::Enabled(true),
+ &companions,
+ );
+
+ assert_eq!(filtered[0].source, original);
+ }
+
+ #[test]
+ fn explicit_source_include_still_returns_a_generated_companion() {
+ let companions = generated_embedding_companion_fields(&embedded_schema());
+ let filtered = apply_source_filter(
+ vec![hit("doc", 0.9, generated_source())],
+ &SourceFilter::Includes(vec!["body_vector".to_string()]),
+ &companions,
+ );
+
+ assert_eq!(filtered[0].source, json!({"body_vector": [0.1, 0.2, 0.3]}));
+ }
+
+ #[test]
+ fn omitted_source_measures_companion_payload_reduction() {
+ let vector = Value::Array((0..384).map(|_| json!(0.123456)).collect());
+ let chunks = Value::Array((0..660).map(|_| vector.clone()).collect());
+ let source = json!({
+ "body": "x".repeat(250_000),
+ "body_vector": vector,
+ "body_vector_chunks": chunks
+ });
+ let companions = generated_embedding_companion_fields(&embedded_schema());
+ let before = hit("doc", 0.9, source);
+ let before_bytes = serde_json::to_vec(&before).unwrap().len();
+ let after = apply_source_filter(vec![before], &SourceFilter::Default, &companions);
+ let after_bytes = serde_json::to_vec(&after[0]).unwrap().len();
+
+ eprintln!("default _search hit payload bytes: before={before_bytes}, after={after_bytes}");
+ assert!(before_bytes > 2_000_000);
+ assert!(after_bytes < 300_000);
+ assert!(after_bytes * 8 < before_bytes);
+ }
+
+ #[test]
+ fn user_declared_vector_suffix_without_embedding_is_not_stripped() {
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("body_vector", FieldType::Keyword))
+ .unwrap();
+ let companions = generated_embedding_companion_fields(&schema);
+ assert!(companions.is_empty());
+
+ let original = json!({"body_vector": "user-owned value"});
+ let filtered = apply_source_filter(
+ vec![hit("doc", 0.9, original.clone())],
+ &SourceFilter::Default,
+ &companions,
+ );
+
+ assert_eq!(filtered[0].source, original);
+ }
+}
+
/// Simple field pattern matching — supports trailing `*` wildcard.
fn field_matches(field: &str, pattern: &str) -> bool {
if let Some(prefix) = pattern.strip_suffix('*') {
@@ -27178,6 +27385,7 @@ fn knn_result_from_scored(
mut scored: Vec<(String, f32, Value, Option<u32>)>,
k: usize,
started: std::time::Instant,
+ generated_companion_fields: &HashSet<String>,
) -> SearchResult {
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
// `k` bounds the kNN candidate pool; `from`/`size` then window into
@@ -27237,7 +27445,7 @@ fn knn_result_from_scored(
}
})
.collect();
- let hits = apply_source_filter(hits, &request.source);
+ let hits = apply_source_filter(hits, &request.source, generated_companion_fields);
SearchResult {
hits,
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 208918097..3cfc4efff 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -34,9 +34,12 @@ async fn test_semantic_vectors_stay_stored_but_not_fts_indexed_after_merge_and_r
async fn assert_queries(idx: &xerj_engine::Index) {
let semantic = idx
- .search(&make_search(json!({
- "semantic": {"field": "content", "query": "quarterly liquidity", "k": 10}
- })))
+ .search(&make_search_with_source(
+ json!({
+ "semantic": {"field": "content", "query": "quarterly liquidity", "k": 10}
+ }),
+ json!(true),
+ ))
.await
.unwrap();
assert_eq!(semantic.total.value, 2);
@@ -401,6 +404,15 @@ fn make_search(query_json: Value) -> SearchRequest {
parse_request(&json!({ "query": query_json, "size": 100 })).expect("parse_request")
}
+fn make_search_with_source(query_json: Value, source: Value) -> SearchRequest {
+ parse_request(&json!({
+ "query": query_json,
+ "size": 100,
+ "_source": source
+ }))
+ .expect("parse_request")
+}
+
fn make_search_with_size(query_json: Value, size: usize) -> SearchRequest {
parse_request(&json!({ "query": query_json, "size": size })).expect("parse_request")
}
@@ -5447,7 +5459,10 @@ async fn test_semantic_text_match_survives_flush() {
idx.flush().await.unwrap();
let post = idx
- .search(&make_search(json!({"match": {"content": "quick fox"}})))
+ .search(&make_search_with_source(
+ json!({"match": {"content": "quick fox"}}),
+ json!(true),
+ ))
.await
.unwrap();
assert_eq!(
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index 2e58721d5..16a0e9eb2 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -1047,9 +1047,16 @@ impl Default for SearchRequest {
}
/// Controls which source fields are returned with each hit.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum SourceFilter {
+ /// `_source` was omitted; the engine applies its default source projection.
+ ///
+ /// This is intentionally a unit variant. With an untagged enum serde
+ /// serialises it as `null`, so it must stay before the value-carrying
+ /// variants for the corresponding round-trip to deserialize correctly.
+ #[default]
+ Default,
/// `true` → return all source fields; `false` → return none.
Enabled(bool),
/// Return only the listed fields.
@@ -1063,12 +1070,6 @@ pub enum SourceFilter {
},
}
-impl Default for SourceFilter {
- fn default() -> Self {
- SourceFilter::Enabled(true)
- }
-}
-
// ─────────────────────────────────────────────────────────────────────────────
// HighlightRequest — per-field highlighting configuration
// ─────────────────────────────────────────────────────────────────────────────
@@ -1138,3 +1139,68 @@ pub struct CollapseField {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub inner_hits: Option<serde_json::Value>,
}
+
+#[cfg(test)]
+mod source_filter_tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn source_filter_shapes_round_trip_through_serde() {
+ let cases = [
+ (SourceFilter::Default, json!(null)),
+ (SourceFilter::Enabled(true), json!(true)),
+ (SourceFilter::Enabled(false), json!(false)),
+ (
+ SourceFilter::Includes(vec!["field".to_string()]),
+ json!(["field"]),
+ ),
+ ];
+
+ for (filter, expected_wire) in cases {
+ let wire = serde_json::to_value(&filter).unwrap();
+ assert_eq!(wire, expected_wire);
+ let round_tripped: SourceFilter = serde_json::from_value(wire).unwrap();
+ assert_eq!(round_tripped, filter);
+ }
+ }
+
+ #[test]
+ fn omitted_source_and_explicit_booleans_keep_distinct_variants() {
+ let omitted = crate::parser::parse_request(&json!({})).unwrap();
+ assert_eq!(omitted.source, SourceFilter::Default);
+
+ let explicitly_enabled = crate::parser::parse_request(&json!({
+ "_source": true
+ }))
+ .unwrap();
+ assert_eq!(explicitly_enabled.source, SourceFilter::Enabled(true));
+
+ let explicitly_disabled = crate::parser::parse_request(&json!({
+ "_source": false
+ }))
+ .unwrap();
+ assert_eq!(explicitly_disabled.source, SourceFilter::Enabled(false));
+ }
+
+ #[test]
+ fn search_request_source_shapes_round_trip_after_parsing() {
+ let cases = [
+ (json!({}), SourceFilter::Default),
+ (json!({"_source": true}), SourceFilter::Enabled(true)),
+ (json!({"_source": false}), SourceFilter::Enabled(false)),
+ (
+ json!({"_source": ["field"]}),
+ SourceFilter::Includes(vec!["field".to_string()]),
+ ),
+ ];
+
+ for (body, expected_source) in cases {
+ let request = crate::parser::parse_request(&body).unwrap();
+ assert_eq!(request.source, expected_source);
+ let serialized = serde_json::to_value(&request).unwrap();
+ let round_tripped: SearchRequest = serde_json::from_value(serialized).unwrap();
+ assert_eq!(round_tripped.source, expected_source);
+ }
+ }
+}