AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(search): stop returning generated embedding vectors in the default _source
bugerbe0137d2c7ee
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0fbaee792..10c8946cc 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -10361,7 +10361,18 @@ impl Index {candidates = scored.len(),"knn served via HNSW (coverage gate passed)");- Some(knn_result_from_scored(request, field, scored, k, started))+ let generated_companion_fields = {+ let schema = self.schema.read().await;+ generated_embedding_companion_fields(&schema.schema)+ };+ Some(knn_result_from_scored(+ request,+ field,+ scored,+ k,+ started,+ &generated_companion_fields,+ ))}/// Brute-force exact KNN against every doc's stored source.@@ -10832,7 +10843,18 @@ impl Index {// ── Rank, cap the candidate pool at k, then paginate ──────────// (shared with the HNSW path so hits format / total semantics// cannot drift between the exact and approximate executors)- let mut result = knn_result_from_scored(request, field, scored, k, started);+ let generated_companion_fields = {+ let schema = self.schema.read().await;+ generated_embedding_companion_fields(&schema.schema)+ };+ let mut result = knn_result_from_scored(+ request,+ field,+ scored,+ k,+ started,+ &generated_companion_fields,+ );result.timed_out = timed_out;if timed_out {result.total.relation = TotalHitsRelation::Gte;@@ -10915,7 +10937,18 @@ impl Index {// passage metadata. Child clauses retain ordinary ranking semantics;// provenance is intentionally omitted until per-clause ownership is// represented in the merged winner.- let mut result = knn_result_from_scored(request, "", merged, k_sum, started);+ let generated_companion_fields = {+ let schema = self.schema.read().await;+ generated_embedding_companion_fields(&schema.schema)+ };+ let mut result = knn_result_from_scored(+ request,+ "",+ merged,+ k_sum,+ started,+ &generated_companion_fields,+ );result.timed_out = any_timed_out;if any_timed_out {result.total.relation = TotalHitsRelation::Gte;@@ -17379,7 +17412,11 @@ impl Index {};// --- Apply _source filtering ---- let page = apply_source_filter(page, &request.source);+ let generated_companion_fields = {+ let schema = self.schema.read().await;+ generated_embedding_companion_fields(&schema.schema)+ };+ let page = apply_source_filter(page, &request.source, &generated_companion_fields);// --- Build profile data if requested ---let profile = if request.profile {@@ -24416,7 +24453,11 @@ fn es_format_to_epoch_ms(s: &str, fmt: &str) -> Option<i64> {None}-fn apply_source_filter(hits: Vec<Hit>, filter: &SourceFilter) -> Vec<Hit> {+fn apply_source_filter(+ hits: Vec<Hit>,+ filter: &SourceFilter,+ generated_companion_fields: &HashSet<String>,+) -> Vec<Hit> {let hits: Vec<Hit> = hits.into_iter().map(|mut hit| {@@ -24424,7 +24465,16 @@ fn apply_source_filter(hits: Vec<Hit>, filter: &SourceFilter) -> Vec<Hit> {hit}).collect();+ let generated_companion_fields: Vec<String> =+ generated_companion_fields.iter().cloned().collect();match filter {+ SourceFilter::Default => hits+ .into_iter()+ .map(|mut h| {+ h.source = filter_object(&h.source, &[], &generated_companion_fields);+ h+ })+ .collect(),SourceFilter::Enabled(true) => hits,// `_source: false`: keep the raw source so the response layer can// still resolve `fields` / `_ignored` / `highlight` against it —@@ -24447,6 +24497,27 @@ fn apply_source_filter(hits: Vec<Hit>, filter: &SourceFilter) -> Vec<Hit> {}}+/// Return the engine-generated embedding companions for a schema.+///+/// This is deliberately derived from `FieldConfig.embedding` rather than from+/// field-name suffixes: a user-owned field called `body_vector` is ordinary+/// source data unless an embedding mapping designates it as a target.+fn generated_embedding_companion_fields(schema: &Schema) -> HashSet<String> {+ let mut companions = HashSet::new();+ for field in &schema.fields {+ let Some(embedding) = &field.embedding else {+ continue;+ };+ let target = embedding+ .target_field+ .clone()+ .unwrap_or_else(|| format!("{}_vector", field.name));+ companions.insert(target.clone());+ companions.insert(format!("{target}_chunks"));+ }+ companions+}+/// Return a filtered copy of a JSON object keeping `includes` and removing `excludes`.////// Supports dotted field paths: `include.field1` matches the nested@@ -24576,6 +24647,142 @@ fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Vacollect_and_filter(source, "", includes, excludes)}+#[cfg(test)]+mod source_filter_tests {+ use super::*;+ use serde_json::json;+ use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType};++ fn embedded_schema() -> Schema {+ let mut schema = Schema::empty();+ schema+ .add_field(+ FieldConfig::new("body", FieldType::Text).with_embedding(EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: None,+ }),+ )+ .unwrap();+ schema+ }++ fn hit(id: &str, score: f32, source: Value) -> Hit {+ Hit {+ id: id.to_string(),+ score,+ source,+ sort: Vec::new(),+ explain: None,+ highlight: None,+ matched_queries: Vec::new(),+ passage: None,+ }+ }++ fn generated_source() -> Value {+ json!({+ "body": "the source text",+ "body_vector": [0.1, 0.2, 0.3],+ "body_vector_chunks": [[0.1, 0.2], [0.2, 0.3]],+ "title": "a user field"+ })+ }++ #[test]+ fn omitted_source_excludes_mapping_derived_companions_only() {+ let companions = generated_embedding_companion_fields(&embedded_schema());+ let hits = vec![+ hit("first", 0.9, generated_source()),+ hit("second", 0.4, generated_source()),+ ];++ let filtered = apply_source_filter(hits, &SourceFilter::Default, &companions);++ assert_eq!(+ filtered+ .iter()+ .map(|hit| (hit.id.as_str(), hit.score))+ .collect::<Vec<_>>(),+ vec![("first", 0.9), ("second", 0.4)]+ );+ for hit in filtered {+ assert_eq!(+ hit.source,+ json!({+ "body": "the source text",+ "title": "a user field"+ })+ );+ }+ }++ #[test]+ fn explicit_source_true_keeps_mapping_derived_companions() {+ let companions = generated_embedding_companion_fields(&embedded_schema());+ let original = generated_source();+ let filtered = apply_source_filter(+ vec![hit("doc", 0.9, original.clone())],+ &SourceFilter::Enabled(true),+ &companions,+ );++ assert_eq!(filtered[0].source, original);+ }++ #[test]+ fn explicit_source_include_still_returns_a_generated_companion() {+ let companions = generated_embedding_companion_fields(&embedded_schema());+ let filtered = apply_source_filter(+ vec![hit("doc", 0.9, generated_source())],+ &SourceFilter::Includes(vec!["body_vector".to_string()]),+ &companions,+ );++ assert_eq!(filtered[0].source, json!({"body_vector": [0.1, 0.2, 0.3]}));+ }++ #[test]+ fn omitted_source_measures_companion_payload_reduction() {+ let vector = Value::Array((0..384).map(|_| json!(0.123456)).collect());+ let chunks = Value::Array((0..660).map(|_| vector.clone()).collect());+ let source = json!({+ "body": "x".repeat(250_000),+ "body_vector": vector,+ "body_vector_chunks": chunks+ });+ let companions = generated_embedding_companion_fields(&embedded_schema());+ let before = hit("doc", 0.9, source);+ let before_bytes = serde_json::to_vec(&before).unwrap().len();+ let after = apply_source_filter(vec![before], &SourceFilter::Default, &companions);+ let after_bytes = serde_json::to_vec(&after[0]).unwrap().len();++ eprintln!("default _search hit payload bytes: before={before_bytes}, after={after_bytes}");+ assert!(before_bytes > 2_000_000);+ assert!(after_bytes < 300_000);+ assert!(after_bytes * 8 < before_bytes);+ }++ #[test]+ fn user_declared_vector_suffix_without_embedding_is_not_stripped() {+ let mut schema = Schema::empty();+ schema+ .add_field(FieldConfig::new("body_vector", FieldType::Keyword))+ .unwrap();+ let companions = generated_embedding_companion_fields(&schema);+ assert!(companions.is_empty());++ let original = json!({"body_vector": "user-owned value"});+ let filtered = apply_source_filter(+ vec![hit("doc", 0.9, original.clone())],+ &SourceFilter::Default,+ &companions,+ );++ assert_eq!(filtered[0].source, original);+ }+}+/// Simple field pattern matching — supports trailing `*` wildcard.fn field_matches(field: &str, pattern: &str) -> bool {if let Some(prefix) = pattern.strip_suffix('*') {@@ -27178,6 +27385,7 @@ fn knn_result_from_scored(mut scored: Vec<(String, f32, Value, Option<u32>)>,k: usize,started: std::time::Instant,+ generated_companion_fields: &HashSet<String>,) -> SearchResult {scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));// `k` bounds the kNN candidate pool; `from`/`size` then window into@@ -27237,7 +27445,7 @@ fn knn_result_from_scored(}}).collect();- let hits = apply_source_filter(hits, &request.source);+ let hits = apply_source_filter(hits, &request.source, generated_companion_fields);SearchResult {hits,diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 208918097..3cfc4efff 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -34,9 +34,12 @@ async fn test_semantic_vectors_stay_stored_but_not_fts_indexed_after_merge_and_rasync fn assert_queries(idx: &xerj_engine::Index) {let semantic = idx- .search(&make_search(json!({- "semantic": {"field": "content", "query": "quarterly liquidity", "k": 10}- })))+ .search(&make_search_with_source(+ json!({+ "semantic": {"field": "content", "query": "quarterly liquidity", "k": 10}+ }),+ json!(true),+ )).await.unwrap();assert_eq!(semantic.total.value, 2);@@ -401,6 +404,15 @@ fn make_search(query_json: Value) -> SearchRequest {parse_request(&json!({ "query": query_json, "size": 100 })).expect("parse_request")}+fn make_search_with_source(query_json: Value, source: Value) -> SearchRequest {+ parse_request(&json!({+ "query": query_json,+ "size": 100,+ "_source": source+ }))+ .expect("parse_request")+}+fn make_search_with_size(query_json: Value, size: usize) -> SearchRequest {parse_request(&json!({ "query": query_json, "size": size })).expect("parse_request")}@@ -5447,7 +5459,10 @@ async fn test_semantic_text_match_survives_flush() {idx.flush().await.unwrap();let post = idx- .search(&make_search(json!({"match": {"content": "quick fox"}})))+ .search(&make_search_with_source(+ json!({"match": {"content": "quick fox"}}),+ json!(true),+ )).await.unwrap();assert_eq!(diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex 2e58721d5..16a0e9eb2 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -1047,9 +1047,16 @@ impl Default for SearchRequest {}/// Controls which source fields are returned with each hit.-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]#[serde(untagged)]pub enum SourceFilter {+ /// `_source` was omitted; the engine applies its default source projection.+ ///+ /// This is intentionally a unit variant. With an untagged enum serde+ /// serialises it as `null`, so it must stay before the value-carrying+ /// variants for the corresponding round-trip to deserialize correctly.+ #[default]+ Default,/// `true` → return all source fields; `false` → return none.Enabled(bool),/// Return only the listed fields.@@ -1063,12 +1070,6 @@ pub enum SourceFilter {},}-impl Default for SourceFilter {- fn default() -> Self {- SourceFilter::Enabled(true)- }-}-// ─────────────────────────────────────────────────────────────────────────────// HighlightRequest — per-field highlighting configuration// ─────────────────────────────────────────────────────────────────────────────@@ -1138,3 +1139,68 @@ pub struct CollapseField {#[serde(default, skip_serializing_if = "Option::is_none")]pub inner_hits: Option<serde_json::Value>,}++#[cfg(test)]+mod source_filter_tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn source_filter_shapes_round_trip_through_serde() {+ let cases = [+ (SourceFilter::Default, json!(null)),+ (SourceFilter::Enabled(true), json!(true)),+ (SourceFilter::Enabled(false), json!(false)),+ (+ SourceFilter::Includes(vec!["field".to_string()]),+ json!(["field"]),+ ),+ ];++ for (filter, expected_wire) in cases {+ let wire = serde_json::to_value(&filter).unwrap();+ assert_eq!(wire, expected_wire);+ let round_tripped: SourceFilter = serde_json::from_value(wire).unwrap();+ assert_eq!(round_tripped, filter);+ }+ }++ #[test]+ fn omitted_source_and_explicit_booleans_keep_distinct_variants() {+ let omitted = crate::parser::parse_request(&json!({})).unwrap();+ assert_eq!(omitted.source, SourceFilter::Default);++ let explicitly_enabled = crate::parser::parse_request(&json!({+ "_source": true+ }))+ .unwrap();+ assert_eq!(explicitly_enabled.source, SourceFilter::Enabled(true));++ let explicitly_disabled = crate::parser::parse_request(&json!({+ "_source": false+ }))+ .unwrap();+ assert_eq!(explicitly_disabled.source, SourceFilter::Enabled(false));+ }++ #[test]+ fn search_request_source_shapes_round_trip_after_parsing() {+ let cases = [+ (json!({}), SourceFilter::Default),+ (json!({"_source": true}), SourceFilter::Enabled(true)),+ (json!({"_source": false}), SourceFilter::Enabled(false)),+ (+ json!({"_source": ["field"]}),+ SourceFilter::Includes(vec!["field".to_string()]),+ ),+ ];++ for (body, expected_source) in cases {+ let request = crate::parser::parse_request(&body).unwrap();+ assert_eq!(request.source, expected_source);+ let serialized = serde_json::to_value(&request).unwrap();+ let round_tripped: SearchRequest = serde_json::from_value(serialized).unwrap();+ assert_eq!(round_tripped.source, expected_source);+ }+ }+}