AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(es-compat): GET /_doc _source_includes/excludes honor nested paths (#850) (#851)
xerj-org3c12e3167544
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 64df8179d..32ca2ed4a 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -3205,29 +3205,11 @@ fn apply_get_doc_source_filter(source: Value, params: &GetDocParams) -> Value {return source;}- // Re-use the same filter_object logic from the search path.- filter_source_object(&source, &includes, &excludes)-}--/// Filter a JSON object by includes/excludes lists (with trailing `*` wildcard support).-fn filter_source_object(source: &Value, includes: &[String], excludes: &[String]) -> Value {- let obj = match source.as_object() {- Some(o) => o,- None => return source.clone(),- };- let mut result = serde_json::Map::new();- for (k, v) in obj {- let keep = if includes.is_empty() {- true- } else {- includes.iter().any(|inc| source_field_matches(k, inc))- };- let excluded = excludes.iter().any(|exc| source_field_matches(k, exc));- if keep && !excluded {- result.insert(k.clone(), v.clone());- }- }- Value::Object(result)+ // Project through the SAME faithful ES `_source` automaton the search path+ // uses (`xerj_engine::filter_object`), so dotted/nested includes+excludes+ // and `*` globs behave identically. The old local top-level-only filter+ // dropped nested includes entirely and no-oped nested excludes (#850).+ xerj_engine::filter_object(&source, &includes, &excludes)}fn source_field_matches(field: &str, pattern: &str) -> bool {@@ -3238,6 +3220,54 @@ fn source_field_matches(field: &str, pattern: &str) -> bool {}}+#[cfg(test)]+mod get_doc_source_filter_tests {+ use super::*;+ use serde_json::json;++ // #850: GET /_doc `_source_includes`/`_source_excludes` must support+ // dotted/nested paths (and `*` globs), like search — the old top-level-only+ // filter dropped a nested include entirely and no-oped a nested exclude.+ #[test]+ fn get_doc_source_filter_handles_nested_paths() {+ let src = || json!({ "user": { "name": "x", "age": 5 }, "id": 1 });++ // Nested include keeps only the requested leaf under its parent.+ let inc = apply_get_doc_source_filter(+ src(),+ &GetDocParams {+ source_includes: Some("user.name".into()),+ ..Default::default()+ },+ );+ assert_eq!(inc, json!({ "user": { "name": "x" } }), "nested include");++ // Nested exclude removes just that leaf, keeps siblings + other keys.+ let exc = apply_get_doc_source_filter(+ src(),+ &GetDocParams {+ source_excludes: Some("user.age".into()),+ ..Default::default()+ },+ );+ assert_eq!(+ exc,+ json!({ "user": { "name": "x" }, "id": 1 }),+ "nested exclude"+ );++ // Top-level literal include is unchanged from before.+ let top = apply_get_doc_source_filter(+ src(),+ &GetDocParams {+ source_includes: Some("id".into()),+ ..Default::default()+ },+ );+ assert_eq!(top, json!({ "id": 1 }), "top-level include");+ }+}+// ─────────────────────────────────────────────────────────────────────────────// DELETE /{index}/_doc/{id}// ─────────────────────────────────────────────────────────────────────────────@@ -17991,7 +18021,7 @@ async fn build_update_get_field(let filtered = if includes.is_empty() && excludes.is_empty() {source} else {- filter_source_object(&source, &includes, &excludes)+ xerj_engine::filter_object(&source, &includes, &excludes)};json!({ "found": true, "_source": filtered })}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 8a118492f..93d9005d8 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -28012,7 +28012,12 @@ fn generated_embedding_companion_fields(schema: &Schema) -> HashSet<String> {/// Supports dotted field paths: `include.field1` matches the nested/// `{"include": {"field1": ...}}` structure. If `includes` is empty,/// all fields are kept before exclusions are applied.-fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Value {+/// Faithful ES `_source` include/exclude projection over a single JSON value:+/// dotted paths and `*` globs that span `.` at any depth, includes then+/// excludes (#602). Exposed so the document-GET path (`_source_includes` /+/// `_source_excludes` on `GET /_doc`) projects identically to search instead+/// of the old top-level-only filter that dropped nested includes.+pub fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Value {let obj = match source.as_object() {Some(o) => o,None => return source.clone(),diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 8b12b5b2b..07d9a23ce 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -49,8 +49,8 @@ mod bounded_ingest_resources_tests;pub use cluster_state::{ClusterStateBlockKind, ClusterStateBootStatus};pub use engine::{EmbeddingExecutionIdentity, Engine, HealthStatus, IndexInfo};pub use index::{- detect_log_format, resolve_date_math, resolve_field_alias, EnrichTable, FieldEncodingInfo,- Index, IndexResponse, IndexStats, LogFormat,+ detect_log_format, filter_object, resolve_date_math, resolve_field_alias, EnrichTable,+ FieldEncodingInfo, Index, IndexResponse, IndexStats, LogFormat,};// Second-brain graph expansion (bounded columnar traversal over an edges// index). Lives under `index` for private-field access; re-exported at the