Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(es-compat): GET /_doc _source_includes/excludes honor nested paths (#850) (#851)

xerj-org3c12e3167544
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 64df8179d..32ca2ed4a 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -3205,29 +3205,11 @@ fn apply_get_doc_source_filter(source: Value, params: &GetDocParams) -> Value {
return source;
}
- // Re-use the same filter_object logic from the search path.
- filter_source_object(&source, &includes, &excludes)
-}
-
-/// Filter a JSON object by includes/excludes lists (with trailing `*` wildcard support).
-fn filter_source_object(source: &Value, includes: &[String], excludes: &[String]) -> Value {
- let obj = match source.as_object() {
- Some(o) => o,
- None => return source.clone(),
- };
- let mut result = serde_json::Map::new();
- for (k, v) in obj {
- let keep = if includes.is_empty() {
- true
- } else {
- includes.iter().any(|inc| source_field_matches(k, inc))
- };
- let excluded = excludes.iter().any(|exc| source_field_matches(k, exc));
- if keep && !excluded {
- result.insert(k.clone(), v.clone());
- }
- }
- Value::Object(result)
+ // Project through the SAME faithful ES `_source` automaton the search path
+ // uses (`xerj_engine::filter_object`), so dotted/nested includes+excludes
+ // and `*` globs behave identically. The old local top-level-only filter
+ // dropped nested includes entirely and no-oped nested excludes (#850).
+ xerj_engine::filter_object(&source, &includes, &excludes)
}
fn source_field_matches(field: &str, pattern: &str) -> bool {
@@ -3238,6 +3220,54 @@ fn source_field_matches(field: &str, pattern: &str) -> bool {
}
}
+#[cfg(test)]
+mod get_doc_source_filter_tests {
+ use super::*;
+ use serde_json::json;
+
+ // #850: GET /_doc `_source_includes`/`_source_excludes` must support
+ // dotted/nested paths (and `*` globs), like search — the old top-level-only
+ // filter dropped a nested include entirely and no-oped a nested exclude.
+ #[test]
+ fn get_doc_source_filter_handles_nested_paths() {
+ let src = || json!({ "user": { "name": "x", "age": 5 }, "id": 1 });
+
+ // Nested include keeps only the requested leaf under its parent.
+ let inc = apply_get_doc_source_filter(
+ src(),
+ &GetDocParams {
+ source_includes: Some("user.name".into()),
+ ..Default::default()
+ },
+ );
+ assert_eq!(inc, json!({ "user": { "name": "x" } }), "nested include");
+
+ // Nested exclude removes just that leaf, keeps siblings + other keys.
+ let exc = apply_get_doc_source_filter(
+ src(),
+ &GetDocParams {
+ source_excludes: Some("user.age".into()),
+ ..Default::default()
+ },
+ );
+ assert_eq!(
+ exc,
+ json!({ "user": { "name": "x" }, "id": 1 }),
+ "nested exclude"
+ );
+
+ // Top-level literal include is unchanged from before.
+ let top = apply_get_doc_source_filter(
+ src(),
+ &GetDocParams {
+ source_includes: Some("id".into()),
+ ..Default::default()
+ },
+ );
+ assert_eq!(top, json!({ "id": 1 }), "top-level include");
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// DELETE /{index}/_doc/{id}
// ─────────────────────────────────────────────────────────────────────────────
@@ -17991,7 +18021,7 @@ async fn build_update_get_field(
let filtered = if includes.is_empty() && excludes.is_empty() {
source
} else {
- filter_source_object(&source, &includes, &excludes)
+ xerj_engine::filter_object(&source, &includes, &excludes)
};
json!({ "found": true, "_source": filtered })
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 8a118492f..93d9005d8 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -28012,7 +28012,12 @@ fn generated_embedding_companion_fields(schema: &Schema) -> HashSet<String> {
/// Supports dotted field paths: `include.field1` matches the nested
/// `{"include": {"field1": ...}}` structure. If `includes` is empty,
/// all fields are kept before exclusions are applied.
-fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Value {
+/// Faithful ES `_source` include/exclude projection over a single JSON value:
+/// dotted paths and `*` globs that span `.` at any depth, includes then
+/// excludes (#602). Exposed so the document-GET path (`_source_includes` /
+/// `_source_excludes` on `GET /_doc`) projects identically to search instead
+/// of the old top-level-only filter that dropped nested includes.
+pub fn filter_object(source: &Value, includes: &[String], excludes: &[String]) -> Value {
let obj = match source.as_object() {
Some(o) => o,
None => return source.clone(),
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 8b12b5b2b..07d9a23ce 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -49,8 +49,8 @@ mod bounded_ingest_resources_tests;
pub use cluster_state::{ClusterStateBlockKind, ClusterStateBootStatus};
pub use engine::{EmbeddingExecutionIdentity, Engine, HealthStatus, IndexInfo};
pub use index::{
- detect_log_format, resolve_date_math, resolve_field_alias, EnrichTable, FieldEncodingInfo,
- Index, IndexResponse, IndexStats, LogFormat,
+ detect_log_format, filter_object, resolve_date_math, resolve_field_alias, EnrichTable,
+ FieldEncodingInfo, Index, IndexResponse, IndexStats, LogFormat,
};
// Second-brain graph expansion (bounded columnar traversal over an edges
// index). Lives under `index` for private-field access; re-exported at the