Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): report flat_object/flattened in _field_caps, not generic object

Vincenzo Lombardoff2430dd13fd
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index e0b8bc168..3b235b389 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -997,7 +997,13 @@ fn validate_runtime_fields(rt: &serde_json::Map<String, Value>) -> Result<(), St
continue;
};
if let Some(t) = obj.get("type").and_then(Value::as_str) {
- if !is_supported_field_type(t) {
+ // `flattened`/`flat_object` are valid *mapping* property types
+ // but not valid *runtime field* types in real ES — runtime
+ // fields are restricted to a small scalar/composite set.
+ // is_supported_field_type is shared with the mapping-properties
+ // path (validate_properties), so this exclusion has to live
+ // here rather than narrowing that shared list.
+ if t == "flattened" || t == "flat_object" || !is_supported_field_type(t) {
return Err(format!(
"Failed to parse mapping: The mapper type [{t}] declared on runtime field [{name}] does not exist. It might have been created within a future version or requires a plugin to be installed. Check the documentation."
));
@@ -17056,6 +17062,30 @@ pub async fn field_caps(
};
let schema = idx.schema().await;
+ // Resolved up front (not just for multi-fields below) so the
+ // per-field loop can consult the mapping's OWN declared type for
+ // fields the native schema collapses to a generic type — namely
+ // `flattened`/`flat_object`, both stored as `FieldType::Object`
+ // since xerj has no dedicated native variant for either. Without
+ // this, `_field_caps` — what OSD/Kibana actually use to discover
+ // index-pattern fields, not `GET _mapping` — reports every such
+ // field as a plain `object`, even though `GET _mapping` correctly
+ // round-trips the real declared type.
+ let stored_mapping = state
+ .engine
+ .index_mappings
+ .get(idx_name.as_str())
+ .map(|v| v.clone())
+ .unwrap_or(Value::Null);
+ let properties = if stored_mapping.is_null() {
+ Value::Object(schema_to_es_properties(&schema))
+ } else {
+ stored_mapping
+ .get("properties")
+ .cloned()
+ .unwrap_or_else(|| json!({}))
+ };
+
for field in &schema.fields {
// Support comma-separated field list and wildcard suffix.
if fields_filter != "*" {
@@ -17067,7 +17097,9 @@ pub async fn field_caps(
}
}
- let es_type = native_type_to_es_str(&field.field_type);
+ let native_es_type = native_type_to_es_str(&field.field_type);
+ let es_type =
+ declared_flattened_type(&properties, &field.name).unwrap_or(native_es_type);
let searchable = field.is_searchable();
let aggregatable = field.is_aggregatable();
@@ -17096,20 +17128,6 @@ pub async fn field_caps(
// worked when queried directly. Walk the same mapping shape
// `GET _mapping` already derives correctly and add each declared
// multi-field as its own field-caps entry.
- let stored_mapping = state
- .engine
- .index_mappings
- .get(idx_name.as_str())
- .map(|v| v.clone())
- .unwrap_or(Value::Null);
- let properties = if stored_mapping.is_null() {
- Value::Object(schema_to_es_properties(&schema))
- } else {
- stored_mapping
- .get("properties")
- .cloned()
- .unwrap_or_else(|| json!({}))
- };
let mut multi_fields = Vec::new();
collect_multi_fields(&properties, "", &mut multi_fields);
for (name, es_type) in multi_fields {
@@ -17191,6 +17209,26 @@ fn collect_multi_fields(properties: &Value, prefix: &str, out: &mut Vec<(String,
}
}
+/// Look up `field_name` (a possibly dotted path) in a mapping `properties`
+/// tree and return its declared ES type string ONLY if it's
+/// `flattened`/`flat_object` — both collapse to the generic
+/// `FieldType::Object` in the native schema (xerj has no dedicated native
+/// variant for either), so `_field_caps`/`global_field_caps` need this to
+/// report the real declared type instead of a plain `object`.
+fn declared_flattened_type<'a>(properties: &'a Value, field_name: &str) -> Option<&'a str> {
+ let mut current = properties;
+ let segments: Vec<&str> = field_name.split('.').collect();
+ for (i, seg) in segments.iter().enumerate() {
+ let def = current.as_object()?.get(*seg)?;
+ if i == segments.len() - 1 {
+ let t = def.get("type").and_then(Value::as_str)?;
+ return matches!(t, "flattened" | "flat_object").then_some(t);
+ }
+ current = def.get("properties")?;
+ }
+ None
+}
+
fn native_type_to_es_str(ft: &FieldType) -> &'static str {
match ft {
FieldType::Text => "text",
@@ -17209,6 +17247,123 @@ fn native_type_to_es_str(ft: &FieldType) -> &'static str {
}
}
+#[cfg(test)]
+mod flat_object_field_caps_tests {
+ //! Regression coverage for a real gap found in review: OpenSearch
+ //! Dashboards discovers index-pattern fields via `_field_caps`, not
+ //! `GET _mapping` — but `flattened`/`flat_object` both collapse to the
+ //! generic native `FieldType::Object`, and `_field_caps` derived its
+ //! reported type purely from that native type, so a `flat_object`
+ //! field surfaced to Dashboards as a plain `object`. `GET _mapping`
+ //! round-tripped the declared type correctly (it returns the stored
+ //! mapping JSON verbatim) while `_field_caps` did not — the mismatch
+ //! was easy to miss testing only via `GET _mapping`.
+ use super::*;
+ use axum::{
+ body::{to_bytes, Body},
+ http::Request,
+ };
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ #[test]
+ fn declared_flattened_type_finds_top_level_flat_object() {
+ let props = json!({"attrs": {"type": "flat_object"}});
+ assert_eq!(
+ declared_flattened_type(&props, "attrs"),
+ Some("flat_object")
+ );
+ }
+
+ #[test]
+ fn declared_flattened_type_finds_flattened_alias() {
+ let props = json!({"labels": {"type": "flattened"}});
+ assert_eq!(declared_flattened_type(&props, "labels"), Some("flattened"));
+ }
+
+ #[test]
+ fn declared_flattened_type_walks_dotted_nested_path() {
+ let props = json!({
+ "obj": {"type": "object", "properties": {"attrs": {"type": "flat_object"}}}
+ });
+ assert_eq!(
+ declared_flattened_type(&props, "obj.attrs"),
+ Some("flat_object")
+ );
+ }
+
+ #[test]
+ fn declared_flattened_type_returns_none_for_other_types() {
+ let props = json!({"name": {"type": "keyword"}});
+ assert_eq!(declared_flattened_type(&props, "name"), None);
+ }
+
+ #[test]
+ fn declared_flattened_type_returns_none_for_missing_field() {
+ let props = json!({"name": {"type": "keyword"}});
+ assert_eq!(declared_flattened_type(&props, "missing"), None);
+ }
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ #[tokio::test]
+ async fn field_caps_reports_flat_object_not_generic_object() {
+ let router = crate::router::build_es_compat_router(test_state());
+
+ let create_response = router
+ .clone()
+ .oneshot(
+ Request::put("/flat-object-caps-e2e")
+ .header("content-type", "application/json")
+ .body(Body::from(
+ json!({"mappings": {"properties": {"attrs": {"type": "flat_object"}}}})
+ .to_string(),
+ ))
+ .expect("request"),
+ )
+ .await
+ .expect("create response");
+ assert!(create_response.status().is_success());
+
+ let caps_response = router
+ .oneshot(
+ Request::get("/flat-object-caps-e2e/_field_caps?fields=attrs")
+ .body(Body::empty())
+ .expect("request"),
+ )
+ .await
+ .expect("field_caps response");
+ assert!(caps_response.status().is_success());
+
+ let bytes = to_bytes(caps_response.into_body(), usize::MAX)
+ .await
+ .expect("response bytes");
+ let response: Value = serde_json::from_slice(&bytes).expect("JSON response");
+ assert!(
+ response["fields"]["attrs"]["flat_object"]["searchable"]
+ .as_bool()
+ .unwrap_or(false),
+ "expected _field_caps to report attrs as flat_object, got: {response}"
+ );
+ assert!(
+ response["fields"]["attrs"].get("object").is_none(),
+ "attrs should not ALSO be reported as a generic object: {response}"
+ );
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// POST /_msearch — Multi-Search API (NDJSON)
// ─────────────────────────────────────────────────────────────────────────────
@@ -25434,6 +25589,25 @@ pub async fn global_field_caps(
};
let schema = idx.schema().await;
+ // See the matching comment in `field_caps`: `flattened`/
+ // `flat_object` both collapse to `FieldType::Object` natively, so
+ // the mapping itself is the only place the real declared type
+ // still lives.
+ let stored_mapping = state
+ .engine
+ .index_mappings
+ .get(index_info.name.as_str())
+ .map(|v| v.clone())
+ .unwrap_or(Value::Null);
+ let properties = if stored_mapping.is_null() {
+ Value::Object(schema_to_es_properties(&schema))
+ } else {
+ stored_mapping
+ .get("properties")
+ .cloned()
+ .unwrap_or_else(|| json!({}))
+ };
+
for field in &schema.fields {
if fields_filter != "*" {
// Support comma-separated field list and simple wildcard suffix.
@@ -25445,7 +25619,9 @@ pub async fn global_field_caps(
}
}
- let es_type = native_type_to_es_str(&field.field_type);
+ let native_es_type = native_type_to_es_str(&field.field_type);
+ let es_type =
+ declared_flattened_type(&properties, &field.name).unwrap_or(native_es_type);
let searchable = field.is_searchable();
let aggregatable = field.is_aggregatable();
@@ -25509,6 +25685,22 @@ pub async fn get_mapping_field(
let schema = idx.schema().await;
let properties = schema_to_es_properties(&schema);
+ // `flattened`/`flat_object` (and anything else that collapses to a
+ // generic native FieldType) only survive as their real declared name
+ // in the stored mapping — the native-schema-derived `properties` above
+ // has already lost that distinction. Used as a per-field override
+ // below so a dynamically-inferred field with no stored-mapping entry
+ // still falls back to the schema-derived type, same as before.
+ let stored_mapping = state
+ .engine
+ .index_mappings
+ .get(index.as_str())
+ .map(|v| v.clone())
+ .unwrap_or(Value::Null);
+ let stored_properties = stored_mapping
+ .get("properties")
+ .cloned()
+ .unwrap_or_else(|| json!({}));
// Collect matching field entries. Supports comma-separated field names and wildcard `*`.
let field_names: Vec<&str> = field.split(',').map(str::trim).collect();
@@ -25523,10 +25715,12 @@ pub async fn get_mapping_field(
}
});
if matches {
- let es_type = field_def
+ let native_es_type = field_def
.get("type")
.and_then(Value::as_str)
.unwrap_or("object");
+ let es_type =
+ declared_flattened_type(&stored_properties, field_name).unwrap_or(native_es_type);
mapping_result.insert(
field_name.clone(),
json!({
diff --git a/engine/tests/es-compat-yaml/yaml/search/341_flat_object.yml b/engine/tests/es-compat-yaml/yaml/search/341_flat_object.yml
new file mode 100644
index 000000000..5aa78ce70
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/341_flat_object.yml
@@ -0,0 +1,141 @@
+---
+"Test exists query on flat_object field":
+ - do:
+ indices.create:
+ index: flat_object_test
+ body:
+ mappings:
+ properties:
+ flat_object:
+ type: flat_object
+ - do:
+ index:
+ index: flat_object_test
+ id: "1"
+ body:
+ flat_object:
+ key: some_value
+ refresh: true
+
+ - do:
+ search:
+ index: flat_object_test
+ body:
+ query:
+ exists:
+ field: flat_object
+
+ - match: { hits.total.value: 1 }
+
+ - do:
+ search:
+ index: flat_object_test
+ body:
+ query:
+ exists:
+ field: flat_object.key
+
+ - match: { hits.total.value: 1 }
+
+ - do:
+ search:
+ index: flat_object_test
+ body:
+ query:
+ exists:
+ field: flat_object.nonexistent_key
+
+ - match: { hits.total.value: 0 }
+
+---
+"Test fields option on flat_object field":
+ - do:
+ indices.create:
+ index: test
+ body:
+ mappings:
+ properties:
+ flat_object:
+ type: flat_object
+
+ - do:
+ index:
+ index: test
+ id: "1"
+ body:
+ flat_object:
+ some_field: some_value
+ refresh: true
+
+ - do:
+ search:
+ index: test
+ body:
+ fields: ["flat_object"]
+
+ - match: { hits.total.value: 1 }
+ - length: { hits.hits: 1 }
+ - length: { hits.hits.0.fields: 1 }
+ - match: { hits.hits.0.fields.flat_object: [ { "some_field": "some_value" } ] }
+
+ - do:
+ search:
+ index: test
+ body:
+ fields: [ "flat*" ]
+
+ - match: { hits.total.value: 1 }
+ - length: { hits.hits: 1 }
+ - length: { hits.hits.0.fields: 1 }
+ - match: { hits.hits.0.fields.flat_object: [ { "some_field": "some_value" } ] }
+
+---
+"flat_object ignore_above single-value field":
+ - do:
+ indices.create:
+ index: ignore_above_flat_object_test
+ body:
+ mappings:
+ properties:
+ name:
+ type: keyword
+ flat:
+ type: flat_object
+ ignore_above: 5
+
+ - do:
+ index:
+ index: ignore_above_flat_object_test
+ id: "1"
+ refresh: true
+ body:
+ name: "A"
+ flat: { "value": "foo", "key": "foo key" }
+
+ - do:
+ index:
+ index: ignore_above_flat_object_test
+ id: "2"
+ refresh: true
+ body:
+ name: "B"
+ flat: { "value": "foo bar", "key": "foo bar key" }
+
+ - do:
+ search:
+ index: ignore_above_flat_object_test
+ sort: name
+ body:
+ fields:
+ - flat
+ query:
+ match_all: {}
+
+ - match: { hits.total.value: 2 }
+ - match: { hits.hits.0._source.flat.value: "foo" }
+ - match: { hits.hits.0._source.flat.key: "foo key" }
+ - match: { hits.hits.1._source.flat.value: "foo bar" }
+ - match: { hits.hits.1._source.flat.key: "foo bar key" }
+
+ - match: { hits.hits.0.fields.flat.0.value: "foo" }
+ - match: { hits.hits.1.fields.flat.0.value: null }