Skip to content

Apache DataFusion SQL Query Engine

fix: propagate field metadata through coalesce, nvl, and nvl2 (#24865)

James Willisbe44fab874ab
diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rs
index 3fb40bb168..8c197d473e 100644
--- a/datafusion/common/src/metadata.rs
+++ b/datafusion/common/src/metadata.rs
@@ -5,6 +5,7 @@
use std::{collections::BTreeMap, sync::Arc};
use arrow::datatypes::{DataType, Field, FieldRef, Metadata};
+use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
use hashbrown::HashMap;
use crate::{DataFusionError, ScalarValue, error::_plan_err};
@@ -104,6 +105,75 @@ pub fn check_metadata_with_storage_equal(
Ok(())
}
+/// Compute the field metadata to propagate for an expression whose result
+/// may come from any one of several inputs coerced to a common type, such as
+/// `COALESCE`, `NVL2`, or `CASE WHEN`.
+///
+/// Metadata is merged across the fields whose data type is not `Null` (an
+/// untyped NULL literal carries no metadata and cannot contribute a typed
+/// value), with earlier fields taking precedence for repeated keys. Unrelated
+/// keys are not required to agree: metadata that arrived with the data (e.g.
+/// from an Arrow file or an embedded Arrow schema in a Parquet file) must not
+/// cause an extension type to be silently dropped from the result.
+///
+/// The Arrow extension type keys are handled as a unit, like in
+/// [`Expr::Cast`] output field derivation:
+/// - Differing values of `ARROW:extension:name` produce a planning error;
+/// the inputs are of different logical types even though they were coerced
+/// to a common storage type.
+/// - `ARROW:extension:metadata` is taken from the field that supplied the
+/// extension name, never combined from another field. Byte-wise differences
+/// in the extension metadata of fields that agree on the extension name are
+/// tolerated, since extension parameters are often JSON-encoded, where byte
+/// equality is too strict.
+///
+/// [`Expr::Cast`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.Expr.html#variant.Cast
+pub fn coerced_fields_metadata<'a>(
+ fields: impl IntoIterator<Item = &'a Field>,
+) -> Result<FieldMetadata, DataFusionError> {
+ let mut merged: BTreeMap<String, String> = BTreeMap::new();
+ let mut extension: Option<(String, Option<String>)> = None;
+
+ for field in fields {
+ if field.data_type().is_null() {
+ continue;
+ }
+ let metadata = FieldMetadata::new_from_field(field);
+ if let Some(name) = metadata.inner().get(EXTENSION_TYPE_NAME_KEY) {
+ match &extension {
+ Some((first_name, _)) if first_name != name => {
+ return _plan_err!(
+ "Cannot merge field metadata with differing values of \
+ '{EXTENSION_TYPE_NAME_KEY}': '{first_name}' and '{name}'"
+ );
+ }
+ Some(_) => {}
+ None => {
+ extension = Some((
+ name.clone(),
+ metadata.inner().get(EXTENSION_TYPE_METADATA_KEY).cloned(),
+ ));
+ }
+ }
+ }
+ for (key, value) in metadata.inner() {
+ if key == EXTENSION_TYPE_NAME_KEY || key == EXTENSION_TYPE_METADATA_KEY {
+ continue;
+ }
+ merged.entry(key.clone()).or_insert_with(|| value.clone());
+ }
+ }
+
+ if let Some((name, extension_metadata)) = extension {
+ merged.insert(EXTENSION_TYPE_NAME_KEY.to_string(), name);
+ if let Some(extension_metadata) = extension_metadata {
+ merged.insert(EXTENSION_TYPE_METADATA_KEY.to_string(), extension_metadata);
+ }
+ }
+
+ Ok(FieldMetadata::from(merged))
+}
+
/// Given a data type represented by storage and optional metadata, generate
/// a user-facing string
///
@@ -412,3 +482,98 @@ impl From<&HashMap<String, String>> for FieldMetadata {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn coerced_fields_metadata_merges_with_earlier_precedence() {
+ let a = Field::new("a", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([
+ ("k".to_string(), "v1".to_string()),
+ ("only_a".to_string(), "a".to_string()),
+ ]),
+ );
+ let b = Field::new("b", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([
+ ("k".to_string(), "v2".to_string()),
+ ("only_b".to_string(), "b".to_string()),
+ ]),
+ );
+ let result = coerced_fields_metadata([&a, &b]).unwrap();
+ assert_eq!(result.inner().get("k").map(String::as_str), Some("v1"));
+ assert_eq!(result.inner().get("only_a").map(String::as_str), Some("a"));
+ assert_eq!(result.inner().get("only_b").map(String::as_str), Some("b"));
+ }
+
+ #[test]
+ fn coerced_fields_metadata_skips_null_typed_fields() {
+ let null = Field::new("null", DataType::Null, true);
+ let b = Field::new("b", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([("k".to_string(), "v".to_string())]),
+ );
+ let result = coerced_fields_metadata([&null, &b]).unwrap();
+ assert_eq!(result.inner().get("k").map(String::as_str), Some("v"));
+ }
+
+ #[test]
+ fn coerced_fields_metadata_empty_when_no_field_has_any() {
+ let a = Field::new("a", DataType::Binary, true);
+ let null = Field::new("null", DataType::Null, true);
+ assert!(coerced_fields_metadata([&a, &null]).unwrap().is_empty());
+ }
+
+ #[test]
+ fn coerced_fields_metadata_errors_on_differing_extension_names() {
+ let a = Field::new("a", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([(
+ EXTENSION_TYPE_NAME_KEY.to_string(),
+ "arrow.json".to_string(),
+ )]),
+ );
+ let b = Field::new("b", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([(
+ EXTENSION_TYPE_NAME_KEY.to_string(),
+ "geoarrow.wkb".to_string(),
+ )]),
+ );
+ let err = coerced_fields_metadata([&a, &b]).unwrap_err();
+ assert!(err.to_string().contains("arrow.json"), "{err}");
+ assert!(err.to_string().contains("geoarrow.wkb"), "{err}");
+ }
+
+ #[test]
+ fn coerced_fields_metadata_keeps_extension_keys_paired() {
+ // The extension metadata travels with the field that supplied the
+ // extension name: byte-wise differences for the same name are
+ // tolerated, and a bare extension name never picks up another
+ // field's parameters.
+ let a = Field::new("a", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([(
+ EXTENSION_TYPE_NAME_KEY.to_string(),
+ "ext.type".to_string(),
+ )]),
+ );
+ let b = Field::new("b", DataType::Binary, true).with_metadata(
+ std::collections::HashMap::from([
+ (EXTENSION_TYPE_NAME_KEY.to_string(), "ext.type".to_string()),
+ (
+ EXTENSION_TYPE_METADATA_KEY.to_string(),
+ "{\"param\": 1}".to_string(),
+ ),
+ ]),
+ );
+ let result = coerced_fields_metadata([&a, &b]).unwrap();
+ assert_eq!(
+ result
+ .inner()
+ .get(EXTENSION_TYPE_NAME_KEY)
+ .map(String::as_str),
+ Some("ext.type")
+ );
+ // "a" supplied the extension name first and carried no parameters;
+ // "b"'s parameters are not grafted onto it.
+ assert_eq!(result.inner().get(EXTENSION_TYPE_METADATA_KEY), None);
+ }
+}
diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs
index 4e45fe06a1..83fab20bad 100644
--- a/datafusion/core/tests/sql/select.rs
+++ b/datafusion/core/tests/sql/select.rs
@@ -580,3 +580,43 @@ async fn test_recursive_cte_batch_schema_stable_with_order_by_limit() -> Result<
}
Ok(())
}
+
+#[tokio::test]
+async fn test_conditional_functions_field_metadata() -> Result<()> {
+ let ctx = SessionContext::new();
+ let a_metadata = HashMap::from([
+ ("shared".to_string(), "a".to_string()),
+ ("only_a".to_string(), "a".to_string()),
+ ]);
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("flag", DataType::Utf8, true).with_metadata(HashMap::from([(
+ "only_flag".to_string(),
+ "flag".to_string(),
+ )])),
+ Field::new("a", DataType::Utf8, true).with_metadata(a_metadata.clone()),
+ Field::new("b", DataType::Utf8, true).with_metadata(HashMap::from([
+ ("shared".to_string(), "b".to_string()),
+ ("only_b".to_string(), "b".to_string()),
+ ])),
+ ]));
+ ctx.register_batch("t", RecordBatch::new_empty(schema))?;
+
+ let merged = HashMap::from([
+ ("shared".to_string(), "a".to_string()),
+ ("only_a".to_string(), "a".to_string()),
+ ("only_b".to_string(), "b".to_string()),
+ ]);
+ // `df.schema()` is the schema of the plan as first built, before these
+ // functions are rewritten to CASE.
+ for (sql, expected) in [
+ ("SELECT coalesce(a, b) FROM t", &merged),
+ ("SELECT nvl(a, b) FROM t", &merged),
+ ("SELECT ifnull(a, b) FROM t", &merged),
+ ("SELECT nvl2(flag, a, b) FROM t", &merged),
+ ("SELECT coalesce(NULL, a) FROM t", &a_metadata),
+ ] {
+ let df = ctx.sql(sql).await?;
+ assert_eq!(df.schema().field(0).metadata(), expected, "{sql}");
+ }
+ Ok(())
+}
diff --git a/datafusion/functions/src/core/coalesce.rs b/datafusion/functions/src/core/coalesce.rs
index d554601f5a..948b95ca0c 100644
--- a/datafusion/functions/src/core/coalesce.rs
+++ b/datafusion/functions/src/core/coalesce.rs
@@ -6,6 +6,7 @@ use arrow::array::{BooleanArray, new_null_array};
use arrow::compute::kernels::zip::zip;
use arrow::compute::{and, is_not_null, is_null};
use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::metadata::coerced_fields_metadata;
use datafusion_common::{Result, exec_err, internal_err, plan_err};
use datafusion_expr::binary::try_type_union_resolution;
use datafusion_expr::conditional_expressions::CaseBuilder;
@@ -77,7 +78,14 @@ impl ScalarUDFImpl for CoalesceFunc {
.find_or_first(|d| !d.is_null())
.unwrap()
.clone();
- Ok(Field::new(self.name(), return_type, nullable).into())
+ // Propagate field metadata (e.g. Arrow extension types) from the
+ // arguments that can supply a value; see `coerced_fields_metadata`
+ // for the merge semantics.
+ let metadata =
+ coerced_fields_metadata(args.arg_fields.iter().map(|f| f.as_ref()))?;
+ Ok(metadata
+ .add_to_field(Field::new(self.name(), return_type, nullable))
+ .into())
}
fn simplify(
diff --git a/datafusion/functions/src/core/nvl2.rs b/datafusion/functions/src/core/nvl2.rs
index b4978b4b10..2aff09fd2e 100644
--- a/datafusion/functions/src/core/nvl2.rs
+++ b/datafusion/functions/src/core/nvl2.rs
@@ -3,6 +3,7 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::metadata::coerced_fields_metadata;
use datafusion_common::{Result, internal_err, plan_err, utils::take_function_args};
use datafusion_expr::{
ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,
@@ -81,7 +82,14 @@ impl ScalarUDFImpl for NVL2Func {
let nullable =
args.arg_fields[1].is_nullable() || args.arg_fields[2].is_nullable();
let return_type = args.arg_fields[1].data_type().clone();
- Ok(Field::new(self.name(), return_type, nullable).into())
+ // The result value comes from the second or third argument; propagate
+ // field metadata (e.g. Arrow extension types) from them. The first
+ // argument is only tested for NULL and does not contribute a value.
+ let metadata =
+ coerced_fields_metadata(args.arg_fields[1..3].iter().map(|f| f.as_ref()))?;
+ Ok(metadata
+ .add_to_field(Field::new(self.name(), return_type, nullable))
+ .into())
}
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.slt
index 4f717f6224..5e0d5d6737 100644
--- a/datafusion/sqllogictest/test_files/metadata.slt
+++ b/datafusion/sqllogictest/test_files/metadata.slt
@@ -548,6 +548,77 @@ select arrow_metadata((select id from table_with_metadata where id = 1));
----
{metadata_key: the id field}
+# coalesce, nvl, ifnull and nvl2: conflicting extension types are a planning error
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ with_metadata(name, 'ARROW:extension:name', 'a.type'),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select nvl2(id,
+ with_metadata(name, 'ARROW:extension:name', 'a.type'),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+# These functions are rewritten to CASE before execution, so arrow_metadata()
+# does not observe the field they compute during planning. The tests below
+# observe it through the planning error instead: the outer coalesce only
+# reports 'a.type' if the inner expression carried it.
+
+# coalesce propagates an extension type its arguments agree on
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ coalesce(
+ with_metadata(name, 'ARROW:extension:name', 'a.type'),
+ with_metadata(l_name, 'ARROW:extension:name', 'a.type')),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+# An untyped NULL argument does not block the extension type
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ coalesce(NULL, with_metadata(name, 'ARROW:extension:name', 'a.type')),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+# An argument without metadata does not block the extension type of a later argument
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ coalesce(upper(name), with_metadata(l_name, 'ARROW:extension:name', 'a.type')),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ nvl(with_metadata(name, 'ARROW:extension:name', 'a.type'), l_name),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ ifnull(with_metadata(name, 'ARROW:extension:name', 'a.type'), l_name),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+# nvl2 propagates the extension type of its second and third arguments
+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'
+select coalesce(
+ nvl2(id, with_metadata(name, 'ARROW:extension:name', 'a.type'), l_name),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+
+# The first argument of nvl2 is only tested for NULL; its extension type does not propagate
+query T
+select coalesce(
+ nvl2(with_metadata(id, 'ARROW:extension:name', 'a.type'), name, l_name),
+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))
+from table_with_metadata;
+----
+NULL
+l_bar
+baz
+
statement ok
drop table table_with_metadata;