Apache DataFusion SQL Query Engine
fix: propagate field metadata through coalesce, nvl, and nvl2 (#24865)
James Willisbe44fab874ab
diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rsindex 3fb40bb168..8c197d473e 100644--- a/datafusion/common/src/metadata.rs+++ b/datafusion/common/src/metadata.rs@@ -5,6 +5,7 @@use std::{collections::BTreeMap, sync::Arc};use arrow::datatypes::{DataType, Field, FieldRef, Metadata};+use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};use hashbrown::HashMap;use crate::{DataFusionError, ScalarValue, error::_plan_err};@@ -104,6 +105,75 @@ pub fn check_metadata_with_storage_equal(Ok(())}+/// Compute the field metadata to propagate for an expression whose result+/// may come from any one of several inputs coerced to a common type, such as+/// `COALESCE`, `NVL2`, or `CASE WHEN`.+///+/// Metadata is merged across the fields whose data type is not `Null` (an+/// untyped NULL literal carries no metadata and cannot contribute a typed+/// value), with earlier fields taking precedence for repeated keys. Unrelated+/// keys are not required to agree: metadata that arrived with the data (e.g.+/// from an Arrow file or an embedded Arrow schema in a Parquet file) must not+/// cause an extension type to be silently dropped from the result.+///+/// The Arrow extension type keys are handled as a unit, like in+/// [`Expr::Cast`] output field derivation:+/// - Differing values of `ARROW:extension:name` produce a planning error;+/// the inputs are of different logical types even though they were coerced+/// to a common storage type.+/// - `ARROW:extension:metadata` is taken from the field that supplied the+/// extension name, never combined from another field. Byte-wise differences+/// in the extension metadata of fields that agree on the extension name are+/// tolerated, since extension parameters are often JSON-encoded, where byte+/// equality is too strict.+///+/// [`Expr::Cast`]: https://docs.rs/datafusion/latest/datafusion/logical_expr/enum.Expr.html#variant.Cast+pub fn coerced_fields_metadata<'a>(+ fields: impl IntoIterator<Item = &'a Field>,+) -> Result<FieldMetadata, DataFusionError> {+ let mut merged: BTreeMap<String, String> = BTreeMap::new();+ let mut extension: Option<(String, Option<String>)> = None;++ for field in fields {+ if field.data_type().is_null() {+ continue;+ }+ let metadata = FieldMetadata::new_from_field(field);+ if let Some(name) = metadata.inner().get(EXTENSION_TYPE_NAME_KEY) {+ match &extension {+ Some((first_name, _)) if first_name != name => {+ return _plan_err!(+ "Cannot merge field metadata with differing values of \+ '{EXTENSION_TYPE_NAME_KEY}': '{first_name}' and '{name}'"+ );+ }+ Some(_) => {}+ None => {+ extension = Some((+ name.clone(),+ metadata.inner().get(EXTENSION_TYPE_METADATA_KEY).cloned(),+ ));+ }+ }+ }+ for (key, value) in metadata.inner() {+ if key == EXTENSION_TYPE_NAME_KEY || key == EXTENSION_TYPE_METADATA_KEY {+ continue;+ }+ merged.entry(key.clone()).or_insert_with(|| value.clone());+ }+ }++ if let Some((name, extension_metadata)) = extension {+ merged.insert(EXTENSION_TYPE_NAME_KEY.to_string(), name);+ if let Some(extension_metadata) = extension_metadata {+ merged.insert(EXTENSION_TYPE_METADATA_KEY.to_string(), extension_metadata);+ }+ }++ Ok(FieldMetadata::from(merged))+}+/// Given a data type represented by storage and optional metadata, generate/// a user-facing string///@@ -412,3 +482,98 @@ impl From<&HashMap<String, String>> for FieldMetadata {}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn coerced_fields_metadata_merges_with_earlier_precedence() {+ let a = Field::new("a", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([+ ("k".to_string(), "v1".to_string()),+ ("only_a".to_string(), "a".to_string()),+ ]),+ );+ let b = Field::new("b", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([+ ("k".to_string(), "v2".to_string()),+ ("only_b".to_string(), "b".to_string()),+ ]),+ );+ let result = coerced_fields_metadata([&a, &b]).unwrap();+ assert_eq!(result.inner().get("k").map(String::as_str), Some("v1"));+ assert_eq!(result.inner().get("only_a").map(String::as_str), Some("a"));+ assert_eq!(result.inner().get("only_b").map(String::as_str), Some("b"));+ }++ #[test]+ fn coerced_fields_metadata_skips_null_typed_fields() {+ let null = Field::new("null", DataType::Null, true);+ let b = Field::new("b", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([("k".to_string(), "v".to_string())]),+ );+ let result = coerced_fields_metadata([&null, &b]).unwrap();+ assert_eq!(result.inner().get("k").map(String::as_str), Some("v"));+ }++ #[test]+ fn coerced_fields_metadata_empty_when_no_field_has_any() {+ let a = Field::new("a", DataType::Binary, true);+ let null = Field::new("null", DataType::Null, true);+ assert!(coerced_fields_metadata([&a, &null]).unwrap().is_empty());+ }++ #[test]+ fn coerced_fields_metadata_errors_on_differing_extension_names() {+ let a = Field::new("a", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([(+ EXTENSION_TYPE_NAME_KEY.to_string(),+ "arrow.json".to_string(),+ )]),+ );+ let b = Field::new("b", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([(+ EXTENSION_TYPE_NAME_KEY.to_string(),+ "geoarrow.wkb".to_string(),+ )]),+ );+ let err = coerced_fields_metadata([&a, &b]).unwrap_err();+ assert!(err.to_string().contains("arrow.json"), "{err}");+ assert!(err.to_string().contains("geoarrow.wkb"), "{err}");+ }++ #[test]+ fn coerced_fields_metadata_keeps_extension_keys_paired() {+ // The extension metadata travels with the field that supplied the+ // extension name: byte-wise differences for the same name are+ // tolerated, and a bare extension name never picks up another+ // field's parameters.+ let a = Field::new("a", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([(+ EXTENSION_TYPE_NAME_KEY.to_string(),+ "ext.type".to_string(),+ )]),+ );+ let b = Field::new("b", DataType::Binary, true).with_metadata(+ std::collections::HashMap::from([+ (EXTENSION_TYPE_NAME_KEY.to_string(), "ext.type".to_string()),+ (+ EXTENSION_TYPE_METADATA_KEY.to_string(),+ "{\"param\": 1}".to_string(),+ ),+ ]),+ );+ let result = coerced_fields_metadata([&a, &b]).unwrap();+ assert_eq!(+ result+ .inner()+ .get(EXTENSION_TYPE_NAME_KEY)+ .map(String::as_str),+ Some("ext.type")+ );+ // "a" supplied the extension name first and carried no parameters;+ // "b"'s parameters are not grafted onto it.+ assert_eq!(result.inner().get(EXTENSION_TYPE_METADATA_KEY), None);+ }+}diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rsindex 4e45fe06a1..83fab20bad 100644--- a/datafusion/core/tests/sql/select.rs+++ b/datafusion/core/tests/sql/select.rs@@ -580,3 +580,43 @@ async fn test_recursive_cte_batch_schema_stable_with_order_by_limit() -> Result<}Ok(())}++#[tokio::test]+async fn test_conditional_functions_field_metadata() -> Result<()> {+ let ctx = SessionContext::new();+ let a_metadata = HashMap::from([+ ("shared".to_string(), "a".to_string()),+ ("only_a".to_string(), "a".to_string()),+ ]);+ let schema = Arc::new(Schema::new(vec![+ Field::new("flag", DataType::Utf8, true).with_metadata(HashMap::from([(+ "only_flag".to_string(),+ "flag".to_string(),+ )])),+ Field::new("a", DataType::Utf8, true).with_metadata(a_metadata.clone()),+ Field::new("b", DataType::Utf8, true).with_metadata(HashMap::from([+ ("shared".to_string(), "b".to_string()),+ ("only_b".to_string(), "b".to_string()),+ ])),+ ]));+ ctx.register_batch("t", RecordBatch::new_empty(schema))?;++ let merged = HashMap::from([+ ("shared".to_string(), "a".to_string()),+ ("only_a".to_string(), "a".to_string()),+ ("only_b".to_string(), "b".to_string()),+ ]);+ // `df.schema()` is the schema of the plan as first built, before these+ // functions are rewritten to CASE.+ for (sql, expected) in [+ ("SELECT coalesce(a, b) FROM t", &merged),+ ("SELECT nvl(a, b) FROM t", &merged),+ ("SELECT ifnull(a, b) FROM t", &merged),+ ("SELECT nvl2(flag, a, b) FROM t", &merged),+ ("SELECT coalesce(NULL, a) FROM t", &a_metadata),+ ] {+ let df = ctx.sql(sql).await?;+ assert_eq!(df.schema().field(0).metadata(), expected, "{sql}");+ }+ Ok(())+}diff --git a/datafusion/functions/src/core/coalesce.rs b/datafusion/functions/src/core/coalesce.rsindex d554601f5a..948b95ca0c 100644--- a/datafusion/functions/src/core/coalesce.rs+++ b/datafusion/functions/src/core/coalesce.rs@@ -6,6 +6,7 @@ use arrow::array::{BooleanArray, new_null_array};use arrow::compute::kernels::zip::zip;use arrow::compute::{and, is_not_null, is_null};use arrow::datatypes::{DataType, Field, FieldRef};+use datafusion_common::metadata::coerced_fields_metadata;use datafusion_common::{Result, exec_err, internal_err, plan_err};use datafusion_expr::binary::try_type_union_resolution;use datafusion_expr::conditional_expressions::CaseBuilder;@@ -77,7 +78,14 @@ impl ScalarUDFImpl for CoalesceFunc {.find_or_first(|d| !d.is_null()).unwrap().clone();- Ok(Field::new(self.name(), return_type, nullable).into())+ // Propagate field metadata (e.g. Arrow extension types) from the+ // arguments that can supply a value; see `coerced_fields_metadata`+ // for the merge semantics.+ let metadata =+ coerced_fields_metadata(args.arg_fields.iter().map(|f| f.as_ref()))?;+ Ok(metadata+ .add_to_field(Field::new(self.name(), return_type, nullable))+ .into())}fn simplify(diff --git a/datafusion/functions/src/core/nvl2.rs b/datafusion/functions/src/core/nvl2.rsindex b4978b4b10..2aff09fd2e 100644--- a/datafusion/functions/src/core/nvl2.rs+++ b/datafusion/functions/src/core/nvl2.rs@@ -3,6 +3,7 @@// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreementsuse arrow::datatypes::{DataType, Field, FieldRef};+use datafusion_common::metadata::coerced_fields_metadata;use datafusion_common::{Result, internal_err, plan_err, utils::take_function_args};use datafusion_expr::{ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,@@ -81,7 +82,14 @@ impl ScalarUDFImpl for NVL2Func {let nullable =args.arg_fields[1].is_nullable() || args.arg_fields[2].is_nullable();let return_type = args.arg_fields[1].data_type().clone();- Ok(Field::new(self.name(), return_type, nullable).into())+ // The result value comes from the second or third argument; propagate+ // field metadata (e.g. Arrow extension types) from them. The first+ // argument is only tested for NULL and does not contribute a value.+ let metadata =+ coerced_fields_metadata(args.arg_fields[1..3].iter().map(|f| f.as_ref()))?;+ Ok(metadata+ .add_to_field(Field::new(self.name(), return_type, nullable))+ .into())}fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex 4f717f6224..5e0d5d6737 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -548,6 +548,77 @@ select arrow_metadata((select id from table_with_metadata where id = 1));----{metadata_key: the id field}+# coalesce, nvl, ifnull and nvl2: conflicting extension types are a planning error+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ with_metadata(name, 'ARROW:extension:name', 'a.type'),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select nvl2(id,+ with_metadata(name, 'ARROW:extension:name', 'a.type'),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++# These functions are rewritten to CASE before execution, so arrow_metadata()+# does not observe the field they compute during planning. The tests below+# observe it through the planning error instead: the outer coalesce only+# reports 'a.type' if the inner expression carried it.++# coalesce propagates an extension type its arguments agree on+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ coalesce(+ with_metadata(name, 'ARROW:extension:name', 'a.type'),+ with_metadata(l_name, 'ARROW:extension:name', 'a.type')),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++# An untyped NULL argument does not block the extension type+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ coalesce(NULL, with_metadata(name, 'ARROW:extension:name', 'a.type')),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++# An argument without metadata does not block the extension type of a later argument+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ coalesce(upper(name), with_metadata(l_name, 'ARROW:extension:name', 'a.type')),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ nvl(with_metadata(name, 'ARROW:extension:name', 'a.type'), l_name),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ ifnull(with_metadata(name, 'ARROW:extension:name', 'a.type'), l_name),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++# nvl2 propagates the extension type of its second and third arguments+statement error Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a\.type' and 'b\.type'+select coalesce(+ nvl2(id, with_metadata(name, 'ARROW:extension:name', 'a.type'), l_name),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;++# The first argument of nvl2 is only tested for NULL; its extension type does not propagate+query T+select coalesce(+ nvl2(with_metadata(id, 'ARROW:extension:name', 'a.type'), name, l_name),+ with_metadata(l_name, 'ARROW:extension:name', 'b.type'))+from table_with_metadata;+----+NULL+l_bar+baz+statement okdrop table table_with_metadata;