Apache DataFusion SQL Query Engine
fix: Unify schema metadata merge logic in unions to "last one wins" (#26881)
Adam Gutglicka4d71ff06ee0
diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rsindex b651f0b7e3..3fb40bb168 100644--- a/datafusion/common/src/metadata.rs+++ b/datafusion/common/src/metadata.rs@@ -122,6 +122,39 @@ pub fn format_type_and_metadata(}}+/// Merges the field or schema metadata of the inputs of a `UNION`.+///+/// Inputs are merged in order: keys present in only some inputs are kept, and+/// when inputs disagree on a key's value, the last input wins. Inputs without+/// metadata (e.g. `NULL` literals) contribute nothing.+///+/// # Example+/// ```+/// # use arrow::datatypes::Metadata;+/// # use datafusion_common::metadata::merge_metadata_for_union;+/// let first = Metadata::new().with("id", "1").with("only_first", "a");+/// let last = Metadata::new().with("id", "2");+/// assert_eq!(+/// merge_metadata_for_union([&first, &last]),+/// Metadata::new().with("id", "2").with("only_first", "a"),+/// );+/// ```+pub fn merge_metadata_for_union<'a>(+ metadatas: impl IntoIterator<Item = &'a Metadata>,+) -> Metadata {+ metadatas+ .into_iter()+ .fold(Metadata::new(), |mut merged, metadata| {+ if merged.is_empty() {+ // Shares the underlying map rather than copying it+ metadata.clone()+ } else {+ merged.extend(metadata);+ merged+ }+ })+}+/// Literal metadata////// Stores metadata associated with a literal expressionsdiff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rsindex 2c59089ccb..5453e4a8f4 100644--- a/datafusion/expr/src/expr.rs+++ b/datafusion/expr/src/expr.rs@@ -633,6 +633,10 @@ pub type SchemaFieldMetadata = Metadata;/// # Returns////// A new `SchemaFieldMetadata` containing only the intersected metadata+#[deprecated(+ since = "56.0.0",+ note = "UNION now merges metadata with `datafusion_common::metadata::merge_metadata_for_union`"+)]pub fn intersect_metadata_for_union<'a>(metadatas: impl IntoIterator<Item = &'a SchemaFieldMetadata>,) -> SchemaFieldMetadata {@@ -4670,6 +4674,7 @@ mod test {}}+ #[expect(deprecated)]mod intersect_metadata_tests {use super::super::intersect_metadata_for_union;use arrow::datatypes::Metadata;diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex ec0cf19c4a..26626d4d32 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -19,7 +19,6 @@ use super::invariants::{use crate::builder::{unique_field_aliases, unnest_with_options};use crate::expr::{Alias, Placeholder, Sort as SortExpr, WindowFunction, WindowFunctionParams,- intersect_metadata_for_union,};use crate::expr_rewriter::{NamePreserver, create_col_from_scalar_expr, normalize_cols, normalize_sorts,@@ -48,7 +47,9 @@ use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef};use datafusion_common::cse::{NormalizeEq, Normalizeable};use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};-use datafusion_common::metadata::check_metadata_with_storage_equal;+use datafusion_common::metadata::{+ check_metadata_with_storage_equal, merge_metadata_for_union,+};use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,};@@ -3663,14 +3664,14 @@ impl Union {let mut field =Field::new(name, data_type.clone(), final_is_nullable);- field.set_metadata(intersect_metadata_for_union(unmerged_metadata));+ field.set_metadata(merge_metadata_for_union(unmerged_metadata));(None, Arc::new(field))},).collect::<Vec<(Option<TableReference>, _)>>();- let union_schema_metadata = intersect_metadata_for_union(+ let union_schema_metadata = merge_metadata_for_union(inputs.iter().map(|input| input.schema().metadata()),);@@ -3741,14 +3742,13 @@ impl Union {};let mut field = Field::new(&name, data_type.clone(), nullable);- let field_metadata = intersect_metadata_for_union(- fields.iter().map(|field| field.metadata()),- );+ let field_metadata =+ merge_metadata_for_union(fields.iter().map(|field| field.metadata()));field.set_metadata(field_metadata);Ok((None, Arc::new(field)))}).collect::<Result<_>>()?;- let union_schema_metadata = intersect_metadata_for_union(+ let union_schema_metadata = merge_metadata_for_union(inputs.iter().map(|input| input.schema().metadata()),);diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex bf3655cc4e..296fccec4f 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -15,6 +15,7 @@ use crate::utils::{NamePreserver, merge_into_schema};use arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};use arrow::temporal_conversions::SECONDS_IN_DAY;use datafusion_common::config::ConfigOptions;+use datafusion_common::metadata::merge_metadata_for_union;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter,};@@ -1690,17 +1691,9 @@ fn coerce_union_schema_with_schema(.iter().map(|f| f.is_nullable()).collect::<Vec<_>>();- let mut union_field_meta = base_schema- .fields()- .iter()- .map(|f| f.metadata().clone())- .collect::<Vec<_>>();-- let mut metadata = base_schema.metadata().clone();for (i, plan) in inputs.iter().enumerate() {let plan_schema = plan.schema();- metadata.extend(plan_schema.metadata().clone());if plan_schema.fields().len() != base_schema.fields().len() {return plan_err!(@@ -1713,10 +1706,9 @@ fn coerce_union_schema_with_schema(}// coerce data type and nullability for each field- for (union_datatype, union_nullable, union_field_map, plan_field) in izip!(+ for (union_datatype, union_nullable, plan_field) in izip!(union_datatypes.iter_mut(),union_nullabilities.iter_mut(),- union_field_meta.iter_mut(),plan_schema.fields().iter()) {let coerced_type =@@ -1734,22 +1726,29 @@ fn coerce_union_schema_with_schema(*union_datatype = coerced_type;*union_nullable = *union_nullable || plan_field.is_nullable();- union_field_map.extend(plan_field.metadata().clone());}}++ let all_schemas = std::iter::once(base_schema)+ .chain(inputs.iter().map(|p| p.schema()))+ .collect::<Vec<_>>();+let union_qualified_fields = izip!(- base_schema.fields(),+ base_schema.fields().iter().enumerate(),union_datatypes.into_iter(),- union_nullabilities,- union_field_meta.into_iter()+ union_nullabilities)- .map(|(field, datatype, nullable, metadata)| {+ .map(|((i, field), datatype, nullable)| {let mut field = Field::new(field.name().clone(), datatype, nullable);- field.set_metadata(metadata);+ field.set_metadata(merge_metadata_for_union(+ all_schemas.iter().map(|schema| schema.field(i).metadata()),+ ));(None, field.into())}).collect::<Vec<_>>();+ let metadata =+ merge_metadata_for_union(all_schemas.iter().map(|schema| schema.metadata()));DFSchema::new_with_metadata(union_qualified_fields, metadata)}diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 830138ac8f..e0d62584f9 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -33,9 +33,10 @@ use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::ObservedStream;use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};-use arrow::datatypes::{Metadata, Schema, SchemaRef};+use arrow::datatypes::{Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;+use datafusion_common::metadata::merge_metadata_for_union;use datafusion_common::stats::NdvFallback;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{@@ -997,47 +998,34 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {}}+ let schemas: Vec<SchemaRef> = inputs.iter().map(|input| input.schema()).collect();let fields = (0..first_field_count).map(|i| {// We take the name from the left side of the union to match how names are coerced during logical planning,// which also uses the left side names.let base_field = first_schema.field(i).clone();- // Select the first nullable field, or the first field if none are- // nullable, before merging metadata. Otherwise, each non-nullable- // candidate would repeat the full metadata merge.- let (input_idx, input) = inputs+ // Take the first nullable field, or the first field if none are+ // nullable.+ let field = schemas.iter()- .enumerate()- .find_or_first(|(_, input)| input.schema().field(i).is_nullable())+ .find_or_first(|schema| schema.field(i).is_nullable())// Empty inputs are rejected above.- .unwrap();- let field = input.schema().field(i).clone();- let mut metadata = field.metadata().clone();+ .unwrap()+ .field(i)+ .clone();- // Preserve precedence: the selected field's metadata comes first,- // followed by the other inputs in their original order.- let other_metadatas = inputs- .iter()- .enumerate()- .filter(|(other_idx, _)| *other_idx != input_idx)- .flat_map(|(_, other_input)| {- other_input.schema().field(i).metadata().clone().into_iter()- });- metadata.extend(other_metadatas);+ let metadata = merge_metadata_for_union(+ schemas.iter().map(|schema| schema.field(i).metadata()),+ );field.with_metadata(metadata).with_name(base_field.name())}).collect::<Vec<_>>();- let all_metadata_merged: Metadata = inputs- .iter()- .flat_map(|i| i.schema().metadata().clone().into_iter())- .collect();+ let metadata =+ merge_metadata_for_union(schemas.iter().map(|schema| schema.metadata()));- Ok(Arc::new(Schema::new_with_metadata(- fields,- all_metadata_merged,- )))+ Ok(Arc::new(Schema::new_with_metadata(fields, metadata)))}/// CombinedRecordBatchStream can be used to combine a Vec of SendableRecordBatchStreams into one@@ -1117,7 +1105,7 @@ mod tests {use crate::test::{self, TestMemoryExec};use arrow::compute::SortOptions;- use arrow::datatypes::{DataType, Field};+ use arrow::datatypes::{DataType, Field, Metadata};use datafusion_common::SplitPoint;use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, ScalarValue};@@ -1727,14 +1715,14 @@ mod tests {async fn test_union_schema_metadata_and_nullability() -> Result<()> {use arrow::array::Int32Array;- // Field metadata from the selected field is merged first, followed by- // the other inputs in order. Schema metadata always follows input order.- for (nullability, expected_field_value) in [- ([false, false, false], "input2"),- ([true, false, false], "input2"),- ([false, true, false], "input2"),- ([false, false, true], "input1"),- ([true, true, true], "input2"),+ // Field and schema metadata are merged in input order, so the last+ // input wins regardless of which field supplies the nullability.+ for nullability in [+ [false, false, false],+ [true, false, false],+ [false, true, false],+ [false, false, true],+ [true, true, true],] {let inputs = nullability.iter()@@ -1760,19 +1748,17 @@ mod tests {.collect::<Result<Vec<_>>>()?;let expected_metadata = Metadata::from([- ("shared".to_owned(), expected_field_value.to_owned()),+ ("shared".to_owned(), "input2".to_owned()),("only0".to_owned(), "kept".to_owned()),("only1".to_owned(), "kept".to_owned()),("only2".to_owned(), "kept".to_owned()),]);- let mut expected_schema_metadata = expected_metadata.clone();- expected_schema_metadata.insert("shared".to_owned(), "input2".to_owned());let expected_schema = Arc::new(Schema::new_with_metadata(vec![Field::new("value0", DataType::Int32, nullability.contains(&true))- .with_metadata(expected_metadata),+ .with_metadata(expected_metadata.clone()),],- expected_schema_metadata,+ expected_metadata,));let union = UnionExec::try_new(inputs)?;diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex 749ac6ba37..4f717f6224 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -127,8 +127,9 @@ NULL NULL l_bar# Regression test: UNION ALL + optimize_projections pruning unused columns causes# metadata loss when one branch has NULL literals (empty metadata) and the other# has field metadata. The optimizer prunes unused columns, triggering recompute_schema-# which rebuilds the Union schema via intersect_metadata_for_union. Previously, this-# intersection would drop metadata when any branch had empty metadata (from NULL literals).+# which rebuilds the Union schema via merge_metadata_for_union. Previously, the+# metadata intersection used there dropped metadata when any branch had empty metadata+# (from NULL literals).# See https://github.com/apache/datafusion/issues/19049query TSELECT name FROM (@@ -170,6 +171,34 @@ no_bazno_fooNULL+# Regression test: conflicting field metadata across UNION legs with different+# nullability. The logical and physical planners must agree on which value wins.+# See https://github.com/apache/datafusion/issues/26880+query I rowsort+SELECT with_metadata(1, 'id', '1') UNION SELECT with_metadata(NULL, 'id', '2');+----+1+NULL++# Same, with the nullable leg first+query I rowsort+SELECT with_metadata(NULL, 'id', '2') UNION SELECT with_metadata(1, 'id', '1');+----+1+NULL++# Same, but the unused column `y` is pruned, so recompute_schema rebuilds the+# Union schema with Union::try_new+query I rowsort+SELECT x FROM (+ SELECT with_metadata(1, 'id', '1') AS x, 1 AS y+ UNION ALL+ SELECT with_metadata(NULL, 'id', '2') AS x, 2 AS y+) GROUP BY x;+----+1+NULL+query P rowsortSELECT tsFROM ((