Skip to content

Apache DataFusion SQL Query Engine

fix: Unify schema metadata merge logic in unions to "last one wins" (#26881)

Adam Gutglicka4d71ff06ee0
diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rs
index b651f0b7e3..3fb40bb168 100644
--- a/datafusion/common/src/metadata.rs
+++ b/datafusion/common/src/metadata.rs
@@ -122,6 +122,39 @@ pub fn format_type_and_metadata(
}
}
+/// Merges the field or schema metadata of the inputs of a `UNION`.
+///
+/// Inputs are merged in order: keys present in only some inputs are kept, and
+/// when inputs disagree on a key's value, the last input wins. Inputs without
+/// metadata (e.g. `NULL` literals) contribute nothing.
+///
+/// # Example
+/// ```
+/// # use arrow::datatypes::Metadata;
+/// # use datafusion_common::metadata::merge_metadata_for_union;
+/// let first = Metadata::new().with("id", "1").with("only_first", "a");
+/// let last = Metadata::new().with("id", "2");
+/// assert_eq!(
+/// merge_metadata_for_union([&first, &last]),
+/// Metadata::new().with("id", "2").with("only_first", "a"),
+/// );
+/// ```
+pub fn merge_metadata_for_union<'a>(
+ metadatas: impl IntoIterator<Item = &'a Metadata>,
+) -> Metadata {
+ metadatas
+ .into_iter()
+ .fold(Metadata::new(), |mut merged, metadata| {
+ if merged.is_empty() {
+ // Shares the underlying map rather than copying it
+ metadata.clone()
+ } else {
+ merged.extend(metadata);
+ merged
+ }
+ })
+}
+
/// Literal metadata
///
/// Stores metadata associated with a literal expressions
diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs
index 2c59089ccb..5453e4a8f4 100644
--- a/datafusion/expr/src/expr.rs
+++ b/datafusion/expr/src/expr.rs
@@ -633,6 +633,10 @@ pub type SchemaFieldMetadata = Metadata;
/// # Returns
///
/// A new `SchemaFieldMetadata` containing only the intersected metadata
+#[deprecated(
+ since = "56.0.0",
+ note = "UNION now merges metadata with `datafusion_common::metadata::merge_metadata_for_union`"
+)]
pub fn intersect_metadata_for_union<'a>(
metadatas: impl IntoIterator<Item = &'a SchemaFieldMetadata>,
) -> SchemaFieldMetadata {
@@ -4670,6 +4674,7 @@ mod test {
}
}
+ #[expect(deprecated)]
mod intersect_metadata_tests {
use super::super::intersect_metadata_for_union;
use arrow::datatypes::Metadata;
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index ec0cf19c4a..26626d4d32 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -19,7 +19,6 @@ use super::invariants::{
use crate::builder::{unique_field_aliases, unnest_with_options};
use crate::expr::{
Alias, Placeholder, Sort as SortExpr, WindowFunction, WindowFunctionParams,
- intersect_metadata_for_union,
};
use crate::expr_rewriter::{
NamePreserver, create_col_from_scalar_expr, normalize_cols, normalize_sorts,
@@ -48,7 +47,9 @@ use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef};
use datafusion_common::cse::{NormalizeEq, Normalizeable};
use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};
-use datafusion_common::metadata::check_metadata_with_storage_equal;
+use datafusion_common::metadata::{
+ check_metadata_with_storage_equal, merge_metadata_for_union,
+};
use datafusion_common::tree_node::{
Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,
};
@@ -3663,14 +3664,14 @@ impl Union {
let mut field =
Field::new(name, data_type.clone(), final_is_nullable);
- field.set_metadata(intersect_metadata_for_union(unmerged_metadata));
+ field.set_metadata(merge_metadata_for_union(unmerged_metadata));
(None, Arc::new(field))
},
)
.collect::<Vec<(Option<TableReference>, _)>>();
- let union_schema_metadata = intersect_metadata_for_union(
+ let union_schema_metadata = merge_metadata_for_union(
inputs.iter().map(|input| input.schema().metadata()),
);
@@ -3741,14 +3742,13 @@ impl Union {
};
let mut field = Field::new(&name, data_type.clone(), nullable);
- let field_metadata = intersect_metadata_for_union(
- fields.iter().map(|field| field.metadata()),
- );
+ let field_metadata =
+ merge_metadata_for_union(fields.iter().map(|field| field.metadata()));
field.set_metadata(field_metadata);
Ok((None, Arc::new(field)))
})
.collect::<Result<_>>()?;
- let union_schema_metadata = intersect_metadata_for_union(
+ let union_schema_metadata = merge_metadata_for_union(
inputs.iter().map(|input| input.schema().metadata()),
);
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index bf3655cc4e..296fccec4f 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -15,6 +15,7 @@ use crate::utils::{NamePreserver, merge_into_schema};
use arrow::datatypes::{DataType, Field, IntervalUnit, Schema, TimeUnit};
use arrow::temporal_conversions::SECONDS_IN_DAY;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::metadata::merge_metadata_for_union;
use datafusion_common::tree_node::{
Transformed, TreeNode, TreeNodeRecursion, TreeNodeRewriter,
};
@@ -1690,17 +1691,9 @@ fn coerce_union_schema_with_schema(
.iter()
.map(|f| f.is_nullable())
.collect::<Vec<_>>();
- let mut union_field_meta = base_schema
- .fields()
- .iter()
- .map(|f| f.metadata().clone())
- .collect::<Vec<_>>();
-
- let mut metadata = base_schema.metadata().clone();
for (i, plan) in inputs.iter().enumerate() {
let plan_schema = plan.schema();
- metadata.extend(plan_schema.metadata().clone());
if plan_schema.fields().len() != base_schema.fields().len() {
return plan_err!(
@@ -1713,10 +1706,9 @@ fn coerce_union_schema_with_schema(
}
// coerce data type and nullability for each field
- for (union_datatype, union_nullable, union_field_map, plan_field) in izip!(
+ for (union_datatype, union_nullable, plan_field) in izip!(
union_datatypes.iter_mut(),
union_nullabilities.iter_mut(),
- union_field_meta.iter_mut(),
plan_schema.fields().iter()
) {
let coerced_type =
@@ -1734,22 +1726,29 @@ fn coerce_union_schema_with_schema(
*union_datatype = coerced_type;
*union_nullable = *union_nullable || plan_field.is_nullable();
- union_field_map.extend(plan_field.metadata().clone());
}
}
+
+ let all_schemas = std::iter::once(base_schema)
+ .chain(inputs.iter().map(|p| p.schema()))
+ .collect::<Vec<_>>();
+
let union_qualified_fields = izip!(
- base_schema.fields(),
+ base_schema.fields().iter().enumerate(),
union_datatypes.into_iter(),
- union_nullabilities,
- union_field_meta.into_iter()
+ union_nullabilities
)
- .map(|(field, datatype, nullable, metadata)| {
+ .map(|((i, field), datatype, nullable)| {
let mut field = Field::new(field.name().clone(), datatype, nullable);
- field.set_metadata(metadata);
+ field.set_metadata(merge_metadata_for_union(
+ all_schemas.iter().map(|schema| schema.field(i).metadata()),
+ ));
(None, field.into())
})
.collect::<Vec<_>>();
+ let metadata =
+ merge_metadata_for_union(all_schemas.iter().map(|schema| schema.metadata()));
DFSchema::new_with_metadata(union_qualified_fields, metadata)
}
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index 830138ac8f..e0d62584f9 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -33,9 +33,10 @@ use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::ObservedStream;
use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};
-use arrow::datatypes::{Metadata, Schema, SchemaRef};
+use arrow::datatypes::{Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::metadata::merge_metadata_for_union;
use datafusion_common::stats::NdvFallback;
use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
@@ -997,47 +998,34 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {
}
}
+ let schemas: Vec<SchemaRef> = inputs.iter().map(|input| input.schema()).collect();
let fields = (0..first_field_count)
.map(|i| {
// We take the name from the left side of the union to match how names are coerced during logical planning,
// which also uses the left side names.
let base_field = first_schema.field(i).clone();
- // Select the first nullable field, or the first field if none are
- // nullable, before merging metadata. Otherwise, each non-nullable
- // candidate would repeat the full metadata merge.
- let (input_idx, input) = inputs
+ // Take the first nullable field, or the first field if none are
+ // nullable.
+ let field = schemas
.iter()
- .enumerate()
- .find_or_first(|(_, input)| input.schema().field(i).is_nullable())
+ .find_or_first(|schema| schema.field(i).is_nullable())
// Empty inputs are rejected above.
- .unwrap();
- let field = input.schema().field(i).clone();
- let mut metadata = field.metadata().clone();
+ .unwrap()
+ .field(i)
+ .clone();
- // Preserve precedence: the selected field's metadata comes first,
- // followed by the other inputs in their original order.
- let other_metadatas = inputs
- .iter()
- .enumerate()
- .filter(|(other_idx, _)| *other_idx != input_idx)
- .flat_map(|(_, other_input)| {
- other_input.schema().field(i).metadata().clone().into_iter()
- });
- metadata.extend(other_metadatas);
+ let metadata = merge_metadata_for_union(
+ schemas.iter().map(|schema| schema.field(i).metadata()),
+ );
field.with_metadata(metadata).with_name(base_field.name())
})
.collect::<Vec<_>>();
- let all_metadata_merged: Metadata = inputs
- .iter()
- .flat_map(|i| i.schema().metadata().clone().into_iter())
- .collect();
+ let metadata =
+ merge_metadata_for_union(schemas.iter().map(|schema| schema.metadata()));
- Ok(Arc::new(Schema::new_with_metadata(
- fields,
- all_metadata_merged,
- )))
+ Ok(Arc::new(Schema::new_with_metadata(fields, metadata)))
}
/// CombinedRecordBatchStream can be used to combine a Vec of SendableRecordBatchStreams into one
@@ -1117,7 +1105,7 @@ mod tests {
use crate::test::{self, TestMemoryExec};
use arrow::compute::SortOptions;
- use arrow::datatypes::{DataType, Field};
+ use arrow::datatypes::{DataType, Field, Metadata};
use datafusion_common::SplitPoint;
use datafusion_common::stats::Precision;
use datafusion_common::{ColumnStatistics, ScalarValue};
@@ -1727,14 +1715,14 @@ mod tests {
async fn test_union_schema_metadata_and_nullability() -> Result<()> {
use arrow::array::Int32Array;
- // Field metadata from the selected field is merged first, followed by
- // the other inputs in order. Schema metadata always follows input order.
- for (nullability, expected_field_value) in [
- ([false, false, false], "input2"),
- ([true, false, false], "input2"),
- ([false, true, false], "input2"),
- ([false, false, true], "input1"),
- ([true, true, true], "input2"),
+ // Field and schema metadata are merged in input order, so the last
+ // input wins regardless of which field supplies the nullability.
+ for nullability in [
+ [false, false, false],
+ [true, false, false],
+ [false, true, false],
+ [false, false, true],
+ [true, true, true],
] {
let inputs = nullability
.iter()
@@ -1760,19 +1748,17 @@ mod tests {
.collect::<Result<Vec<_>>>()?;
let expected_metadata = Metadata::from([
- ("shared".to_owned(), expected_field_value.to_owned()),
+ ("shared".to_owned(), "input2".to_owned()),
("only0".to_owned(), "kept".to_owned()),
("only1".to_owned(), "kept".to_owned()),
("only2".to_owned(), "kept".to_owned()),
]);
- let mut expected_schema_metadata = expected_metadata.clone();
- expected_schema_metadata.insert("shared".to_owned(), "input2".to_owned());
let expected_schema = Arc::new(Schema::new_with_metadata(
vec![
Field::new("value0", DataType::Int32, nullability.contains(&true))
- .with_metadata(expected_metadata),
+ .with_metadata(expected_metadata.clone()),
],
- expected_schema_metadata,
+ expected_metadata,
));
let union = UnionExec::try_new(inputs)?;
diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.slt
index 749ac6ba37..4f717f6224 100644
--- a/datafusion/sqllogictest/test_files/metadata.slt
+++ b/datafusion/sqllogictest/test_files/metadata.slt
@@ -127,8 +127,9 @@ NULL NULL l_bar
# Regression test: UNION ALL + optimize_projections pruning unused columns causes
# metadata loss when one branch has NULL literals (empty metadata) and the other
# has field metadata. The optimizer prunes unused columns, triggering recompute_schema
-# which rebuilds the Union schema via intersect_metadata_for_union. Previously, this
-# intersection would drop metadata when any branch had empty metadata (from NULL literals).
+# which rebuilds the Union schema via merge_metadata_for_union. Previously, the
+# metadata intersection used there dropped metadata when any branch had empty metadata
+# (from NULL literals).
# See https://github.com/apache/datafusion/issues/19049
query T
SELECT name FROM (
@@ -170,6 +171,34 @@ no_baz
no_foo
NULL
+# Regression test: conflicting field metadata across UNION legs with different
+# nullability. The logical and physical planners must agree on which value wins.
+# See https://github.com/apache/datafusion/issues/26880
+query I rowsort
+SELECT with_metadata(1, 'id', '1') UNION SELECT with_metadata(NULL, 'id', '2');
+----
+1
+NULL
+
+# Same, with the nullable leg first
+query I rowsort
+SELECT with_metadata(NULL, 'id', '2') UNION SELECT with_metadata(1, 'id', '1');
+----
+1
+NULL
+
+# Same, but the unused column `y` is pruned, so recompute_schema rebuilds the
+# Union schema with Union::try_new
+query I rowsort
+SELECT x FROM (
+ SELECT with_metadata(1, 'id', '1') AS x, 1 AS y
+ UNION ALL
+ SELECT with_metadata(NULL, 'id', '2') AS x, 2 AS y
+) GROUP BY x;
+----
+1
+NULL
+
query P rowsort
SELECT ts
FROM ((