Apache DataFusion SQL Query Engine
perf: avoid quadratic metadata merging in union schemas (#26790)
Kumar Ujjawal7ca68cf35680
diff --git a/datafusion/physical-plan/benches/union_schema.rs b/datafusion/physical-plan/benches/union_schema.rsindex b0a946f511..08386e3ce8 100644--- a/datafusion/physical-plan/benches/union_schema.rs+++ b/datafusion/physical-plan/benches/union_schema.rs@@ -12,6 +12,9 @@//! the scan runs, fails on the final child, then the full merge runs anyway.//! - `names_differ`: the first field is renamed in every child but the first,//! the common real-world unequal union where equality fails immediately.+//!+//! The non-nullable cases also require scanning all children to determine+//! output nullability, rather than stopping at the first nullable field.use std::collections::HashMap;use std::sync::Arc;@@ -32,10 +35,10 @@ fn metadata(tag: &str, i: usize) -> HashMap<String, String> {.collect()}-fn flat_schema() -> Schema {+fn flat_schema(nullable: bool) -> Schema {let fields: Vec<Field> = (0..NUM_FIELDS).map(|i| {- Field::new(format!("col_{i}"), DataType::Int64, true)+ Field::new(format!("col_{i}"), DataType::Int64, nullable).with_metadata(metadata("f", i))}).collect();@@ -107,8 +110,9 @@ fn children_last_differs(schema: &Schema, n: usize) -> Vec<Arc<dyn ExecutionPlanfn bench_union_construction(c: &mut Criterion) {for (suffix, schema, sizes) in [- ("", flat_schema(), &[100usize, 1000, 4000][..]),+ ("", flat_schema(true), &[100usize, 1000, 4000][..]),("_nested", nested_schema(), &[1000, 4000][..]),+ ("_non_nullable", flat_schema(false), &[10, 100, 1000][..]),] {let mut group = c.benchmark_group(format!("union_exec_try_new{suffix}"));for &n in sizes {diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 542c9db334..830138ac8f 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -33,7 +33,7 @@ use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::ObservedStream;use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};-use arrow::datatypes::{Field, Metadata, Schema, SchemaRef};+use arrow::datatypes::{Metadata, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::NdvFallback;@@ -970,8 +970,8 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {let first_schema = inputs[0].schema();// Fast path: when every input already shares the first input's schema, the- // field-by-field metadata/nullability merge below is redundant work that- // scales as O(n^2 * fields). This is common in practice: unions built from+ // field-by-field metadata/nullability merge below is redundant work.+ // This is common in practice: unions built from// repartitioned copies of the same plan (e.g. observed in InfluxDB) hand us// children that all carry the exact same schema. A pointer-equality check// catches the shared-`Arc` case for free, and a content `==` comparison@@ -1003,31 +1003,29 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {// which also uses the left side names.let base_field = first_schema.field(i).clone();- // Coerce metadata and nullability across all inputs-- inputs+ // Select the first nullable field, or the first field if none are+ // nullable, before merging metadata. Otherwise, each non-nullable+ // candidate would repeat the full metadata merge.+ let (input_idx, input) = inputs.iter().enumerate()- .map(|(input_idx, input)| {- let field = input.schema().field(i).clone();- let mut metadata = field.metadata().clone();-- let other_metadatas = inputs- .iter()- .enumerate()- .filter(|(other_idx, _)| *other_idx != input_idx)- .flat_map(|(_, other_input)| {- other_input.schema().field(i).metadata().clone().into_iter()- });-- metadata.extend(other_metadatas);- field.with_metadata(metadata)- })- .find_or_first(Field::is_nullable)- // We can unwrap this because if inputs was empty, this would've already panic'ed when we- // indexed into inputs[0].- .unwrap()- .with_name(base_field.name())+ .find_or_first(|(_, input)| input.schema().field(i).is_nullable())+ // Empty inputs are rejected above.+ .unwrap();+ let field = input.schema().field(i).clone();+ let mut metadata = field.metadata().clone();++ // Preserve precedence: the selected field's metadata comes first,+ // followed by the other inputs in their original order.+ let other_metadatas = inputs+ .iter()+ .enumerate()+ .filter(|(other_idx, _)| *other_idx != input_idx)+ .flat_map(|(_, other_input)| {+ other_input.schema().field(i).metadata().clone().into_iter()+ });+ metadata.extend(other_metadatas);+ field.with_metadata(metadata).with_name(base_field.name())}).collect::<Vec<_>>();@@ -1119,7 +1117,7 @@ mod tests {use crate::test::{self, TestMemoryExec};use arrow::compute::SortOptions;- use arrow::datatypes::DataType;+ use arrow::datatypes::{DataType, Field};use datafusion_common::SplitPoint;use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, ScalarValue};@@ -1725,6 +1723,84 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_union_schema_metadata_and_nullability() -> Result<()> {+ use arrow::array::Int32Array;++ // Field metadata from the selected field is merged first, followed by+ // the other inputs in order. Schema metadata always follows input order.+ for (nullability, expected_field_value) in [+ ([false, false, false], "input2"),+ ([true, false, false], "input2"),+ ([false, true, false], "input2"),+ ([false, false, true], "input1"),+ ([true, true, true], "input2"),+ ] {+ let inputs = nullability+ .iter()+ .enumerate()+ .map(|(i, &nullable)| {+ let metadata = Metadata::from([+ ("shared".to_owned(), format!("input{i}")),+ (format!("only{i}"), "kept".to_owned()),+ ]);+ let field =+ Field::new(format!("value{i}"), DataType::Int32, nullable)+ .with_metadata(metadata.clone());+ let schema =+ Arc::new(Schema::new_with_metadata(vec![field], metadata));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![i as i32]))],+ )?;+ let input: Arc<dyn ExecutionPlan> =+ TestMemoryExec::try_new_exec(&[vec![batch]], schema, None)?;+ Ok(input)+ })+ .collect::<Result<Vec<_>>>()?;++ let expected_metadata = Metadata::from([+ ("shared".to_owned(), expected_field_value.to_owned()),+ ("only0".to_owned(), "kept".to_owned()),+ ("only1".to_owned(), "kept".to_owned()),+ ("only2".to_owned(), "kept".to_owned()),+ ]);+ let mut expected_schema_metadata = expected_metadata.clone();+ expected_schema_metadata.insert("shared".to_owned(), "input2".to_owned());+ let expected_schema = Arc::new(Schema::new_with_metadata(+ vec![+ Field::new("value0", DataType::Int32, nullability.contains(&true))+ .with_metadata(expected_metadata),+ ],+ expected_schema_metadata,+ ));++ let union = UnionExec::try_new(inputs)?;+ assert_eq!(union.schema(), expected_schema, "{nullability:?}");+ let batches = collect(union, Arc::new(TaskContext::default())).await?;+ datafusion_common::assert_batches_sorted_eq!(+ [+ "+--------+",+ "| value0 |",+ "+--------+",+ "| 0 |",+ "| 1 |",+ "| 2 |",+ "+--------+"+ ],+ &batches+ );+ for batch in batches {+ assert_eq!(+ batch.schema().fields(),+ expected_schema.fields(),+ "{nullability:?}"+ );+ }+ }+ Ok(())+ }+#[test]fn test_union_schema_mismatch() {// Test that UnionExec properly rejects inputs with different field counts