Skip to content

Apache DataFusion SQL Query Engine

perf: avoid quadratic metadata merging in union schemas (#26790)

Kumar Ujjawal7ca68cf35680
diff --git a/datafusion/physical-plan/benches/union_schema.rs b/datafusion/physical-plan/benches/union_schema.rs
index b0a946f511..08386e3ce8 100644
--- a/datafusion/physical-plan/benches/union_schema.rs
+++ b/datafusion/physical-plan/benches/union_schema.rs
@@ -12,6 +12,9 @@
//! the scan runs, fails on the final child, then the full merge runs anyway.
//! - `names_differ`: the first field is renamed in every child but the first,
//! the common real-world unequal union where equality fails immediately.
+//!
+//! The non-nullable cases also require scanning all children to determine
+//! output nullability, rather than stopping at the first nullable field.
use std::collections::HashMap;
use std::sync::Arc;
@@ -32,10 +35,10 @@ fn metadata(tag: &str, i: usize) -> HashMap<String, String> {
.collect()
}
-fn flat_schema() -> Schema {
+fn flat_schema(nullable: bool) -> Schema {
let fields: Vec<Field> = (0..NUM_FIELDS)
.map(|i| {
- Field::new(format!("col_{i}"), DataType::Int64, true)
+ Field::new(format!("col_{i}"), DataType::Int64, nullable)
.with_metadata(metadata("f", i))
})
.collect();
@@ -107,8 +110,9 @@ fn children_last_differs(schema: &Schema, n: usize) -> Vec<Arc<dyn ExecutionPlan
fn bench_union_construction(c: &mut Criterion) {
for (suffix, schema, sizes) in [
- ("", flat_schema(), &[100usize, 1000, 4000][..]),
+ ("", flat_schema(true), &[100usize, 1000, 4000][..]),
("_nested", nested_schema(), &[1000, 4000][..]),
+ ("_non_nullable", flat_schema(false), &[10, 100, 1000][..]),
] {
let mut group = c.benchmark_group(format!("union_exec_try_new{suffix}"));
for &n in sizes {
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index 542c9db334..830138ac8f 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -33,7 +33,7 @@ use crate::statistics::{ChildStats, StatisticsArgs};
use crate::stream::ObservedStream;
use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};
-use arrow::datatypes::{Field, Metadata, Schema, SchemaRef};
+use arrow::datatypes::{Metadata, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::NdvFallback;
@@ -970,8 +970,8 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {
let first_schema = inputs[0].schema();
// Fast path: when every input already shares the first input's schema, the
- // field-by-field metadata/nullability merge below is redundant work that
- // scales as O(n^2 * fields). This is common in practice: unions built from
+ // field-by-field metadata/nullability merge below is redundant work.
+ // This is common in practice: unions built from
// repartitioned copies of the same plan (e.g. observed in InfluxDB) hand us
// children that all carry the exact same schema. A pointer-equality check
// catches the shared-`Arc` case for free, and a content `==` comparison
@@ -1003,31 +1003,29 @@ fn union_schema(inputs: &[Arc<dyn ExecutionPlan>]) -> Result<SchemaRef> {
// which also uses the left side names.
let base_field = first_schema.field(i).clone();
- // Coerce metadata and nullability across all inputs
-
- inputs
+ // Select the first nullable field, or the first field if none are
+ // nullable, before merging metadata. Otherwise, each non-nullable
+ // candidate would repeat the full metadata merge.
+ let (input_idx, input) = inputs
.iter()
.enumerate()
- .map(|(input_idx, input)| {
- let field = input.schema().field(i).clone();
- let mut metadata = field.metadata().clone();
-
- let other_metadatas = inputs
- .iter()
- .enumerate()
- .filter(|(other_idx, _)| *other_idx != input_idx)
- .flat_map(|(_, other_input)| {
- other_input.schema().field(i).metadata().clone().into_iter()
- });
-
- metadata.extend(other_metadatas);
- field.with_metadata(metadata)
- })
- .find_or_first(Field::is_nullable)
- // We can unwrap this because if inputs was empty, this would've already panic'ed when we
- // indexed into inputs[0].
- .unwrap()
- .with_name(base_field.name())
+ .find_or_first(|(_, input)| input.schema().field(i).is_nullable())
+ // Empty inputs are rejected above.
+ .unwrap();
+ let field = input.schema().field(i).clone();
+ let mut metadata = field.metadata().clone();
+
+ // Preserve precedence: the selected field's metadata comes first,
+ // followed by the other inputs in their original order.
+ let other_metadatas = inputs
+ .iter()
+ .enumerate()
+ .filter(|(other_idx, _)| *other_idx != input_idx)
+ .flat_map(|(_, other_input)| {
+ other_input.schema().field(i).metadata().clone().into_iter()
+ });
+ metadata.extend(other_metadatas);
+ field.with_metadata(metadata).with_name(base_field.name())
})
.collect::<Vec<_>>();
@@ -1119,7 +1117,7 @@ mod tests {
use crate::test::{self, TestMemoryExec};
use arrow::compute::SortOptions;
- use arrow::datatypes::DataType;
+ use arrow::datatypes::{DataType, Field};
use datafusion_common::SplitPoint;
use datafusion_common::stats::Precision;
use datafusion_common::{ColumnStatistics, ScalarValue};
@@ -1725,6 +1723,84 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn test_union_schema_metadata_and_nullability() -> Result<()> {
+ use arrow::array::Int32Array;
+
+ // Field metadata from the selected field is merged first, followed by
+ // the other inputs in order. Schema metadata always follows input order.
+ for (nullability, expected_field_value) in [
+ ([false, false, false], "input2"),
+ ([true, false, false], "input2"),
+ ([false, true, false], "input2"),
+ ([false, false, true], "input1"),
+ ([true, true, true], "input2"),
+ ] {
+ let inputs = nullability
+ .iter()
+ .enumerate()
+ .map(|(i, &nullable)| {
+ let metadata = Metadata::from([
+ ("shared".to_owned(), format!("input{i}")),
+ (format!("only{i}"), "kept".to_owned()),
+ ]);
+ let field =
+ Field::new(format!("value{i}"), DataType::Int32, nullable)
+ .with_metadata(metadata.clone());
+ let schema =
+ Arc::new(Schema::new_with_metadata(vec![field], metadata));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![i as i32]))],
+ )?;
+ let input: Arc<dyn ExecutionPlan> =
+ TestMemoryExec::try_new_exec(&[vec![batch]], schema, None)?;
+ Ok(input)
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ let expected_metadata = Metadata::from([
+ ("shared".to_owned(), expected_field_value.to_owned()),
+ ("only0".to_owned(), "kept".to_owned()),
+ ("only1".to_owned(), "kept".to_owned()),
+ ("only2".to_owned(), "kept".to_owned()),
+ ]);
+ let mut expected_schema_metadata = expected_metadata.clone();
+ expected_schema_metadata.insert("shared".to_owned(), "input2".to_owned());
+ let expected_schema = Arc::new(Schema::new_with_metadata(
+ vec![
+ Field::new("value0", DataType::Int32, nullability.contains(&true))
+ .with_metadata(expected_metadata),
+ ],
+ expected_schema_metadata,
+ ));
+
+ let union = UnionExec::try_new(inputs)?;
+ assert_eq!(union.schema(), expected_schema, "{nullability:?}");
+ let batches = collect(union, Arc::new(TaskContext::default())).await?;
+ datafusion_common::assert_batches_sorted_eq!(
+ [
+ "+--------+",
+ "| value0 |",
+ "+--------+",
+ "| 0 |",
+ "| 1 |",
+ "| 2 |",
+ "+--------+"
+ ],
+ &batches
+ );
+ for batch in batches {
+ assert_eq!(
+ batch.schema().fields(),
+ expected_schema.fields(),
+ "{nullability:?}"
+ );
+ }
+ }
+ Ok(())
+ }
+
#[test]
fn test_union_schema_mismatch() {
// Test that UnionExec properly rejects inputs with different field counts