Apache DataFusion SQL Query Engine
perf: balance file groups by size for scans with an output ordering (#26458)
Jay Zhana5ca29189d1f
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 283d22767f..295bace596 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -4760,7 +4760,7 @@ fn parallelization_multiple_files() -> Result<()> {@r"SortRequiredExec: [a@0 ASC]FilterExec: c@2 = 0- DataSourceExec: file_groups={8 groups: [[x:0..25], [y:0..25], [x:25..50], [y:25..50], [x:50..75], [y:50..75], [x:75..100], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ DataSourceExec: file_groups={8 groups: [[x:0..25], [x:25..50], [x:50..75], [x:75..100], [y:0..25], [y:25..50], [y:50..75], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet");let plan_8_sort = test_config_concurrency_8.to_plan(plan, &SORT_DISTRIB_DISTRIB);assert_plan!(plan_8_distrib, plan_8_sort);diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 64b12951ef..c94979a472 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -872,6 +872,23 @@ impl DataSource for FileScanConfig {return Ok(None);}+ // An ordered scan can use the size-balanced split of an unordered+ // scan when every new group is still sorted. That split keeps the+ // files in group order and only moves the group boundaries, so groups+ // like `[f1, f2], [f3, f4]` with a large `f1` become+ // `[f1 head], [f1 tail, f2], ...`.+ if output_ordering.is_some()+ && let Some(balanced) = self.file_source.repartitioned(+ target_partitions,+ repartition_file_min_size,+ None,+ self,+ )?+ && balanced.validated_output_ordering() == self.validated_output_ordering()+ {+ return Ok(Some(Arc::new(balanced)));+ }+let source = self.file_source.repartitioned(target_partitions,repartition_file_min_size,@@ -3361,6 +3378,176 @@ mod tests {))}+ fn ordered_scan_with_groups(file_groups: Vec<FileGroup>) -> FileScanConfig {+ let schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));+ let sort_expr = PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)));+ FileScanConfigBuilder::new(+ ObjectStoreUrl::local_filesystem(),+ Arc::new(MockSource::new(schema)),+ )+ .with_file_groups(file_groups)+ .with_output_ordering(vec![LexOrdering::new(vec![sort_expr]).unwrap()])+ .build()+ }++ fn sized_file_with_stats(+ name: &str,+ size: u64,+ min: f64,+ max: f64,+ ) -> PartitionedFile {+ let mut file = make_file_with_stats(name, min, max);+ file.object_meta.size = size;+ file+ }++ /// The files of each group, with whether each file reads only a range+ fn group_files(config: &FileScanConfig) -> Vec<Vec<(&str, bool)>> {+ config+ .file_groups+ .iter()+ .map(|group| {+ group+ .iter()+ .map(|f| (f.object_meta.location.as_ref(), f.range.is_some()))+ .collect()+ })+ .collect()+ }++ fn group_sizes(config: &FileScanConfig) -> Vec<u64> {+ config+ .file_groups+ .iter()+ .map(|group| group.iter().map(|f| f.effective_size()).sum())+ .collect()+ }++ /// An ordered scan whose files are sorted across groups gets the same+ /// size-balanced split as an unordered scan and keeps its ordering.+ #[test]+ fn repartition_balances_ordered_groups() -> Result<()> {+ let config = ordered_scan_with_groups(vec![+ FileGroup::new(vec![+ sized_file_with_stats("f1", 8000, 0.0, 9.0),+ sized_file_with_stats("f2", 8000, 10.0, 19.0),+ ]),+ FileGroup::new(vec![+ sized_file_with_stats("f3", 1000, 20.0, 29.0),+ sized_file_with_stats("f4", 1000, 30.0, 39.0),+ ]),+ ]);+ let ordering = config.output_ordering.first().cloned();+ assert_eq!(group_sizes(&config), vec![16000, 2000]);++ let repartitioned = DataSource::repartitioned(&config, 4, 0, ordering)?+ .expect("ordered groups should be rebalanced");+ let repartitioned = repartitioned+ .downcast_ref::<FileScanConfig>()+ .expect("expected FileScanConfig");+ assert_eq!(group_sizes(repartitioned), vec![4500, 4500, 4500, 4500]);+ assert_eq!(+ repartitioned.validated_output_ordering(),+ config.validated_output_ordering()+ );+ Ok(())+ }++ /// If moving the group boundaries would put overlapping files in one+ /// group, the scan keeps its groups and its ordering.+ #[test]+ fn repartition_keeps_ordered_groups_when_rebalancing_breaks_order() -> Result<()> {+ let config = ordered_scan_with_groups(vec![+ FileGroup::new(vec![+ sized_file_with_stats("f1", 8000, 0.0, 9.0),+ sized_file_with_stats("f2", 8000, 10.0, 19.0),+ ]),+ FileGroup::new(vec![+ sized_file_with_stats("f3", 1000, 5.0, 15.0),+ sized_file_with_stats("f4", 1000, 16.0, 25.0),+ ]),+ ]);+ let ordering = config.output_ordering.first().cloned();+ assert_eq!(config.validated_output_ordering().len(), 1);++ assert!(DataSource::repartitioned(&config, 4, 0, ordering)?.is_none());+ Ok(())+ }++ /// With `NULLS FIRST`, a file with NULLs can only be the first file of a+ /// group, so a size-balanced split that puts it after another file is not+ /// used.+ #[test]+ fn repartition_keeps_ordered_groups_when_rebalancing_misplaces_nulls() -> Result<()> {+ let mut with_nulls = sized_file_with_stats("f2", 1000, 10.0, 19.0);+ Arc::make_mut(with_nulls.statistics.as_mut().unwrap()).column_statistics[0]+ .null_count = Precision::Exact(1);+ let config = ordered_scan_with_groups(vec![+ FileGroup::new(vec![sized_file_with_stats("f1", 8000, 0.0, 9.0)]),+ FileGroup::new(vec![with_nulls]),+ ]);+ let ordering = config.output_ordering.first().cloned();+ assert_eq!(config.validated_output_ordering().len(), 1);++ let repartitioned = DataSource::repartitioned(&config, 4, 0, ordering)?+ .expect("the large file should be split");+ let repartitioned = repartitioned+ .downcast_ref::<FileScanConfig>()+ .expect("expected FileScanConfig");+ assert_eq!(+ group_files(repartitioned),+ vec![+ vec![("f1", true)],+ vec![("f2", true)],+ vec![("f1", true)],+ vec![("f1", true)],+ ]+ );+ assert_eq!(+ repartitioned.validated_output_ordering(),+ config.validated_output_ordering()+ );+ Ok(())+ }++ /// If moving the group boundaries would put overlapping files in one+ /// group, the ordered split is used instead: it keeps multi-file groups+ /// and only splits single-file groups.+ #[test]+ fn repartition_falls_back_to_ordered_split_when_rebalancing_breaks_order()+ -> Result<()> {+ let config = ordered_scan_with_groups(vec![+ FileGroup::new(vec![+ sized_file_with_stats("f1", 8000, 0.0, 9.0),+ sized_file_with_stats("f2", 8000, 10.0, 19.0),+ ]),+ FileGroup::new(vec![sized_file_with_stats("f3", 1000, 5.0, 15.0)]),+ ]);+ let ordering = config.output_ordering.first().cloned();+ assert_eq!(config.validated_output_ordering().len(), 1);++ let repartitioned = DataSource::repartitioned(&config, 4, 0, ordering)?+ .expect("the single-file group should be split");+ let repartitioned = repartitioned+ .downcast_ref::<FileScanConfig>()+ .expect("expected FileScanConfig");+ assert_eq!(+ group_files(repartitioned),+ vec![+ vec![("f1", false), ("f2", false)],+ vec![("f3", true)],+ vec![("f3", true)],+ vec![("f3", true)],+ ]+ );+ assert_eq!(+ repartitioned.validated_output_ordering(),+ config.validated_output_ordering()+ );+ Ok(())+ }+fn make_exact_empty_file(name: &str) -> PartitionedFile {PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(Statistics {diff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.sltindex a877b88d5e..6da3689988 100644--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt@@ -340,6 +340,221 @@ WHERE partition_col = 'D'GROUP BY int_col;----+# Min/max statistics ignore NULLs. Files read one after another in a single+# partition keep the declared ordering only if their NULLs end up at the start+# (NULLS FIRST) or at the end (NULLS LAST) of the combined stream.+statement ok+set datafusion.execution.target_partitions = 1;++# NULLS LAST with NULLs in the first file: needs a sort+query I+COPY (SELECT * FROM (VALUES (1), (NULL)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_bad/1.parquet'+STORED AS PARQUET;+----+2++query I+COPY (SELECT * FROM (VALUES (2), (3)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_bad/2.parquet'+STORED AS PARQUET;+----+2++statement ok+CREATE EXTERNAL TABLE nulls_last_bad (a BIGINT)+STORED AS PARQUET+WITH ORDER (a ASC NULLS LAST)+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_last_bad';++query TT+EXPLAIN SELECT a FROM nulls_last_bad ORDER BY a ASC NULLS LAST;+----+logical_plan+01)Sort: nulls_last_bad.a ASC NULLS LAST+02)--TableScan: nulls_last_bad projection=[a]+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_bad/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_bad/2.parquet]]}, projection=[a], file_type=parquet, sort_order_for_reorder=[a@0 ASC NULLS LAST]++query I+SELECT a FROM nulls_last_bad ORDER BY a ASC NULLS LAST;+----+1+2+3+NULL++# NULLS LAST with NULLs only in the last file: the ordering is kept+query I+COPY (SELECT * FROM (VALUES (1), (2)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_ok/1.parquet'+STORED AS PARQUET;+----+2++query I+COPY (SELECT * FROM (VALUES (3), (NULL)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_ok/2.parquet'+STORED AS PARQUET;+----+2++statement ok+CREATE EXTERNAL TABLE nulls_last_ok (a BIGINT)+STORED AS PARQUET+WITH ORDER (a ASC NULLS LAST)+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_last_ok';++query TT+EXPLAIN SELECT a FROM nulls_last_ok ORDER BY a ASC NULLS LAST;+----+logical_plan+01)Sort: nulls_last_ok.a ASC NULLS LAST+02)--TableScan: nulls_last_ok projection=[a]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_ok/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_ok/2.parquet]]}, projection=[a], output_ordering=[a@0 ASC NULLS LAST], file_type=parquet++query I+SELECT a FROM nulls_last_ok ORDER BY a ASC NULLS LAST;+----+1+2+3+NULL++# NULLS FIRST with NULLs in the last file: needs a sort+query I+COPY (SELECT * FROM (VALUES (1), (2)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_bad/1.parquet'+STORED AS PARQUET;+----+2++query I+COPY (SELECT * FROM (VALUES (NULL), (3)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_bad/2.parquet'+STORED AS PARQUET;+----+2++statement ok+CREATE EXTERNAL TABLE nulls_first_bad (a BIGINT)+STORED AS PARQUET+WITH ORDER (a ASC NULLS FIRST)+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_first_bad';++query TT+EXPLAIN SELECT a FROM nulls_first_bad ORDER BY a ASC NULLS FIRST;+----+logical_plan+01)Sort: nulls_first_bad.a ASC NULLS FIRST+02)--TableScan: nulls_first_bad projection=[a]+physical_plan+01)SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_bad/2.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_bad/1.parquet]]}, projection=[a], file_type=parquet, sort_order_for_reorder=[a@0 ASC]++query I+SELECT a FROM nulls_first_bad ORDER BY a ASC NULLS FIRST;+----+NULL+1+2+3++# NULLS FIRST with NULLs only in the first file: the ordering is kept+query I+COPY (SELECT * FROM (VALUES (NULL), (1)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_ok/1.parquet'+STORED AS PARQUET;+----+2++query I+COPY (SELECT * FROM (VALUES (2), (3)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_ok/2.parquet'+STORED AS PARQUET;+----+2++statement ok+CREATE EXTERNAL TABLE nulls_first_ok (a BIGINT)+STORED AS PARQUET+WITH ORDER (a ASC NULLS FIRST)+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_first_ok';++query TT+EXPLAIN SELECT a FROM nulls_first_ok ORDER BY a ASC NULLS FIRST;+----+logical_plan+01)Sort: nulls_first_ok.a ASC NULLS FIRST+02)--TableScan: nulls_first_ok projection=[a]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_ok/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_ok/2.parquet]]}, projection=[a], output_ordering=[a@0 ASC], file_type=parquet++query I+SELECT a FROM nulls_first_ok ORDER BY a ASC NULLS FIRST;+----+NULL+1+2+3++# Declaring INT over files that store BIGINT leaves the file statistics+# without usable min/max values or null counts. The files cannot be proven+# sorted.+query I+COPY (SELECT * FROM (VALUES (100), (200)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/cast_bounds/1.parquet'+STORED AS PARQUET;+----+2++query I+COPY (SELECT * FROM (VALUES (1), (2), (3)) AS t(a))+TO 'test_files/scratch/parquet_sorted_statistics/cast_bounds/2.parquet'+STORED AS PARQUET;+----+3++statement ok+CREATE EXTERNAL TABLE cast_bounds (a INT)+STORED AS PARQUET+WITH ORDER (a ASC)+LOCATION 'test_files/scratch/parquet_sorted_statistics/cast_bounds';++query TT+EXPLAIN SELECT a FROM cast_bounds ORDER BY a ASC;+----+logical_plan+01)Sort: cast_bounds.a ASC NULLS LAST+02)--TableScan: cast_bounds projection=[a]+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/cast_bounds/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/cast_bounds/2.parquet]]}, projection=[a], file_type=parquet, sort_order_for_reorder=[a@0 ASC NULLS LAST]++query I+SELECT a FROM cast_bounds ORDER BY a ASC;+----+1+2+3+100+200++statement ok+DROP TABLE cast_bounds;++statement ok+DROP TABLE nulls_last_bad;++statement ok+DROP TABLE nulls_last_ok;++statement ok+DROP TABLE nulls_first_bad;++statement ok+DROP TABLE nulls_first_ok;+# Config reset# The SLT runner sets `target_partitions` to 4 instead of using the default, sodiff --git a/datafusion/sqllogictest/test_files/repartition_scan.slt b/datafusion/sqllogictest/test_files/repartition_scan.sltindex 150d5b1b43..edb0ce3404 100644--- a/datafusion/sqllogictest/test_files/repartition_scan.slt+++ b/datafusion/sqllogictest/test_files/repartition_scan.slt@@ -147,6 +147,53 @@ DROP TABLE parquet_table;statement okDROP TABLE parquet_table_with_order;+## An ordered scan gets size-balanced groups when statistics prove that+## every new group is still sorted++statement ok+COPY (VALUES (1), (2), (3), (4), (5))+TO 'test_files/scratch/repartition_scan/ordered_files/1.parquet'+STORED AS PARQUET;++statement ok+COPY (VALUES (100), (200))+TO 'test_files/scratch/repartition_scan/ordered_files/2.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE ordered_files(column1 bigint)+STORED AS PARQUET+LOCATION 'test_files/scratch/repartition_scan/ordered_files'+WITH ORDER (column1 ASC);++query I+SELECT column1 FROM ordered_files WHERE column1 <> 42 ORDER BY column1;+----+1+2+3+4+5+100+200++# A group may span files when reading them in order keeps the declared+# ordering: all of 1.parquet's values are below 2.parquet's+query TT+EXPLAIN SELECT column1 FROM ordered_files WHERE column1 <> 42 ORDER BY column1;+----+logical_plan+01)Sort: ordered_files.column1 ASC NULLS LAST+02)--Filter: ordered_files.column1 != Int64(42)+03)----TableScan: ordered_files projection=[column1], partial_filters=[ordered_files.column1 != Int64(42)]+physical_plan+01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST]+02)--FilterExec: column1@0 != 42+03)----DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/1.parquet:0..258], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/1.parquet:258..516], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/1.parquet:516..521, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/2.parquet:0..253], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/2.parquet:253..510]]}, projection=[column1], output_ordering=[column1@0 ASC NULLS LAST], file_type=parquet, predicate=column1@0 != 42, pruning_predicate=column1_null_count@2 != row_count@3 AND (column1_min@0 != 42 OR 42 != column1_max@1), required_guarantees=[column1 not in (42)]++statement ok+DROP TABLE ordered_files;+## Either row group range assignment reads each row of a split file oncestatement ok