Skip to content

Apache DataFusion SQL Query Engine

perf: balance file groups by size for scans with an output ordering (#26458)

Jay Zhana5ca29189d1f
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index 283d22767f..295bace596 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -4760,7 +4760,7 @@ fn parallelization_multiple_files() -> Result<()> {
@r"
SortRequiredExec: [a@0 ASC]
FilterExec: c@2 = 0
- DataSourceExec: file_groups={8 groups: [[x:0..25], [y:0..25], [x:25..50], [y:25..50], [x:50..75], [y:50..75], [x:75..100], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ DataSourceExec: file_groups={8 groups: [[x:0..25], [x:25..50], [x:50..75], [x:75..100], [y:0..25], [y:25..50], [y:50..75], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
");
let plan_8_sort = test_config_concurrency_8.to_plan(plan, &SORT_DISTRIB_DISTRIB);
assert_plan!(plan_8_distrib, plan_8_sort);
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 64b12951ef..c94979a472 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -872,6 +872,23 @@ impl DataSource for FileScanConfig {
return Ok(None);
}
+ // An ordered scan can use the size-balanced split of an unordered
+ // scan when every new group is still sorted. That split keeps the
+ // files in group order and only moves the group boundaries, so groups
+ // like `[f1, f2], [f3, f4]` with a large `f1` become
+ // `[f1 head], [f1 tail, f2], ...`.
+ if output_ordering.is_some()
+ && let Some(balanced) = self.file_source.repartitioned(
+ target_partitions,
+ repartition_file_min_size,
+ None,
+ self,
+ )?
+ && balanced.validated_output_ordering() == self.validated_output_ordering()
+ {
+ return Ok(Some(Arc::new(balanced)));
+ }
+
let source = self.file_source.repartitioned(
target_partitions,
repartition_file_min_size,
@@ -3361,6 +3378,176 @@ mod tests {
))
}
+ fn ordered_scan_with_groups(file_groups: Vec<FileGroup>) -> FileScanConfig {
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
+ let sort_expr = PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)));
+ FileScanConfigBuilder::new(
+ ObjectStoreUrl::local_filesystem(),
+ Arc::new(MockSource::new(schema)),
+ )
+ .with_file_groups(file_groups)
+ .with_output_ordering(vec![LexOrdering::new(vec![sort_expr]).unwrap()])
+ .build()
+ }
+
+ fn sized_file_with_stats(
+ name: &str,
+ size: u64,
+ min: f64,
+ max: f64,
+ ) -> PartitionedFile {
+ let mut file = make_file_with_stats(name, min, max);
+ file.object_meta.size = size;
+ file
+ }
+
+ /// The files of each group, with whether each file reads only a range
+ fn group_files(config: &FileScanConfig) -> Vec<Vec<(&str, bool)>> {
+ config
+ .file_groups
+ .iter()
+ .map(|group| {
+ group
+ .iter()
+ .map(|f| (f.object_meta.location.as_ref(), f.range.is_some()))
+ .collect()
+ })
+ .collect()
+ }
+
+ fn group_sizes(config: &FileScanConfig) -> Vec<u64> {
+ config
+ .file_groups
+ .iter()
+ .map(|group| group.iter().map(|f| f.effective_size()).sum())
+ .collect()
+ }
+
+ /// An ordered scan whose files are sorted across groups gets the same
+ /// size-balanced split as an unordered scan and keeps its ordering.
+ #[test]
+ fn repartition_balances_ordered_groups() -> Result<()> {
+ let config = ordered_scan_with_groups(vec![
+ FileGroup::new(vec![
+ sized_file_with_stats("f1", 8000, 0.0, 9.0),
+ sized_file_with_stats("f2", 8000, 10.0, 19.0),
+ ]),
+ FileGroup::new(vec![
+ sized_file_with_stats("f3", 1000, 20.0, 29.0),
+ sized_file_with_stats("f4", 1000, 30.0, 39.0),
+ ]),
+ ]);
+ let ordering = config.output_ordering.first().cloned();
+ assert_eq!(group_sizes(&config), vec![16000, 2000]);
+
+ let repartitioned = DataSource::repartitioned(&config, 4, 0, ordering)?
+ .expect("ordered groups should be rebalanced");
+ let repartitioned = repartitioned
+ .downcast_ref::<FileScanConfig>()
+ .expect("expected FileScanConfig");
+ assert_eq!(group_sizes(repartitioned), vec![4500, 4500, 4500, 4500]);
+ assert_eq!(
+ repartitioned.validated_output_ordering(),
+ config.validated_output_ordering()
+ );
+ Ok(())
+ }
+
+ /// If moving the group boundaries would put overlapping files in one
+ /// group, the scan keeps its groups and its ordering.
+ #[test]
+ fn repartition_keeps_ordered_groups_when_rebalancing_breaks_order() -> Result<()> {
+ let config = ordered_scan_with_groups(vec![
+ FileGroup::new(vec![
+ sized_file_with_stats("f1", 8000, 0.0, 9.0),
+ sized_file_with_stats("f2", 8000, 10.0, 19.0),
+ ]),
+ FileGroup::new(vec![
+ sized_file_with_stats("f3", 1000, 5.0, 15.0),
+ sized_file_with_stats("f4", 1000, 16.0, 25.0),
+ ]),
+ ]);
+ let ordering = config.output_ordering.first().cloned();
+ assert_eq!(config.validated_output_ordering().len(), 1);
+
+ assert!(DataSource::repartitioned(&config, 4, 0, ordering)?.is_none());
+ Ok(())
+ }
+
+ /// With `NULLS FIRST`, a file with NULLs can only be the first file of a
+ /// group, so a size-balanced split that puts it after another file is not
+ /// used.
+ #[test]
+ fn repartition_keeps_ordered_groups_when_rebalancing_misplaces_nulls() -> Result<()> {
+ let mut with_nulls = sized_file_with_stats("f2", 1000, 10.0, 19.0);
+ Arc::make_mut(with_nulls.statistics.as_mut().unwrap()).column_statistics[0]
+ .null_count = Precision::Exact(1);
+ let config = ordered_scan_with_groups(vec![
+ FileGroup::new(vec![sized_file_with_stats("f1", 8000, 0.0, 9.0)]),
+ FileGroup::new(vec![with_nulls]),
+ ]);
+ let ordering = config.output_ordering.first().cloned();
+ assert_eq!(config.validated_output_ordering().len(), 1);
+
+ let repartitioned = DataSource::repartitioned(&config, 4, 0, ordering)?
+ .expect("the large file should be split");
+ let repartitioned = repartitioned
+ .downcast_ref::<FileScanConfig>()
+ .expect("expected FileScanConfig");
+ assert_eq!(
+ group_files(repartitioned),
+ vec![
+ vec![("f1", true)],
+ vec![("f2", true)],
+ vec![("f1", true)],
+ vec![("f1", true)],
+ ]
+ );
+ assert_eq!(
+ repartitioned.validated_output_ordering(),
+ config.validated_output_ordering()
+ );
+ Ok(())
+ }
+
+ /// If moving the group boundaries would put overlapping files in one
+ /// group, the ordered split is used instead: it keeps multi-file groups
+ /// and only splits single-file groups.
+ #[test]
+ fn repartition_falls_back_to_ordered_split_when_rebalancing_breaks_order()
+ -> Result<()> {
+ let config = ordered_scan_with_groups(vec![
+ FileGroup::new(vec![
+ sized_file_with_stats("f1", 8000, 0.0, 9.0),
+ sized_file_with_stats("f2", 8000, 10.0, 19.0),
+ ]),
+ FileGroup::new(vec![sized_file_with_stats("f3", 1000, 5.0, 15.0)]),
+ ]);
+ let ordering = config.output_ordering.first().cloned();
+ assert_eq!(config.validated_output_ordering().len(), 1);
+
+ let repartitioned = DataSource::repartitioned(&config, 4, 0, ordering)?
+ .expect("the single-file group should be split");
+ let repartitioned = repartitioned
+ .downcast_ref::<FileScanConfig>()
+ .expect("expected FileScanConfig");
+ assert_eq!(
+ group_files(repartitioned),
+ vec![
+ vec![("f1", false), ("f2", false)],
+ vec![("f3", true)],
+ vec![("f3", true)],
+ vec![("f3", true)],
+ ]
+ );
+ assert_eq!(
+ repartitioned.validated_output_ordering(),
+ config.validated_output_ordering()
+ );
+ Ok(())
+ }
+
fn make_exact_empty_file(name: &str) -> PartitionedFile {
PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(
Statistics {
diff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
index a877b88d5e..6da3689988 100644
--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
@@ -340,6 +340,221 @@ WHERE partition_col = 'D'
GROUP BY int_col;
----
+# Min/max statistics ignore NULLs. Files read one after another in a single
+# partition keep the declared ordering only if their NULLs end up at the start
+# (NULLS FIRST) or at the end (NULLS LAST) of the combined stream.
+statement ok
+set datafusion.execution.target_partitions = 1;
+
+# NULLS LAST with NULLs in the first file: needs a sort
+query I
+COPY (SELECT * FROM (VALUES (1), (NULL)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_bad/1.parquet'
+STORED AS PARQUET;
+----
+2
+
+query I
+COPY (SELECT * FROM (VALUES (2), (3)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_bad/2.parquet'
+STORED AS PARQUET;
+----
+2
+
+statement ok
+CREATE EXTERNAL TABLE nulls_last_bad (a BIGINT)
+STORED AS PARQUET
+WITH ORDER (a ASC NULLS LAST)
+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_last_bad';
+
+query TT
+EXPLAIN SELECT a FROM nulls_last_bad ORDER BY a ASC NULLS LAST;
+----
+logical_plan
+01)Sort: nulls_last_bad.a ASC NULLS LAST
+02)--TableScan: nulls_last_bad projection=[a]
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_bad/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_bad/2.parquet]]}, projection=[a], file_type=parquet, sort_order_for_reorder=[a@0 ASC NULLS LAST]
+
+query I
+SELECT a FROM nulls_last_bad ORDER BY a ASC NULLS LAST;
+----
+1
+2
+3
+NULL
+
+# NULLS LAST with NULLs only in the last file: the ordering is kept
+query I
+COPY (SELECT * FROM (VALUES (1), (2)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_ok/1.parquet'
+STORED AS PARQUET;
+----
+2
+
+query I
+COPY (SELECT * FROM (VALUES (3), (NULL)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_last_ok/2.parquet'
+STORED AS PARQUET;
+----
+2
+
+statement ok
+CREATE EXTERNAL TABLE nulls_last_ok (a BIGINT)
+STORED AS PARQUET
+WITH ORDER (a ASC NULLS LAST)
+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_last_ok';
+
+query TT
+EXPLAIN SELECT a FROM nulls_last_ok ORDER BY a ASC NULLS LAST;
+----
+logical_plan
+01)Sort: nulls_last_ok.a ASC NULLS LAST
+02)--TableScan: nulls_last_ok projection=[a]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_ok/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_last_ok/2.parquet]]}, projection=[a], output_ordering=[a@0 ASC NULLS LAST], file_type=parquet
+
+query I
+SELECT a FROM nulls_last_ok ORDER BY a ASC NULLS LAST;
+----
+1
+2
+3
+NULL
+
+# NULLS FIRST with NULLs in the last file: needs a sort
+query I
+COPY (SELECT * FROM (VALUES (1), (2)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_bad/1.parquet'
+STORED AS PARQUET;
+----
+2
+
+query I
+COPY (SELECT * FROM (VALUES (NULL), (3)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_bad/2.parquet'
+STORED AS PARQUET;
+----
+2
+
+statement ok
+CREATE EXTERNAL TABLE nulls_first_bad (a BIGINT)
+STORED AS PARQUET
+WITH ORDER (a ASC NULLS FIRST)
+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_first_bad';
+
+query TT
+EXPLAIN SELECT a FROM nulls_first_bad ORDER BY a ASC NULLS FIRST;
+----
+logical_plan
+01)Sort: nulls_first_bad.a ASC NULLS FIRST
+02)--TableScan: nulls_first_bad projection=[a]
+physical_plan
+01)SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_bad/2.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_bad/1.parquet]]}, projection=[a], file_type=parquet, sort_order_for_reorder=[a@0 ASC]
+
+query I
+SELECT a FROM nulls_first_bad ORDER BY a ASC NULLS FIRST;
+----
+NULL
+1
+2
+3
+
+# NULLS FIRST with NULLs only in the first file: the ordering is kept
+query I
+COPY (SELECT * FROM (VALUES (NULL), (1)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_ok/1.parquet'
+STORED AS PARQUET;
+----
+2
+
+query I
+COPY (SELECT * FROM (VALUES (2), (3)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/nulls_first_ok/2.parquet'
+STORED AS PARQUET;
+----
+2
+
+statement ok
+CREATE EXTERNAL TABLE nulls_first_ok (a BIGINT)
+STORED AS PARQUET
+WITH ORDER (a ASC NULLS FIRST)
+LOCATION 'test_files/scratch/parquet_sorted_statistics/nulls_first_ok';
+
+query TT
+EXPLAIN SELECT a FROM nulls_first_ok ORDER BY a ASC NULLS FIRST;
+----
+logical_plan
+01)Sort: nulls_first_ok.a ASC NULLS FIRST
+02)--TableScan: nulls_first_ok projection=[a]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_ok/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/nulls_first_ok/2.parquet]]}, projection=[a], output_ordering=[a@0 ASC], file_type=parquet
+
+query I
+SELECT a FROM nulls_first_ok ORDER BY a ASC NULLS FIRST;
+----
+NULL
+1
+2
+3
+
+# Declaring INT over files that store BIGINT leaves the file statistics
+# without usable min/max values or null counts. The files cannot be proven
+# sorted.
+query I
+COPY (SELECT * FROM (VALUES (100), (200)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/cast_bounds/1.parquet'
+STORED AS PARQUET;
+----
+2
+
+query I
+COPY (SELECT * FROM (VALUES (1), (2), (3)) AS t(a))
+TO 'test_files/scratch/parquet_sorted_statistics/cast_bounds/2.parquet'
+STORED AS PARQUET;
+----
+3
+
+statement ok
+CREATE EXTERNAL TABLE cast_bounds (a INT)
+STORED AS PARQUET
+WITH ORDER (a ASC)
+LOCATION 'test_files/scratch/parquet_sorted_statistics/cast_bounds';
+
+query TT
+EXPLAIN SELECT a FROM cast_bounds ORDER BY a ASC;
+----
+logical_plan
+01)Sort: cast_bounds.a ASC NULLS LAST
+02)--TableScan: cast_bounds projection=[a]
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/cast_bounds/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/cast_bounds/2.parquet]]}, projection=[a], file_type=parquet, sort_order_for_reorder=[a@0 ASC NULLS LAST]
+
+query I
+SELECT a FROM cast_bounds ORDER BY a ASC;
+----
+1
+2
+3
+100
+200
+
+statement ok
+DROP TABLE cast_bounds;
+
+statement ok
+DROP TABLE nulls_last_bad;
+
+statement ok
+DROP TABLE nulls_last_ok;
+
+statement ok
+DROP TABLE nulls_first_bad;
+
+statement ok
+DROP TABLE nulls_first_ok;
+
# Config reset
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
diff --git a/datafusion/sqllogictest/test_files/repartition_scan.slt b/datafusion/sqllogictest/test_files/repartition_scan.slt
index 150d5b1b43..edb0ce3404 100644
--- a/datafusion/sqllogictest/test_files/repartition_scan.slt
+++ b/datafusion/sqllogictest/test_files/repartition_scan.slt
@@ -147,6 +147,53 @@ DROP TABLE parquet_table;
statement ok
DROP TABLE parquet_table_with_order;
+## An ordered scan gets size-balanced groups when statistics prove that
+## every new group is still sorted
+
+statement ok
+COPY (VALUES (1), (2), (3), (4), (5))
+TO 'test_files/scratch/repartition_scan/ordered_files/1.parquet'
+STORED AS PARQUET;
+
+statement ok
+COPY (VALUES (100), (200))
+TO 'test_files/scratch/repartition_scan/ordered_files/2.parquet'
+STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE ordered_files(column1 bigint)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/repartition_scan/ordered_files'
+WITH ORDER (column1 ASC);
+
+query I
+SELECT column1 FROM ordered_files WHERE column1 <> 42 ORDER BY column1;
+----
+1
+2
+3
+4
+5
+100
+200
+
+# A group may span files when reading them in order keeps the declared
+# ordering: all of 1.parquet's values are below 2.parquet's
+query TT
+EXPLAIN SELECT column1 FROM ordered_files WHERE column1 <> 42 ORDER BY column1;
+----
+logical_plan
+01)Sort: ordered_files.column1 ASC NULLS LAST
+02)--Filter: ordered_files.column1 != Int64(42)
+03)----TableScan: ordered_files projection=[column1], partial_filters=[ordered_files.column1 != Int64(42)]
+physical_plan
+01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST]
+02)--FilterExec: column1@0 != 42
+03)----DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/1.parquet:0..258], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/1.parquet:258..516], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/1.parquet:516..521, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/2.parquet:0..253], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/ordered_files/2.parquet:253..510]]}, projection=[column1], output_ordering=[column1@0 ASC NULLS LAST], file_type=parquet, predicate=column1@0 != 42, pruning_predicate=column1_null_count@2 != row_count@3 AND (column1_min@0 != 42 OR 42 != column1_max@1), required_guarantees=[column1 not in (42)]
+
+statement ok
+DROP TABLE ordered_files;
+
## Either row group range assignment reads each row of a split file once
statement ok