Apache DataFusion SQL Query Engine
fix: declare Hive partitioned scans as Range (#25279)
George Stamatakis5026ac452492
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex a72d2c2bc7..c94ddcef22 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -25,13 +25,13 @@ use async_trait::async_trait;use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};use datafusion_common::stats::{Precision, is_known_empty};use datafusion_common::{- Constraints, DFSchema, SchemaExt, Statistics, internal_datafusion_err, plan_err,- project_schema,+ Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err,+ plan_err, project_schema,};use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::{- FileScanConfig, FileScanConfigBuilder, output_partitioning_from_partition_fields,+ FileScanConfig, FileScanConfigBuilder, output_range_partitioning_from_split_points,};use datafusion_datasource::file_sink_config::{FileOutputMode, FileSinkConfig};#[expect(deprecated)]@@ -69,6 +69,8 @@ pub struct ListFilesResult {pub statistics: Statistics,/// Whether files are grouped by partition values.pub grouped_by_partition: bool,+ /// Boundaries between file groups, empty unless `grouped_by_partition` is true.+ pub partition_split_points: Vec<SplitPoint>,}/// Built in [`TableProvider`] that reads data from one or more files as a single table.@@ -668,6 +670,7 @@ impl ListingTable {file_groups: mut partitioned_file_lists,statistics,grouped_by_partition: partitioned_by_file_group,+ partition_split_points,} = self.list_files_for_scan(state, &partition_filters, statistic_file_limit).await?;@@ -687,6 +690,7 @@ impl ListingTable {.config_options().execution.split_file_groups_by_statistics;+ let mut regrouped_by_statistics = false;match split_file_groups_by_statistics.then(|| {output_ordering.first().map(|output_ordering| {@@ -704,6 +708,7 @@ impl ListingTable {Some(Ok(new_groups)) => {if new_groups.len() <= file_group_count {partitioned_file_lists = new_groups;+ regrouped_by_statistics = true;} else {log::debug!("attempted to split file groups by statistics, but there were more file groups than target_partitions; falling back to unordered"@@ -737,6 +742,17 @@ impl ListingTable {)?}};+ Some(output_partitioning)+ } else if partitioned_by_file_group && !regrouped_by_statistics {+ output_range_partitioning_from_split_points(+ &self.table_schema,+ &table_partition_cols.into(),+ partition_split_points,+ )?+ } else {+ None+ };+ if let Some(output_partitioning) = &output_partitioning {let partition_count = output_partitioning.partition_count();if partitioned_file_lists.len() != partition_count {return plan_err!(@@ -744,19 +760,7 @@ impl ListingTable {partitioned_file_lists.len());}- Some(output_partitioning)- } else if partitioned_by_file_group {- // Files are grouped by partition column values: declare output- // partitioning on those columns so the optimizer can skip- // repartitioning for aggregates and joins on the partition columns.- output_partitioning_from_partition_fields(- &self.table_schema,- &table_partition_cols.clone().into(),- partitioned_file_lists.len(),- )- } else {- None- };+ }let Some(object_store_url) =self.table_paths.first().map(ListingTableUrl::object_store)@@ -1005,6 +1009,7 @@ impl ListingTable {file_groups: vec![],statistics: Statistics::new_unknown(&self.file_schema),grouped_by_partition: false,+ partition_split_points: vec![],});};let (file_group, inexact_stats) = self@@ -1018,21 +1023,23 @@ impl ListingTable {// skip repartitioning for aggregates and joins on partition columns.let threshold = ctx.config_options().optimizer.preserve_file_partitions;- let (file_groups, grouped_by_partition) =+ let (file_groups, grouped_by_partition, partition_split_points) =if threshold > 0 && !self.options.table_partition_cols.is_empty() {- let grouped = file_group.group_by_partition_values(file_group_count);+ let (grouped, split_points) = file_group+ .group_by_partition_values_with_split_points(file_group_count);if grouped.len() >= threshold {- (grouped, true)+ (grouped, true, split_points)} else {let all_files: Vec<_> =grouped.into_iter().flat_map(|g| g.into_inner()).collect();(FileGroup::new(all_files).split_files(file_group_count),false,+ vec![],)}} else {- (file_group.split_files(file_group_count), false)+ (file_group.split_files(file_group_count), false, vec![])};self.list_files_result_from_groups(@@ -1040,6 +1047,7 @@ impl ListingTable {file_groups,inexact_stats,grouped_by_partition,+ partition_split_points,)}@@ -1067,6 +1075,7 @@ impl ListingTable {file_groups: vec![],statistics: Statistics::new_unknown(&self.file_schema),grouped_by_partition: false,+ partition_split_points: vec![],});};let (file_group, inexact_stats) =@@ -1078,7 +1087,7 @@ impl ListingTable {let file_groups =self.filter_declared_file_groups_by_partition_filters(file_groups, filters)?;- self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false)+ self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false, vec![])}fn filter_declared_file_groups_by_partition_filters(@@ -1113,6 +1122,7 @@ impl ListingTable {file_groups: Vec<FileGroup>,inexact_stats: bool,grouped_by_partition: bool,+ partition_split_points: Vec<SplitPoint>,) -> datafusion_common::Result<ListFilesResult> {let (file_groups, stats) = compute_all_files_statistics(file_groups,@@ -1129,6 +1139,7 @@ impl ListingTable {file_groups,statistics: stats,grouped_by_partition,+ partition_split_points,})}diff --git a/datafusion/datasource/src/file_groups.rs b/datafusion/datasource/src/file_groups.rsindex 84594be54b..73cf484846 100644--- a/datafusion/datasource/src/file_groups.rs+++ b/datafusion/datasource/src/file_groups.rs@@ -19,8 +19,8 @@use crate::{FileRange, PartitionedFile};use arrow::compute::SortOptions;-use datafusion_common::Statistics;use datafusion_common::utils::compare_rows;+use datafusion_common::{ScalarValue, SplitPoint, Statistics};use itertools::Itertools;use std::cmp::{Ordering, min};use std::collections::{BinaryHeap, HashMap};@@ -488,19 +488,33 @@ impl FileGroup {////// Note: May return fewer groups than `max_target_partitions` when the/// number of unique partition values is less than the target.- #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key+ ///+ /// Wrapper around [`Self::group_by_partition_values_with_split_points`] that discards+ /// the split points.pub fn group_by_partition_values(self,max_target_partitions: usize,) -> Vec<FileGroup> {+ self.group_by_partition_values_with_split_points(max_target_partitions)+ .0+ }++ /// Groups files by partition value and returns the [`SplitPoint`]s between groups.+ ///+ /// Values are sorted with [`SortOptions::default()`] and cut into at most+ /// `max_target_partitions` contiguous chunks so routing a file's `partition_values`+ /// through the split points yields the group it was placed in.+ #[allow(clippy::allow_attributes, clippy::mutable_key_type)]+ pub fn group_by_partition_values_with_split_points(+ self,+ max_target_partitions: usize,+ ) -> (Vec<FileGroup>, Vec<SplitPoint>) {if self.is_empty() || max_target_partitions == 0 {- return vec![];+ return (vec![], vec![]);}- let mut partition_groups: HashMap<- Vec<datafusion_common::ScalarValue>,- Vec<PartitionedFile>,- > = HashMap::new();+ let mut partition_groups: HashMap<Vec<ScalarValue>, Vec<PartitionedFile>> =+ HashMap::new();for file in self.files {partition_groups@@ -511,7 +525,9 @@ impl FileGroup {let num_unique_partitions = partition_groups.len();- // Sort for deterministic bucket assignment across query executions.+ // Sort for deterministic bucket assignment across query executions. Must match+ // the ordering the caller declares, otherwise the split points would not+ // describe the groups.let mut sorted_partitions: Vec<_> = partition_groups.into_iter().collect();let sort_options =vec![@@ -522,24 +538,31 @@ impl FileGroup {compare_rows(&a.0, &b.0, &sort_options).unwrap_or(Ordering::Equal)});- if num_unique_partitions <= max_target_partitions {- sorted_partitions- .into_iter()- .map(|(_, files)| FileGroup::new(files))- .collect()- } else {- // Merge into max_target_partitions buckets using round-robin.- // This maintains grouping by partition value as we are merging groups which already- // contain all values for a partition key.- let mut target_groups = vec![vec![]; max_target_partitions];-- for (idx, (_, files)) in sorted_partitions.into_iter().enumerate() {- let bucket = idx % max_target_partitions;- target_groups[bucket].extend(files);+ let bucket_count = min(num_unique_partitions, max_target_partitions);+ let mut groups = Vec::with_capacity(bucket_count);+ let mut split_points = Vec::with_capacity(bucket_count.saturating_sub(1));++ // Every chunk is non-empty because `bucket_count <= num_unique_partitions`, so+ // its first tuple is a well defined lower bound.+ let mut iter = sorted_partitions.into_iter();+ for bucket in 0..bucket_count {+ let start = bucket * num_unique_partitions / bucket_count;+ let end = (bucket + 1) * num_unique_partitions / bucket_count;+ let mut files = Vec::new();+ for offset in start..end {+ let (values, group_files) = iter+ .next()+ .expect("contiguous chunking never runs past the sorted partitions");+ if offset == start && bucket > 0 {+ split_points.push(SplitPoint::new(values));+ }+ files.extend(group_files);}-- target_groups.into_iter().map(FileGroup::new).collect()+ groups.push(FileGroup::new(files));}+ debug_assert_eq!(split_points.len() + 1, groups.len());++ (groups, split_points)}}@@ -632,7 +655,9 @@ impl DerefMut for CompareByRangeSize {#[cfg(test)]mod test {use super::*;- use datafusion_common::ScalarValue;+ use datafusion_physical_expr::RangePartitioning;+ use datafusion_physical_expr::expressions::Column;+ use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};/// Empty file won't get partitioned#[test]@@ -1276,8 +1301,8 @@ mod test {#[test]fn test_group_by_partition_values_more_groups_than_target() {- // Each file has a single partition value. The number of partition values > max_target_partitions, so- // they should be round-robin distributed into groups.+ // More values than `max_target_partitions`, so they are cut into contiguous+ // chunks whose sizes differ by at most one.let fg = FileGroup::new(vec![pfile_with_pv("a", "p1"),pfile_with_pv("b", "p2"),@@ -1287,8 +1312,153 @@ mod test {]);let groups = fg.group_by_partition_values(3);assert_eq!(groups.len(), 3);- assert_eq!(groups[0].len(), 2);+ assert_eq!(groups[0].len(), 1);assert_eq!(groups[1].len(), 2);- assert_eq!(groups[2].len(), 1);+ assert_eq!(groups[2].len(), 2);+ // Groups must be contiguous key intervals in sorted order.+ assert_eq!(group_pvs(&groups[0]), vec!["p1"]);+ assert_eq!(group_pvs(&groups[1]), vec!["p2", "p3"]);+ assert_eq!(group_pvs(&groups[2]), vec!["p4", "p5"]);+ }++ fn group_pvs(group: &FileGroup) -> Vec<String> {+ group+ .iter()+ .map(|f| f.partition_values[0].to_string())+ .collect()+ }++ /// Mirrors `RangeRouter` semantics: the partition index is the number of split points+ /// that are `<=` the key under the given sort options.+ fn range_route(key: &[ScalarValue], split_points: &[SplitPoint]) -> usize {+ let sort_options = vec![SortOptions::default(); key.len()];+ split_points+ .iter()+ .take_while(|sp| {+ compare_rows(sp.values(), key, &sort_options).unwrap()+ != Ordering::Greater+ })+ .count()+ }++ #[test]+ fn test_group_by_partition_values_split_points_describe_groups() {+ let fg = FileGroup::new(vec![+ pfile_with_pv("a", "p1"),+ pfile_with_pv("b", "p2"),+ pfile_with_pv("c", "p3"),+ pfile_with_pv("d", "p4"),+ pfile_with_pv("e", "p5"),+ pfile_with_pv("f", "p5"),+ ]);+ let (groups, split_points) = fg.group_by_partition_values_with_split_points(3);+ assert_eq!(groups.len(), 3);+ assert_eq!(split_points.len(), 2);+ assert_eq!(split_points[0].values(), &[ScalarValue::from("p2")]);+ assert_eq!(split_points[1].values(), &[ScalarValue::from("p4")]);++ // Every file routes (RangeRouter semantics) to the group it was placed in.+ for (idx, group) in groups.iter().enumerate() {+ for file in group.iter() {+ assert_eq!(range_route(&file.partition_values, &split_points), idx);+ }+ }+ }++ #[test]+ fn test_group_by_partition_values_split_points_fewer_values_than_target() {+ let fg = FileGroup::new(vec![+ pfile_with_pv("a", "p1"),+ pfile_with_pv("b", "p1"),+ pfile_with_pv("c", "p2"),+ ]);+ let (groups, split_points) = fg.group_by_partition_values_with_split_points(4);+ assert_eq!(groups.len(), 2);+ assert_eq!(split_points.len(), 1);+ assert_eq!(split_points[0].values(), &[ScalarValue::from("p2")]);+ for (idx, group) in groups.iter().enumerate() {+ for file in group.iter() {+ assert_eq!(range_route(&file.partition_values, &split_points), idx);+ }+ }+ }++ #[test]+ fn test_group_by_partition_values_split_points_single_target_partition() {+ // One partition holds every value, so there is no boundary to describe.+ let fg = FileGroup::new(vec![+ pfile_with_pv("a", "p1"),+ pfile_with_pv("b", "p2"),+ pfile_with_pv("c", "p3"),+ pfile_with_pv("d", "p3"),+ ]);+ let (groups, split_points) = fg.group_by_partition_values_with_split_points(1);+ assert_eq!(groups.len(), 1);+ assert_eq!(groups[0].len(), 4);+ assert!(split_points.is_empty());+ }++ fn pfile_with_pvs(path: &str, pvs: &[Option<&str>]) -> PartitionedFile {+ let mut file = pfile(path, 10);+ file.partition_values = pvs+ .iter()+ .map(|v| ScalarValue::Utf8(v.map(str::to_string)))+ .collect();+ file+ }++ /// Two partition columns with NULLs in either position.+ fn compound_key_file_group() -> FileGroup {+ FileGroup::new(vec![+ pfile_with_pvs("a", &[Some("2026"), Some("09")]),+ pfile_with_pvs("b", &[Some("2026"), Some("10")]),+ pfile_with_pvs("c", &[None, Some("09")]),+ pfile_with_pvs("d", &[Some("2025"), None]),+ pfile_with_pvs("e", &[Some("2025"), Some("12")]),+ ])+ }++ #[test]+ fn test_group_by_partition_values_split_points_null_and_compound_keys() {+ let fg = compound_key_file_group();+ let (groups, split_points) = fg.group_by_partition_values_with_split_points(2);+ assert_eq!(groups.len(), 2);+ assert_eq!(split_points.len(), 1);+ assert_eq!(+ split_points[0].values(),+ &[ScalarValue::from("2025"), ScalarValue::from("12")]+ );+ for (idx, group) in groups.iter().enumerate() {+ for file in group.iter() {+ assert_eq!(range_route(&file.partition_values, &split_points), idx);+ }+ }+ }++ #[test]+ fn test_group_by_partition_values_split_point_containing_null() {+ let fg = compound_key_file_group();+ let (groups, split_points) = fg.group_by_partition_values_with_split_points(3);+ assert_eq!(groups.len(), 3);+ assert_eq!(+ split_points[0].values(),+ &[ScalarValue::from("2025"), ScalarValue::Utf8(None)]+ );+ assert_eq!(+ split_points[1].values(),+ &[ScalarValue::from("2026"), ScalarValue::from("09")]+ );+ for (idx, group) in groups.iter().enumerate() {+ for file in group.iter() {+ assert_eq!(range_route(&file.partition_values, &split_points), idx);+ }+ }+ let ordering = LexOrdering::new(vec![+ PhysicalSortExpr::new_default(Arc::new(Column::new("year", 0))),+ PhysicalSortExpr::new_default(Arc::new(Column::new("month", 1))),+ ])+ .unwrap();+ let range = RangePartitioning::try_new(ordering, split_points).unwrap();+ assert_eq!(range.partition_count(), 3);}}diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 07144d8191..64b12951ef 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -39,8 +39,8 @@ use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{- Constraint, Constraints, Result, ScalarValue, Statistics, internal_datafusion_err,- internal_err,+ Constraint, Constraints, Result, ScalarValue, SplitPoint, Statistics,+ internal_datafusion_err, internal_err,};use datafusion_execution::{SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl,@@ -52,7 +52,9 @@ use datafusion_common::stats::{Precision, is_known_empty};use datafusion_physical_expr::expressions::{BinaryExpr, Column};use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};use datafusion_physical_expr::utils::reassign_expr_columns;-use datafusion_physical_expr::{EquivalenceProperties, Partitioning, split_conjunction};+use datafusion_physical_expr::{+ EquivalenceProperties, Partitioning, RangePartitioning, split_conjunction,+};use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile};use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};@@ -613,15 +615,52 @@ impl From<FileScanConfig> for FileScanConfigBuilder {}}-/// Builds output partitioning over `partition_cols` (resolved to their indices in-/// `schema`) with `partition_count` partitions. Returns `None` when there are no-/// partition columns. Callers use this to declare the output partitioning of a scan-/// whose file groups are organized by partition column values.+/// Builds `Partitioning::Hash` over `partition_cols` (resolved to their indices in+/// `schema`). Returns `None` when there are no partition columns.+#[deprecated(+ since = "56.0.0",+ note = "Use `output_range_partitioning_from_split_points` instead"+)]pub fn output_partitioning_from_partition_fields(schema: &Schema,partition_cols: &Fields,partition_count: usize,) -> Option<Partitioning> {+ let exprs = partition_column_exprs(schema, partition_cols)?;+ Some(Partitioning::Hash(exprs, partition_count))+}++/// Builds the [`Partitioning::Range`] for file groups from+/// [`FileGroup::group_by_partition_values_with_split_points`], each column sorted with+/// `SortOptions::default()` to match how the groups were cut. Returns `None` when there+/// are no partition columns or one is missing from `schema`, and errors when+/// `split_points` are not strictly ordered. Split point values must have the partition+/// column types in `schema`.+pub fn output_range_partitioning_from_split_points(+ schema: &Schema,+ partition_cols: &Fields,+ split_points: Vec<SplitPoint>,+) -> Result<Option<Partitioning>> {+ let Some(exprs) = partition_column_exprs(schema, partition_cols) else {+ return Ok(None);+ };+ let sort_exprs = exprs+ .into_iter()+ .map(PhysicalSortExpr::new_default)+ .collect::<Vec<_>>();+ let Some(ordering) = LexOrdering::new(sort_exprs) else {+ return Ok(None);+ };+ let range = RangePartitioning::try_new(ordering, split_points)?;+ Ok(Some(Partitioning::Range(range)))+}++/// Resolves `partition_cols` to `Column` expressions in `schema`, or `None` when there+/// are no partition columns or one is missing.+fn partition_column_exprs(+ schema: &Schema,+ partition_cols: &Fields,+) -> Option<Vec<Arc<dyn PhysicalExpr>>> {if partition_cols.is_empty() {return None;}@@ -635,8 +674,7 @@ pub fn output_partitioning_from_partition_fields(.position(|field| field.name() == name)?;exprs.push(Arc::new(Column::new(name, idx)));}-- Some(Partitioning::Hash(exprs, partition_count))+ Some(exprs)}fn project_output_partitioning(@@ -3150,6 +3188,14 @@ mod tests {let partitioning = config.output_partitioning();assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_)));++ let partitioning = output_range_partitioning_from_split_points(+ config.file_source.table_schema().table_schema(),+ config.table_partition_cols(),+ vec![],+ )+ .unwrap();+ assert!(partitioning.is_none());}#[test]@@ -3174,22 +3220,44 @@ mod tests {FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]),];- config.output_partitioning = output_partitioning_from_partition_fields(+ let dict = |v: &str| wrap_partition_value_in_dict(ScalarValue::from(v));+ config.output_partitioning = output_range_partitioning_from_split_points(config.file_source.table_schema().table_schema(),config.table_partition_cols(),- config.file_groups.len(),- );+ vec![+ SplitPoint::new(vec![dict("2026-09")]),+ SplitPoint::new(vec![dict("2026-10")]),+ ],+ )+ .unwrap();let partitioning = config.output_partitioning();match partitioning {- Partitioning::Hash(exprs, num_partitions) => {- assert_eq!(num_partitions, 3);- assert_eq!(exprs.len(), 1);- assert_eq!(exprs[0].downcast_ref::<Column>().unwrap().name(), "date");+ Partitioning::Range(range) => {+ assert_eq!(range.partition_count(), 3);+ assert_eq!(range.ordering().len(), 1);+ let sort_expr = &range.ordering()[0];+ assert_eq!(+ sort_expr.expr.downcast_ref::<Column>().unwrap().name(),+ "date"+ );+ assert_eq!(sort_expr.options, arrow::compute::SortOptions::default());+ assert_eq!(range.split_points().len(), 2);}- _ => panic!("Expected Hash partitioning"),+ other => panic!("Expected Range partitioning, got {other}"),}+ let err = output_range_partitioning_from_split_points(+ config.file_source.table_schema().table_schema(),+ config.table_partition_cols(),+ vec![+ SplitPoint::new(vec![dict("2026-10")]),+ SplitPoint::new(vec![dict("2026-09")]),+ ],+ )+ .unwrap_err();+ assert!(err.to_string().contains("strictly ordered"), "{err}");+// Test multiple partition columnslet multiple_partition_cols = vec![Field::new("year", wrap_partition_type_in_dict(DataType::Utf8), false),@@ -3206,24 +3274,25 @@ mod tests {FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),];- config.output_partitioning = output_partitioning_from_partition_fields(+ config.output_partitioning = output_range_partitioning_from_split_points(config.file_source.table_schema().table_schema(),config.table_partition_cols(),- config.file_groups.len(),- );+ vec![SplitPoint::new(vec![dict("2026"), dict("09")])],+ )+ .unwrap();let partitioning = config.output_partitioning();match partitioning {- Partitioning::Hash(exprs, num_partitions) => {- assert_eq!(num_partitions, 2);- assert_eq!(exprs.len(), 2);- let col_names: Vec<_> = exprs+ Partitioning::Range(range) => {+ assert_eq!(range.partition_count(), 2);+ let col_names: Vec<_> = range+ .ordering().iter()- .map(|e| e.downcast_ref::<Column>().unwrap().name())+ .map(|e| e.expr.downcast_ref::<Column>().unwrap().name()).collect();assert_eq!(col_names, vec!["year", "month"]);}- _ => panic!("Expected Hash partitioning"),+ other => panic!("Expected Range partitioning, got {other}"),}}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex eca97556d0..5309f4c6f5 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1217,13 +1217,9 @@ impl HashJoinExec {return false;}- // `preserve_file_partitions` can report Hive-style file groups as Hash- // partitioned even though their partition indexes do not follow the- // hash router used by partitioned dynamic filters. Reject Hash inputs- // because the metadata cannot distinguish those scans from a real hash- // repartition. Compatible Range inputs remain safe because matching- // ordering and split points align each build filter with its probe- // partition. Other unsupported layouts are rejected.+ // Under `preserve_file_partitions`, file groups can report Hash without following+ // the dynamic filter hash router and look like a real hash repartition, so reject+ // Hash. Range inputs with matching split points are safe.// Follow-up work: enable dynamic filtering for preserve_file_partitioned scans (issue #20195).// https://github.com/apache/datafusion/issues/20195if config.optimizer.preserve_file_partitions > 0diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rsindex a215a01e84..4082158367 100644--- a/datafusion/proto/tests/cases/plans/sources.rs+++ b/datafusion/proto/tests/cases/plans/sources.rs@@ -1144,3 +1144,84 @@ fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> {Ok(())}++#[test]+fn roundtrip_parquet_exec_range_output_partitioning_dict_split_points() -> Result<()> {+ let file_schema = Arc::new(Schema::new(vec![Field::new(+ "col",+ wrap_partition_type_in_dict(DataType::Utf8),+ false,+ )]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let output_partitioning = Partitioning::Range(RangePartitioning::try_new(+ LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(+ "col", 0,+ )))])+ .unwrap(),+ vec![SplitPoint::new(vec![wrap_partition_value_in_dict(+ ScalarValue::from("B"),+ )])],+ )?);+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-1.parquet".to_string(),+ 1024,+ )]),+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-2.parquet".to_string(),+ 1024,+ )]),+ ])+ .with_output_partitioning(Some(output_partitioning.clone()))+ .build();++ assert_eq!(+ roundtrip_file_scan_config(scan_config)?.output_partitioning,+ Some(output_partitioning)+ );++ Ok(())+}++#[test]+fn roundtrip_parquet_exec_range_output_partitioning_utf8view_split_points() -> Result<()>+{+ let file_schema = Arc::new(Schema::new(vec![Field::new(+ "col",+ DataType::Utf8View,+ false,+ )]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let output_partitioning = Partitioning::Range(RangePartitioning::try_new(+ LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(+ "col", 0,+ )))])+ .unwrap(),+ vec![SplitPoint::new(vec![ScalarValue::Utf8View(Some(+ "B".to_string(),+ ))])],+ )?);+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-1.parquet".to_string(),+ 1024,+ )]),+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-2.parquet".to_string(),+ 1024,+ )]),+ ])+ .with_output_partitioning(Some(output_partitioning.clone()))+ .build();++ assert_eq!(+ roundtrip_file_scan_config(scan_config)?.output_partitioning,+ Some(output_partitioning)+ );++ Ok(())+}diff --git a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt b/datafusion/sqllogictest/test_files/preserve_file_partitioning.sltindex f4aab94ea6..3fb4a3e17c 100644--- a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt+++ b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt@@ -125,7 +125,7 @@ STORED AS PARQUET;1# Create high-cardinality fact table (5 partitions > 3 target_partitions)-# For testing partition merging with consistent hashing+# For testing partition merging into contiguous value rangesquery ICOPY (SELECT column1 as timestamp, column2 as value FROM (VALUES(TIMESTAMP '2023-01-01T09:00:00', 100.0)@@ -258,7 +258,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), sum(fact_table.value)@2 as sum(fact_table.value)]02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), sum(fact_table.value)]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet# Verify results with optimization match results without optimizationquery TIR rowsort@@ -320,7 +320,7 @@ physical_plan01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), avg(fact_table_ordered.value)@2 as avg(fact_table_ordered.value)]03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full-04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet+04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquetquery TIRSELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey;@@ -418,7 +418,7 @@ physical_plan06)----------FilterExec: service@2 = log07)------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=108)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible+09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligiblequery TTTIR rowsortSELECT f.f_dkey, MAX(d.env), MAX(d.service), count(*), sum(f.value)@@ -493,7 +493,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[f_dkey@2 as f_dkey, timestamp@0 as timestamp, value@1 as value, row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn]02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquetquery TPRI rowsortSELECT f_dkey, timestamp, value,@@ -513,8 +513,7 @@ C 2023-01-01T09:00:20 310.2 3########### TEST 9: High-Cardinality Partitions (more partitions than target_partitions)-# Since num_partitions > target_partitions (5 > 3), files are merged using-# round-robin assignment to ensure exactly target_partitions groups are created.+# Sorted partition values are cut into contiguous chunks.########### First verify results without optimization@@ -537,7 +536,7 @@ statement okset datafusion.optimizer.preserve_file_partitions = 1;# Verify the plan uses SinglePartitioned mode with no RepartitionExec-# The 5 partitions are merged into 3 file groups using round-robin assignment+# 5 values merge into 3 contiguous groups: [A], [B, C], [D, E]query TTEXPLAIN SELECT f_dkey, count(*), sum(value) FROM high_cardinality_table GROUP BY f_dkey;----@@ -548,7 +547,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), sum(high_cardinality_table.value)@2 as sum(high_cardinality_table.value)]02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), sum(high_cardinality_table.value)]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=B/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=E/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=B/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=C/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=D/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/high_cardinality/f_dkey=E/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (D)], 3), file_type=parquet# Verify results with optimization match results without optimizationquery TIR rowsort@@ -658,9 +657,7 @@ C prod 2017.6########### TEST 12: Partitioned Join with Matching Partition Counts - With Optimization# Both tables have 3 partitions matching target_partitions=3-# No RepartitionExec needed for join - partitions already satisfy the requirement-# Dynamic filter pushdown is disabled in this mode because preserve_file_partitions-# reports Hash partitioning for Hive-style file groups, which are not hash-routed.+# Identical split points prove co-partitioning, so no RepartitionExec is needed.##########statement ok@@ -685,8 +682,8 @@ physical_plan02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=303)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)]04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0]-05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Hash([d_dkey@1], 3), file_type=parquet-06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Hash([f_dkey@1], 3), file_type=parquet+05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), file_type=parquet+06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligiblequery TTR rowsortSELECT f.f_dkey, d.env, sum(f.value)@@ -722,7 +719,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[f_dkey@0 as f_dkey, timestamp@1 as timestamp, count(Int64(1))@2 as count(*), avg(fact_table.value)@3 as avg(fact_table.value)]02)--AggregateExec: mode=SinglePartitioned, gby=[f_dkey@2 as f_dkey, timestamp@0 as timestamp], aggr=[count(Int64(1)), avg(fact_table.value)]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquetquery TPIR rowsortSELECT f_dkey, timestamp,@@ -752,6 +749,196 @@ C 2023-01-01T09:00:40 1 300C 2023-01-01T09:12:40 1 250C 2023-01-01T09:12:50 1 275.4+##########+# TEST 14: Partitioned join, same group count, different partition values+# fact has {A, B, C}, dimension has {A, C, D}. Split points differ, so the scans are not+# co-partitioned and one side must be range repartitioned.+##########++query I+COPY (SELECT 'dev' as env, 'log' as service)+TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=A/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT 'prod' as env, 'log' as service)+TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=C/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT 'prod' as env, 'trace' as service)+TO 'test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=D/data.parquet'+STORED AS PARQUET;+----+1++statement ok+CREATE EXTERNAL TABLE dimension_table_skewed (env STRING, service STRING)+STORED AS PARQUET+PARTITIONED BY (d_dkey STRING)+LOCATION 'test_files/scratch/preserve_file_partitioning/dimension_skewed/';++statement ok+set datafusion.execution.target_partitions = 3;++statement ok+set datafusion.optimizer.preserve_file_partitions = 1;++statement ok+set datafusion.optimizer.hash_join_single_partition_threshold = 0;++statement ok+set datafusion.optimizer.hash_join_single_partition_threshold_rows = 0;++# Exactly one join input gets a RepartitionExec(Range).+query TT+EXPLAIN SELECT f.f_dkey, d.env, sum(f.value)+FROM fact_table f+INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey+GROUP BY f.f_dkey, d.env;+----+logical_plan+01)Aggregate: groupBy=[[f.f_dkey, d.env]], aggr=[[sum(f.value)]]+02)--Projection: f.value, f.f_dkey, d.env+03)----Inner Join: f.f_dkey = d.d_dkey+04)------SubqueryAlias: f+05)--------TableScan: fact_table projection=[value, f_dkey]+06)------SubqueryAlias: d+07)--------TableScan: dimension_table_skewed projection=[env, d_dkey]+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)]+02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3+03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)]+04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0]+05)--------RepartitionExec: partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), input_partitions=3+06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=C/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_skewed/d_dkey=D/data.parquet]]}, projection=[env, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(C), (D)], 3), file_type=parquet+07)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible++# Only A and C match.+query TTR rowsort+SELECT f.f_dkey, d.env, sum(f.value)+FROM fact_table f+INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey+GROUP BY f.f_dkey, d.env;+----+A dev 772.4+C prod 2017.6++# Same result with the optimization disabled.+statement ok+set datafusion.optimizer.preserve_file_partitions = 0;++query TTR rowsort+SELECT f.f_dkey, d.env, sum(f.value)+FROM fact_table f+INNER JOIN dimension_table_skewed d ON f.f_dkey = d.d_dkey+GROUP BY f.f_dkey, d.env;+----+A dev 772.4+C prod 2017.6++##########+# TEST 15: Partitioned join against a non partitioned VALUES input+# The inserted RepartitionExec reuses the scan's split points, so partition i on both+# sides covers the same key interval.+##########++statement ok+set datafusion.optimizer.preserve_file_partitions = 1;++query TT+EXPLAIN SELECT f.f_dkey, d.env, sum(f.value)+FROM fact_table f+INNER JOIN (VALUES ('A', 'dev'), ('B', 'prod'), ('C', 'prod'), ('Z', 'none')) AS d(d_dkey, env)+ ON f.f_dkey = d.d_dkey+GROUP BY f.f_dkey, d.env;+----+logical_plan+01)Aggregate: groupBy=[[f.f_dkey, d.env]], aggr=[[sum(f.value)]]+02)--Projection: f.value, f.f_dkey, d.env+03)----Inner Join: f.f_dkey = CAST(d.d_dkey AS Utf8View)+04)------SubqueryAlias: f+05)--------TableScan: fact_table projection=[value, f_dkey]+06)------SubqueryAlias: d+07)--------Projection: column1 AS d_dkey, column2 AS env+08)----------Values: (Utf8("A"), Utf8("dev")), (Utf8("B"), Utf8("prod")), (Utf8("C"), Utf8("prod")), (Utf8("Z"), Utf8("none"))+physical_plan+01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)]+02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=3+03)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)]+04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(d.d_dkey AS Utf8View)@2, f_dkey@1)], projection=[value@3, f_dkey@4, env@1]+05)--------RepartitionExec: partitioning=Range([CAST(d.d_dkey AS Utf8View)@2 ASC], [(B), (C)], 3), input_partitions=1+06)----------ProjectionExec: expr=[column1@0 as d_dkey, column2@1 as env, CAST(column1@0 AS Utf8View) as CAST(d.d_dkey AS Utf8View)]+07)------------DataSourceExec: partitions=1, partition_sizes=[1]+08)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible++query TTR rowsort+SELECT f.f_dkey, d.env, sum(f.value)+FROM fact_table f+INNER JOIN (VALUES ('A', 'dev'), ('B', 'prod'), ('C', 'prod'), ('Z', 'none')) AS d(d_dkey, env)+ ON f.f_dkey = d.d_dkey+GROUP BY f.f_dkey, d.env;+----+A dev 772.4+B prod 614.4+C prod 2017.6++##########+# TEST 16: Self join on merged groups keeps every row+# 5 values merge into [A], [B, C], [D, E], and all must stay routable.+##########++query TII rowsort+SELECT a.f_dkey, count(*), count(b.value)+FROM high_cardinality_table a+INNER JOIN high_cardinality_table b ON a.f_dkey = b.f_dkey+GROUP BY a.f_dkey;+----+A 1 1+B 1 1+C 1 1+D 1 1+E 1 1++##########+# TEST 17: Statistics regrouping drops the Range claim+# The re-cut groups no longer follow partition values, so no output partitioning is+# declared, even when the group count is unchanged.+##########++statement ok+set datafusion.optimizer.preserve_file_partitions = 1;++statement ok+set datafusion.execution.split_file_groups_by_statistics = true;++query TT+EXPLAIN SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey;+----+logical_plan+01)Sort: fact_table_ordered.f_dkey ASC NULLS LAST+02)--Projection: fact_table_ordered.f_dkey, count(Int64(1)) AS count(*), avg(fact_table_ordered.value)+03)----Aggregate: groupBy=[[fact_table_ordered.f_dkey]], aggr=[[count(Int64(1)), avg(fact_table_ordered.value)]]+04)------TableScan: fact_table_ordered projection=[value, f_dkey]+physical_plan+01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]+02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, count(Int64(1))@1 as count(*), avg(fact_table_ordered.value)@2 as avg(fact_table_ordered.value)]+03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full+04)------RepartitionExec: partitioning=Hash([f_dkey@0], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST+05)--------AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full+06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet++query TIR+SELECT f_dkey, count(*), avg(value) FROM fact_table_ordered GROUP BY f_dkey ORDER BY f_dkey;+----+A 7 110.342857142857+B 7 87.771428571429+C 7 288.228571428571+########### CLEANUP##########@@ -769,6 +956,9 @@ reset datafusion.optimizer.hash_join_single_partition_threshold;statement okreset datafusion.optimizer.hash_join_single_partition_threshold_rows;+statement ok+reset datafusion.execution.split_file_groups_by_statistics;+statement okDROP TABLE fact_table;@@ -781,5 +971,8 @@ DROP TABLE dimension_table;statement okDROP TABLE dimension_table_partitioned;+statement ok+DROP TABLE dimension_table_skewed;+statement okDROP TABLE high_cardinality_table;diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.sltindex 09bf131e5e..aaebd04e51 100644--- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt+++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt@@ -164,7 +164,7 @@ physical_plan03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full04)------RepartitionExec: partitioning=Hash([f_dkey@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 ASC NULLS LAST05)--------AggregateExec: mode=Partial, gby=[f_dkey@2 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full-06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet+06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet# Verify results without subset satisfactionquery TPIR rowsort@@ -204,7 +204,7 @@ physical_plan01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST]02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)@1 as time_bin, count(Int64(1))@2 as count(*), avg(fact_table_ordered.value)@3 as avg(fact_table_ordered.value)]03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@2 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)], aggr=[count(Int64(1)), avg(fact_table_ordered.value)], group_clustering_mode=Full-04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet+04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet# Verify results match with subset satisfactionquery TPIR rowsort@@ -251,7 +251,7 @@ physical_plan02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano(\"IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }\"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]03)----SortExec: expr=[f_dkey@2 ASC NULLS LAST, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0) ASC NULLS LAST, timestamp@0 ASC NULLS LAST], preserve_partitioning=[true]04)------RepartitionExec: partitioning=Hash([f_dkey@2, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@0)], 3), input_partitions=3-05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet+05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet# Verify results without subset satisfactionquery TPRI rowsort@@ -292,7 +292,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[f_dkey@2 as f_dkey, timestamp@0 as timestamp, value@1 as value, row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rn]02)--BoundedWindowAggExec: wdw=[row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [fact_table_ordered.f_dkey, date_bin(IntervalMonthDayNano(\"IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }\"),fact_table_ordered.timestamp)] ORDER BY [fact_table_ordered.timestamp ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet# Verify results match with subset satisfactionquery TPRI rowsort@@ -379,8 +379,8 @@ physical_plan11)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]12)----------------------CoalescePartitionsExec13)------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]-14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible+14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible# Verify results without subset satisfactionquery TPR rowsort@@ -474,8 +474,8 @@ physical_plan09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]10)------------------CoalescePartitionsExec11)--------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]-12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible+12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible# Verify results match with subset satisfactionquery TPR rowsort