Apache DataFusion SQL Query Engine
Add `ListingOptions::output_partitioning` and `FileScanConfig::output_partitioning` for pre-defined file partitioning (#22657)
Gene Bordegaray63ad99181a1b
diff --git a/Cargo.lock b/Cargo.lockindex c97b3aec96..1b84a6b979 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2633,7 +2633,6 @@ dependencies = ["chrono","clap","datafusion",- "datafusion-datasource","datafusion-spark","datafusion-substrait","env_logger",diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rsindex 4f83ec4b37..796fca372b 100644--- a/datafusion/catalog-listing/src/helpers.rs+++ b/datafusion/catalog-listing/src/helpers.rs@@ -325,7 +325,7 @@ pub fn evaluate_partition_prefix<'a>(}}-fn filter_partitions(+pub fn filter_partitioned_file(pf: PartitionedFile,filters: &[Expr],df_schema: &DFSchema,@@ -447,7 +447,7 @@ pub async fn pruned_partition_list<'a>())}).try_filter_map(move |pf| {- futures::future::ready(filter_partitions(pf, filters, &df_schema))+ futures::future::ready(filter_partitioned_file(pf, filters, &df_schema))}).boxed())}diff --git a/datafusion/catalog-listing/src/options.rs b/datafusion/catalog-listing/src/options.rsindex 55840eb0e3..8e14fce341 100644--- a/datafusion/catalog-listing/src/options.rs+++ b/datafusion/catalog-listing/src/options.rs@@ -20,7 +20,7 @@ use datafusion_catalog::Session;use datafusion_common::plan_err;use datafusion_datasource::ListingTableUrl;use datafusion_datasource::file_format::FileFormat;-use datafusion_expr::SortExpr;+use datafusion_expr::{Partitioning, SortExpr};use futures::StreamExt;use futures::TryStreamExt;use itertools::Itertools;@@ -53,6 +53,46 @@ pub struct ListingOptions {/// multiple equivalent orderings, the outer `Vec` will have a/// single element.pub file_sort_order: Vec<Vec<SortExpr>>,+ /// Declared output partitioning for scans from this table.+ ///+ /// Expressions are logical expressions over the full table schema. When set,+ /// [`ListingTable`](crate::ListingTable) creates one file group per+ /// declared output partition. When unset, file grouping uses the scan-time+ /// [`SessionConfig::target_partitions`](datafusion_execution::config::SessionConfig::target_partitions).+ ///+ /// Files are listed in path order, split into whole-file groups across the+ /// declared partition count, and then padded with trailing empty groups when+ /// needed. DataFusion does not route files by partition values or validate+ /// row placement, so callers must ensure file group `i` contains rows for+ /// partition `i`. Layouts that require explicit file-to-partition assignment+ /// are not supported.+ ///+ /// For example, range partitioning on column `a` with split points+ /// `[10, 20, 30]` declares four output partitions. With three path-ordered+ /// files, the trailing partition is preserved as empty:+ ///+ /// ```text+ /// files in path order: f0, f1, f2+ ///+ /// file groups:+ /// partition 0: [f0]+ /// partition 1: [f1]+ /// partition 2: [f2]+ /// partition 3: []+ /// ```+ ///+ /// With five path-ordered files, a partition can contain multiple files:+ ///+ /// ```text+ /// files in path order: f0, f1, f2, f3, f4+ ///+ /// file groups:+ /// partition 0: [f0, f1]+ /// partition 1: [f2, f3]+ /// partition 2: [f4]+ /// partition 3: []+ /// ```+ pub output_partitioning: Option<Partitioning>,}impl ListingOptions {@@ -66,6 +106,7 @@ impl ListingOptions {format,table_partition_cols: vec![],file_sort_order: vec![],+ output_partitioning: None,}}@@ -113,6 +154,17 @@ impl ListingOptions {self}+ /// Set declared output partitioning.+ ///+ /// See [`Self::output_partitioning`] for the contract.+ pub fn with_output_partitioning(+ mut self,+ output_partitioning: Option<Partitioning>,+ ) -> Self {+ self.output_partitioning = output_partitioning;+ self+ }+/// Set `table partition columns` on [`ListingOptions`] and returns self.////// "partition columns," used to support [Hive Partitioning], arediff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex 04feec2b6e..36d85b981c 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -16,14 +16,17 @@// under the License.use crate::config::SchemaSource;-use crate::helpers::{expr_applicable_for_cols, pruned_partition_list};+use crate::helpers::{+ expr_applicable_for_cols, filter_partitioned_file, pruned_partition_list,+};use crate::{ListingOptions, ListingTableConfig};use arrow::datatypes::{Field, Schema, SchemaBuilder, SchemaRef};use async_trait::async_trait;use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};use datafusion_common::stats::Precision;use datafusion_common::{- Constraints, SchemaExt, Statistics, internal_datafusion_err, plan_err, project_schema,+ Constraints, DFSchema, SchemaExt, Statistics, internal_datafusion_err, plan_err,+ project_schema,};use datafusion_datasource::file::FileSource;use datafusion_datasource::file_groups::FileGroup;@@ -37,8 +40,10 @@ use datafusion_datasource::{use datafusion_execution::cache::cache_manager::{FileStatisticsCache, TableScopedPath};use datafusion_expr::dml::InsertOp;use datafusion_expr::execution_props::ExecutionProps;-use datafusion_expr::{Expr, TableProviderFilterPushDown, TableType};-use datafusion_physical_expr::create_lex_ordering;+use datafusion_expr::{+ Expr, Partitioning as LogicalPartitioning, TableProviderFilterPushDown, TableType,+};+use datafusion_physical_expr::{create_lex_ordering, create_physical_partitioning};use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::ExecutionPlan;@@ -448,6 +453,20 @@ fn derive_common_ordering_from_files(file_groups: &[FileGroup]) -> Option<LexOrd}}+fn filter_file_group_by_partition_filters(+ file_group: FileGroup,+ filters: &[Expr],+ df_schema: &DFSchema,+) -> datafusion_common::Result<FileGroup> {+ let files = file_group+ .into_inner()+ .into_iter()+ .map(|file| filter_partitioned_file(file, filters, df_schema))+ .filter_map(Result::transpose)+ .collect::<datafusion_common::Result<Vec<_>>>()?;+ Ok(FileGroup::new(files))+}+// Expressions can be used for partition pruning if they can be evaluated using// only the partition columns and there are partition columns.fn can_be_evaluated_for_partition_pruning(@@ -515,9 +534,19 @@ impl TableProvider for ListingTable {can_be_evaluated_for_partition_pruning(&table_partition_col_names, filter)});- // We should not limit the number of partitioned files to scan if there are filters and limit- // at the same time. This is because the limit should be applied after the filters are applied.- let statistic_file_limit = if filters.is_empty() { limit } else { None };+ let declared_output_partitioning = self.options.output_partitioning.as_ref();++ // We should not limit files before assigning declared output partitions+ // or before applying non-partition filters.+ let statistic_file_limit =+ if filters.is_empty() && declared_output_partitioning.is_none() {+ limit+ } else {+ None+ };+ let file_group_count = declared_output_partitioning+ .and_then(LogicalPartitioning::partition_count)+ .unwrap_or_else(|| state.config().target_partitions());let ListFilesResult {file_groups: mut partitioned_file_lists,@@ -537,17 +566,19 @@ impl TableProvider for ListingTable {state.execution_props(),&partitioned_file_lists,)?;- match state- .config_options()- .execution- .split_file_groups_by_statistics+ let split_file_groups_by_statistics = declared_output_partitioning.is_none()+ && state+ .config_options()+ .execution+ .split_file_groups_by_statistics;+ match split_file_groups_by_statistics.then(|| {output_ordering.first().map(|output_ordering| {FileScanConfig::split_groups_by_statistics_with_target_partitions(&self.table_schema,&partitioned_file_lists,output_ordering,- state.config().target_partitions(),+ file_group_count,)})})@@ -555,7 +586,7 @@ impl TableProvider for ListingTable {{Some(Err(e)) => log::debug!("failed to split file groups by statistics: {e}"),Some(Ok(new_groups)) => {- if new_groups.len() <= state.config().target_partitions() {+ if new_groups.len() <= file_group_count {partitioned_file_lists = new_groups;} else {log::debug!(@@ -566,6 +597,41 @@ impl TableProvider for ListingTable {None => {} // no ordering required};+ let output_partitioning = if let Some(output_partitioning) =+ declared_output_partitioning+ {+ let output_partitioning = match output_partitioning {+ LogicalPartitioning::RoundRobinBatch(_) => {+ return datafusion_common::not_impl_err!(+ "RoundRobinBatch output partitioning is not supported for ListingTable"+ );+ }+ LogicalPartitioning::DistributeBy(_) => {+ return datafusion_common::not_impl_err!(+ "DistributeBy output partitioning is not supported for ListingTable"+ );+ }+ LogicalPartitioning::Hash(_, _) | LogicalPartitioning::Range(_) => {+ let df_schema = DFSchema::try_from(Arc::clone(&self.table_schema))?;+ create_physical_partitioning(+ output_partitioning,+ &df_schema,+ state.execution_props(),+ )?+ }+ };+ let partition_count = output_partitioning.partition_count();+ if partitioned_file_lists.len() != partition_count {+ return plan_err!(+ "ListingTable output_partitioning has {partition_count} partitions, but the scan has {} file groups",+ partitioned_file_lists.len()+ );+ }+ Some(output_partitioning)+ } else {+ None+ };+let Some(object_store_url) =self.table_paths.first().map(ListingTableUrl::object_store)else {@@ -575,24 +641,23 @@ impl TableProvider for ListingTable {};let file_source = self.create_file_source();+ let scan_config = FileScanConfigBuilder::new(object_store_url, file_source)+ .with_file_groups(partitioned_file_lists)+ .with_constraints(self.constraints.clone())+ .with_statistics(statistics)+ .with_projection_indices(projection)?+ .with_limit(limit)+ .with_output_ordering(output_ordering)+ .with_output_partitioning(output_partitioning)+ .with_expr_adapter(self.expr_adapter_factory.clone())+ .with_partitioned_by_file_group(partitioned_by_file_group)+ .build();// create the execution planlet plan = self.options.format- .create_physical_plan(- state,- FileScanConfigBuilder::new(object_store_url, file_source)- .with_file_groups(partitioned_file_lists)- .with_constraints(self.constraints.clone())- .with_statistics(statistics)- .with_projection_indices(projection)?- .with_limit(limit)- .with_output_ordering(output_ordering)- .with_expr_adapter(self.expr_adapter_factory.clone())- .with_partitioned_by_file_group(partitioned_by_file_group)- .build(),- )+ .create_physical_plan(state, scan_config).await?;Ok(ScanResult::new(plan))@@ -704,28 +769,41 @@ impl ListingTable {/// Get the list of files for a scan as well as the file level statistics./// The list is grouped to let the execution plan know how the files should/// be distributed to different threads / executors.+ ///+ /// If [`ListingOptions::output_partitioning`] is set, returns one file+ /// group per declared partition, including empty trailing groups.pub async fn list_files_for_scan<'a>(&'a self,ctx: &'a dyn Session,filters: &'a [Expr],limit: Option<usize>,) -> datafusion_common::Result<ListFilesResult> {- let store = if let Some(url) = self.table_paths.first() {- ctx.runtime_env().object_store(url)?+ if let Some(output_partitioning) = self.options.output_partitioning.as_ref() {+ self.list_files_for_declared_output_partitioning(+ ctx,+ output_partitioning,+ filters,+ )+ .await} else {- return Ok(ListFilesResult {- file_groups: vec![],- statistics: Statistics::new_unknown(&self.file_schema),- grouped_by_partition: false,- });- };+ self.list_files_for_regular_scan(ctx, filters, limit).await+ }+ }++ async fn collect_files_for_scan<'a>(+ &'a self,+ ctx: &'a dyn Session,+ store: &'a Arc<dyn ObjectStore>,+ listing_time_filters: &'a [Expr],+ file_limit: Option<usize>,+ ) -> datafusion_common::Result<(FileGroup, bool)> {// list files (with partitions)let file_list = future::try_join_all(self.table_paths.iter().map(|table_path| {pruned_partition_list(ctx,store.as_ref(),table_path,- filters,+ listing_time_filters,&self.options.file_extension,&self.options.table_partition_cols,)@@ -739,7 +817,7 @@ impl ListingTable {.map(|part_file| async {let part_file = part_file?;let (statistics, ordering) = if ctx.config().collect_statistics() {- self.do_collect_statistics_and_ordering(ctx, &store, &part_file)+ self.do_collect_statistics_and_ordering(ctx, store, &part_file).await?} else {(Arc::new(Statistics::new_unknown(&self.file_schema)), None)@@ -751,8 +829,34 @@ impl ListingTable {.boxed().buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency);- let (file_group, inexact_stats) =- get_files_with_limit(files, limit, ctx.config().collect_statistics()).await?;+ get_files_with_limit(files, file_limit, ctx.config().collect_statistics()).await+ }++ async fn list_files_for_regular_scan<'a>(+ &'a self,+ ctx: &'a dyn Session,+ filters: &'a [Expr],+ limit: Option<usize>,+ ) -> datafusion_common::Result<ListFilesResult> {+ let file_group_count = ctx.config().target_partitions();+ if file_group_count == 0 {+ return plan_err!(+ "ListingTable requires target_partitions to be greater than zero"+ );+ }++ let store = if let Some(url) = self.table_paths.first() {+ ctx.runtime_env().object_store(url)?+ } else {+ return Ok(ListFilesResult {+ file_groups: vec![],+ statistics: Statistics::new_unknown(&self.file_schema),+ grouped_by_partition: false,+ });+ };+ let (file_group, inexact_stats) = self+ .collect_files_for_scan(ctx, &store, filters, limit)+ .await?;// Threshold: 0 = disabled, N > 0 = enabled when distinct_keys >= N//@@ -763,26 +867,100 @@ impl ListingTable {let (file_groups, grouped_by_partition) =if threshold > 0 && !self.options.table_partition_cols.is_empty() {- let grouped = file_group- .group_by_partition_values(ctx.config().target_partitions());+ let grouped = file_group.group_by_partition_values(file_group_count);if grouped.len() >= threshold {(grouped, true)} else {let all_files: Vec<_> =grouped.into_iter().flat_map(|g| g.into_inner()).collect();(- FileGroup::new(all_files)- .split_files(ctx.config().target_partitions()),+ FileGroup::new(all_files).split_files(file_group_count),false,)}} else {- (- file_group.split_files(ctx.config().target_partitions()),- false,- )+ (file_group.split_files(file_group_count), false)};+ self.list_files_result_from_groups(+ ctx,+ file_groups,+ inexact_stats,+ grouped_by_partition,+ )+ }++ async fn list_files_for_declared_output_partitioning<'a>(+ &'a self,+ ctx: &'a dyn Session,+ output_partitioning: &LogicalPartitioning,+ filters: &'a [Expr],+ ) -> datafusion_common::Result<ListFilesResult> {+ let Some(file_group_count) = output_partitioning.partition_count() else {+ return datafusion_common::not_impl_err!(+ "DistributeBy output partitioning is not supported for ListingTable"+ );+ };+ if file_group_count == 0 {+ return plan_err!(+ "ListingTable output_partitioning requires at least one partition"+ );+ }++ let store = if let Some(url) = self.table_paths.first() {+ ctx.runtime_env().object_store(url)?+ } else {+ return Ok(ListFilesResult {+ file_groups: vec![],+ statistics: Statistics::new_unknown(&self.file_schema),+ grouped_by_partition: false,+ });+ };+ let (file_group, inexact_stats) =+ self.collect_files_for_scan(ctx, &store, &[], None).await?;+ let mut file_groups = file_group.split_files(file_group_count);+ if !file_groups.is_empty() {+ file_groups.resize_with(file_group_count, || FileGroup::new(vec![]));+ }+ let file_groups =+ self.filter_declared_file_groups_by_partition_filters(file_groups, filters)?;++ self.list_files_result_from_groups(ctx, file_groups, inexact_stats, false)+ }++ fn filter_declared_file_groups_by_partition_filters(+ &self,+ file_groups: Vec<FileGroup>,+ filters: &[Expr],+ ) -> datafusion_common::Result<Vec<FileGroup>> {+ if filters.is_empty() {+ return Ok(file_groups);+ }++ let df_schema = DFSchema::from_unqualified_fields(+ self.options+ .table_partition_cols+ .iter()+ .map(|(name, data_type)| Field::new(name, data_type.clone(), true))+ .collect(),+ Default::default(),+ )?;++ file_groups+ .into_iter()+ .map(|file_group| {+ filter_file_group_by_partition_filters(file_group, filters, &df_schema)+ })+ .collect::<datafusion_common::Result<Vec<_>>>()+ }++ fn list_files_result_from_groups(+ &self,+ ctx: &dyn Session,+ file_groups: Vec<FileGroup>,+ inexact_stats: bool,+ grouped_by_partition: bool,+ ) -> datafusion_common::Result<ListFilesResult> {let (file_groups, stats) = compute_all_files_statistics(file_groups,self.schema(),diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 50b3855a0a..39c20f9b78 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -123,7 +123,7 @@ mod tests {},};use arrow::{compute::SortOptions, record_batch::RecordBatch};- use arrow_schema::{DataType, Field, Schema, SchemaRef};+ use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};use datafusion_catalog::TableProvider;use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig, SchemaSource,@@ -137,13 +137,18 @@ mod tests {use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_format::FileFormat;use datafusion_expr::dml::InsertOp;- use datafusion_expr::{BinaryExpr, LogicalPlanBuilder, Operator};+ use datafusion_expr::{+ BinaryExpr, LogicalPlanBuilder, Operator, Partitioning as LogicalPartitioning,+ RangePartitioning as LogicalRangePartitioning,+ };use datafusion_physical_expr::PhysicalSortExpr;- use datafusion_physical_expr::expressions::binary;+ use datafusion_physical_expr::expressions::{Column, binary};use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::statistics::StatisticsArgs;- use datafusion_physical_plan::{ExecutionPlanProperties, collect};+ use datafusion_physical_plan::{+ ExecutionPlanProperties, Partitioning, RangePartitioning, SplitPoint, collect,+ };use std::collections::HashMap;use std::io::Write;use std::sync::Arc;@@ -176,6 +181,21 @@ mod tests {.collect()}+ fn listing_table_with_files(+ ctx: &SessionContext,+ files: &[&str],+ table_path: &str,+ options: ListingOptions,+ schema: Schema,+ ) -> Result<ListingTable> {+ register_test_store(ctx, &files.iter().map(|f| (*f, 10)).collect::<Vec<_>>());++ let config = ListingTableConfig::new(ListingTableUrl::parse(table_path)?)+ .with_listing_options(options)+ .with_schema(Arc::new(schema));+ ListingTable::try_new(config)+ }+#[tokio::test]async fn test_schema_source_tracking_comprehensive() -> Result<()> {let ctx = SessionContext::new();@@ -1289,6 +1309,236 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_list_files_uses_declared_output_partitioning_count() -> Result<()> {+ let files = ["bucket/key-prefix/file0", "bucket/key-prefix/file1"];++ let ctx = SessionContext::new_with_config(+ SessionConfig::new().with_target_partitions(1),+ );+ let opt = ListingOptions::new(Arc::new(JsonFormat::default()))+ .with_file_extension_opt(Some(""))+ .with_output_partitioning(Some(LogicalPartitioning::Range(+ LogicalRangePartitioning::try_new(+ vec![col("a").sort(true, true)],+ vec![+ SplitPoint::new(vec![ScalarValue::from(10i32)]),+ SplitPoint::new(vec![ScalarValue::from(20i32)]),+ SplitPoint::new(vec![ScalarValue::from(30i32)]),+ ],+ )?,+ )));+ let table = listing_table_with_files(+ &ctx,+ &files,+ "test:///bucket/key-prefix/",+ opt,+ Schema::new(vec![Field::new("a", DataType::Int32, false)]),+ )?;++ let result = table.list_files_for_scan(&ctx.state(), &[], None).await?;+ let group_sizes = result+ .file_groups+ .iter()+ .map(|group| group.len())+ .collect::<Vec<_>>();++ assert_eq!(group_sizes, vec![1, 1, 0, 0]);++ Ok(())+ }++ #[tokio::test]+ async fn test_range_output_partitioning_normalizes_split_point_types() -> Result<()> {+ let files = ["bucket/key-prefix/file0"];++ let ctx = SessionContext::new();+ let output_partitioning =+ LogicalPartitioning::Range(LogicalRangePartitioning::try_new(+ vec![col("a").sort(true, true)],+ vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond(+ Some(123_000_000_000),+ None,+ )])],+ )?);+ let expected_output_partitioning =+ Partitioning::Range(RangePartitioning::try_new(+ LexOrdering::new(vec![PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ SortOptions::default(),+ )])+ .unwrap(),+ vec![SplitPoint::new(vec![ScalarValue::TimestampSecond(+ Some(123),+ None,+ )])],+ )?);++ let opt = ListingOptions::new(Arc::new(JsonFormat::default()))+ .with_file_extension_opt(Some(""))+ .with_output_partitioning(Some(output_partitioning));+ let table = listing_table_with_files(+ &ctx,+ &files,+ "test:///bucket/key-prefix/",+ opt,+ Schema::new(vec![Field::new(+ "a",+ DataType::Timestamp(TimeUnit::Second, None),+ false,+ )]),+ )?;++ let scan = table.scan(&ctx.state(), None, &[], None).await?;+ assert_eq!(scan.output_partitioning(), &expected_output_partitioning);++ Ok(())+ }++ #[tokio::test]+ async fn test_range_output_partitioning_rejects_invalid_split_point_type()+ -> Result<()> {+ let files = ["bucket/key-prefix/file0"];++ let ctx = SessionContext::new();+ let output_partitioning =+ LogicalPartitioning::Range(LogicalRangePartitioning::try_new(+ vec![col("a").sort(true, true)],+ vec![SplitPoint::new(vec![ScalarValue::Utf8(Some(+ "not-an-int".to_string(),+ ))])],+ )?);++ let opt = ListingOptions::new(Arc::new(JsonFormat::default()))+ .with_file_extension_opt(Some(""))+ .with_output_partitioning(Some(output_partitioning));+ let table = listing_table_with_files(+ &ctx,+ &files,+ "test:///bucket/key-prefix/",+ opt,+ Schema::new(vec![Field::new("a", DataType::Int32, false)]),+ )?;++ let err = table.scan(&ctx.state(), None, &[], None).await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "Range output partitioning split point 0 value 0 with type Utf8 cannot be represented exactly as ordering expression type Int32"+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_range_output_partitioning_rejects_lossy_timestamp_split_point()+ -> Result<()> {+ let files = ["bucket/key-prefix/file0"];++ let ctx = SessionContext::new();+ let output_partitioning =+ LogicalPartitioning::Range(LogicalRangePartitioning::try_new(+ vec![col("a").sort(true, true)],+ vec![SplitPoint::new(vec![ScalarValue::TimestampNanosecond(+ Some(123_456),+ None,+ )])],+ )?);++ let opt = ListingOptions::new(Arc::new(JsonFormat::default()))+ .with_file_extension_opt(Some(""))+ .with_output_partitioning(Some(output_partitioning));+ let table = listing_table_with_files(+ &ctx,+ &files,+ "test:///bucket/key-prefix/",+ opt,+ Schema::new(vec![Field::new(+ "a",+ DataType::Timestamp(TimeUnit::Second, None),+ false,+ )]),+ )?;++ let err = table.scan(&ctx.state(), None, &[], None).await.unwrap_err();+ assert_contains!(+ err.to_string(),+ "Range output partitioning split point 0 value 0 with type Timestamp(ns) cannot be represented exactly as ordering expression type Timestamp(s)"+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_partition_filter_preserves_declared_output_partitioning() -> Result<()>+ {+ let files = ["bucket/test/pid=1/file1", "bucket/test/pid=2/file2"];++ let ctx = SessionContext::new();+ let output_partitioning =+ LogicalPartitioning::Range(LogicalRangePartitioning::try_new(+ vec![col("pid").sort(true, true)],+ vec![SplitPoint::new(vec![ScalarValue::from(2i32)])],+ )?);+ let expected_output_partitioning =+ Partitioning::Range(RangePartitioning::try_new(+ LexOrdering::new(vec![PhysicalSortExpr::new(+ Arc::new(Column::new("pid", 1)),+ SortOptions::default(),+ )])+ .unwrap(),+ vec![SplitPoint::new(vec![ScalarValue::from(2i32)])],+ )?);++ let opt = ListingOptions::new(Arc::new(JsonFormat::default()))+ .with_file_extension_opt(Some(""))+ .with_table_partition_cols(vec![("pid".to_string(), DataType::Int32)])+ .with_output_partitioning(Some(output_partitioning.clone()));++ let table = listing_table_with_files(+ &ctx,+ &files,+ "test:///bucket/test/",+ opt,+ Schema::new(vec![Field::new("a", DataType::Boolean, false)]),+ )?;++ let unfiltered = table.scan(&ctx.state(), None, &[], None).await?;+ assert_eq!(+ unfiltered.output_partitioning(),+ &expected_output_partitioning+ );++ let filter = Expr::eq(col("pid"), lit(2_i32));+ let file_groups = table+ .list_files_for_scan(&ctx.state(), std::slice::from_ref(&filter), None)+ .await?+ .file_groups+ .into_iter()+ .map(|group| {+ group+ .into_inner()+ .into_iter()+ .map(|file| file.path().to_string())+ .collect::<Vec<_>>()+ })+ .collect::<Vec<_>>();+ assert_eq!(+ file_groups,+ vec![+ Vec::<String>::new(),+ vec!["bucket/test/pid=2/file2".to_string()]+ ]+ );++ let filtered = table.scan(&ctx.state(), None, &[filter], None).await?;+ assert_eq!(+ filtered.output_partitioning(),+ &expected_output_partitioning+ );++ Ok(())+ }+#[tokio::test]async fn test_listing_table_prunes_extra_files_in_hive() -> Result<()> {let files = [diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 190a08da12..9cd2fd1131 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -32,10 +32,11 @@ use crate::logical_expr::{Aggregate, EmptyRelation, Join, Projection, Sort, TableScan, Unnest, Values, Window,};use crate::logical_expr::{- Expr, LogicalPlan, Partitioning as LogicalPartitioning, PlanType, Repartition,- UserDefinedLogicalNode,+ Expr, LogicalPlan, PlanType, Repartition, UserDefinedLogicalNode,+};+use crate::physical_expr::{+ create_physical_expr, create_physical_exprs, create_physical_partitioning,};-use crate::physical_expr::{create_physical_expr, create_physical_exprs};use crate::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy};use crate::physical_plan::analyze::AnalyzeExec;use crate::physical_plan::explain::ExplainExec;@@ -52,8 +53,8 @@ use crate::physical_plan::union::UnionExec;use crate::physical_plan::unnest::UnnestExec;use crate::physical_plan::windows::{BoundedWindowAggExec, WindowAggExec};use crate::physical_plan::{- ExecutionPlan, ExecutionPlanProperties, InputOrderMode, Partitioning, PhysicalExpr,- WindowExpr, displayable, windows,+ ExecutionPlan, ExecutionPlanProperties, InputOrderMode, PhysicalExpr, WindowExpr,+ displayable, windows,};use crate::schema_equivalence::schema_satisfied_by;@@ -98,7 +99,7 @@ use datafusion_physical_expr::aggregate::{};use datafusion_physical_expr::expressions::Literal;use datafusion_physical_expr::{- LexOrdering, PhysicalSortExpr, RangePartitioning, create_physical_sort_exprs,+ LexOrdering, PhysicalSortExpr, create_physical_sort_exprs,};use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::empty::EmptyExec;@@ -1251,41 +1252,11 @@ impl DefaultPhysicalPlanner {}) => {let physical_input = children.one()?;let input_dfschema = input.as_ref().schema();- let physical_partitioning = match partitioning_scheme {- LogicalPartitioning::RoundRobinBatch(n) => {- Partitioning::RoundRobinBatch(*n)- }- LogicalPartitioning::Hash(expr, n) => {- let runtime_expr = expr- .iter()- .map(|e| {- create_physical_expr(e, input_dfschema, execution_props)- })- .collect::<Result<Vec<_>>>()?;- Partitioning::Hash(runtime_expr, *n)- }- LogicalPartitioning::Range(range) => {- let sort_exprs = create_physical_sort_exprs(- range.ordering(),- input_dfschema,- execution_props,- )?;- let ordering = LexOrdering::new(sort_exprs).ok_or_else(|| {- internal_datafusion_err!(- "Range repartitioning requires non-empty ordering"- )- })?;- Partitioning::Range(RangePartitioning::try_new(- ordering,- range.split_points().to_vec(),- )?)- }- LogicalPartitioning::DistributeBy(_) => {- return not_impl_err!(- "Physical plan does not support DistributeBy partitioning"- );- }- };+ let physical_partitioning = create_physical_partitioning(+ partitioning_scheme,+ input_dfschema,+ execution_props,+ )?;Arc::new(RepartitionExec::try_new(physical_input,physical_partitioning,@@ -3249,8 +3220,8 @@ mod tests {use crate::datasource::MemTable;use crate::datasource::file_format::options::CsvReadOptions;use crate::physical_plan::{- DisplayAs, DisplayFormatType, PlanProperties, SendableRecordBatchStream,- expressions,+ DisplayAs, DisplayFormatType, Partitioning, PlanProperties,+ SendableRecordBatchStream, expressions,};use crate::prelude::{SessionConfig, SessionContext};use crate::test_util::{scan_empty, scan_empty_with_partitions};@@ -3271,8 +3242,8 @@ mod tests {use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::{Accumulator, AggregateUDF, AggregateUDFImpl, ExprFunctionExt, LogicalPlanBuilder,- RangePartitioning, Signature, TableSource, UserDefinedLogicalNodeCore,- Volatility, WindowFunctionDefinition, col, lit,+ Partitioning as LogicalPartitioning, RangePartitioning, Signature, TableSource,+ UserDefinedLogicalNodeCore, Volatility, WindowFunctionDefinition, col, lit,};use datafusion_functions_aggregate::count::{count_all, count_udaf};use datafusion_functions_aggregate::expr_fn::sum;diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex b1ba0584c9..21d733458c 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -40,7 +40,7 @@ use datafusion_expr::Operator;use crate::source::OpenArgs;use datafusion_physical_expr::expressions::{BinaryExpr, Column};-use datafusion_physical_expr::projection::ProjectionExprs;+use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};use datafusion_physical_expr::utils::reassign_expr_columns;use datafusion_physical_expr::{EquivalenceProperties, Partitioning, split_conjunction};use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;@@ -205,7 +205,17 @@ pub struct FileScanConfig {////// If the number of file partitions > target_partitions, the file partitions will be grouped/// in a round-robin fashion such that number of file partitions = target_partitions.+ ///+ /// Follow-up: remove this redundant field in favor of+ /// `output_partitioning`, see <https://github.com/apache/datafusion/issues/23099>.pub partitioned_by_file_group: bool,+ /// Declared physical output partitioning for this scan.+ ///+ /// Expressions are against the full table schema, before scan projection or+ /// filtering. `ListingTable` validates partition count before building the+ /// scan, and direct builders with mismatched counts fall back to+ /// `UnknownPartitioning`.+ pub output_partitioning: Option<Partitioning>,}/// A builder for [`FileScanConfig`]'s.@@ -274,6 +284,7 @@ pub struct FileScanConfigBuilder {file_groups: Vec<FileGroup>,statistics: Option<Statistics>,output_ordering: Vec<LexOrdering>,+ output_partitioning: Option<Partitioning>,file_compression_type: Option<FileCompressionType>,batch_size: Option<usize>,expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,@@ -297,6 +308,7 @@ impl FileScanConfigBuilder {file_groups: vec![],statistics: None,output_ordering: vec![],+ output_partitioning: None,file_compression_type: None,limit: None,preserve_order: false,@@ -463,6 +475,15 @@ impl FileScanConfigBuilder {self}+ /// Set declared physical output partitioning for this scan.+ pub fn with_output_partitioning(+ mut self,+ output_partitioning: Option<Partitioning>,+ ) -> Self {+ self.output_partitioning = output_partitioning;+ self+ }+/// Set the file compression typepub fn with_file_compression_type(mut self,@@ -521,6 +542,7 @@ impl FileScanConfigBuilder {file_groups,statistics,output_ordering,+ output_partitioning,file_compression_type,batch_size,expr_adapter_factory: expr_adapter,@@ -550,6 +572,7 @@ impl FileScanConfigBuilder {expr_adapter_factory: expr_adapter,statistics,partitioned_by_file_group,+ output_partitioning,}}}@@ -562,6 +585,7 @@ impl From<FileScanConfig> for FileScanConfigBuilder {file_groups: config.file_groups,statistics: Some(config.statistics),output_ordering: config.output_ordering,+ output_partitioning: config.output_partitioning,file_compression_type: Some(config.file_compression_type),limit: config.limit,preserve_order: config.preserve_order,@@ -573,6 +597,52 @@ impl From<FileScanConfig> for FileScanConfigBuilder {}}+fn hash_partitioning_from_partition_fields(+ schema: &Schema,+ partition_cols: &Fields,+ partition_count: usize,+) -> Option<Partitioning> {+ if partition_cols.is_empty() {+ return None;+ }++ let mut exprs: Vec<Arc<dyn PhysicalExpr>> = Vec::with_capacity(partition_cols.len());+ for partition_col in partition_cols {+ let name = partition_col.name();+ let idx = schema+ .fields()+ .iter()+ .position(|field| field.name() == name)?;+ exprs.push(Arc::new(Column::new(name, idx)));+ }++ Some(Partitioning::Hash(exprs, partition_count))+}++fn project_output_partitioning(+ partitioning: &Partitioning,+ mapping: &ProjectionMapping,+ input_schema: &SchemaRef,+ partition_count: usize,+) -> Partitioning {+ let input_eq_properties = EquivalenceProperties::new(Arc::clone(input_schema));+ match partitioning {+ Partitioning::Hash(exprs, _) => {+ let projected_exprs = input_eq_properties+ .project_expressions(exprs, mapping)+ .collect::<Option<Vec<_>>>();+ projected_exprs+ .map(|exprs| Partitioning::Hash(exprs, partition_count))+ .unwrap_or_else(|| Partitioning::UnknownPartitioning(partition_count))+ }+ Partitioning::Range(_)+ | Partitioning::RoundRobinBatch(_)+ | Partitioning::UnknownPartitioning(_) => {+ partitioning.project(mapping, &input_eq_properties)+ }+ }+}+impl DataSource for FileScanConfig {fn open(&self,@@ -660,6 +730,10 @@ impl DataSource for FileScanConfig {display_orderings(f, &orderings)?;+ if self.output_partitioning.is_some() {+ write!(f, ", output_partitioning={}", self.output_partitioning())?;+ }+if !self.constraints.is_empty() {write!(f, ", {}", self.constraints)?;}@@ -683,10 +757,9 @@ impl DataSource for FileScanConfig {repartition_file_min_size: usize,output_ordering: Option<LexOrdering>,) -> Result<Option<Arc<dyn DataSource>>> {- // When files are grouped by partition values, we cannot allow byte-range- // splitting. It would mix rows from different partition values across- // file groups, breaking the Hash partitioning.- if self.partitioned_by_file_group {+ // When file groups define output partitioning, repartitioning files+ // would invalidate the partition-to-file-group mapping.+ if self.output_partitioning.is_some() || self.partitioned_by_file_group {return Ok(None);}@@ -702,13 +775,18 @@ impl DataSource for FileScanConfig {/// Returns the output partitioning for this file scan.///- /// When `partitioned_by_file_group` is true, this returns `Partitioning::Hash` on- /// the Hive partition columns, allowing the optimizer to skip hash repartitioning- /// for aggregates and joins on those columns.+ /// When `output_partitioning` is set, this returns the declared partitioning+ /// after applying scan projection. When `partitioned_by_file_group` is true,+ /// this returns `Partitioning::Hash` on the Hive partition columns, allowing+ /// the optimizer to skip hash repartitioning for aggregates and joins on+ /// those columns.+ ///+ /// If projection or partition count validation fails, this returns+ /// `UnknownPartitioning`.////// Tradeoffs- /// - Benefit: Eliminates `RepartitionExec` and `SortExec` for queries with- /// `GROUP BY` or `ORDER BY` on partition columns.+ /// - Benefit: Eliminates `RepartitionExec` and `SortExec` for queries whose+ /// required distribution is satisfied by the scan's output partitioning./// - Cost: Files are grouped by partition values rather than split by byte/// ranges, which may reduce I/O parallelism when partition sizes are uneven./// For simple aggregations without `ORDER BY`, this cost may outweigh the benefit.@@ -717,39 +795,45 @@ impl DataSource for FileScanConfig {/// - Idea: Could allow byte-range splitting within partition-aware groups,/// preserving I/O parallelism while maintaining partition semantics.fn output_partitioning(&self) -> Partitioning {- if self.partitioned_by_file_group {- let partition_cols = self.table_partition_cols();- if !partition_cols.is_empty() {- let projected_schema = match self.projected_schema() {- Ok(schema) => schema,- Err(_) => {- debug!(- "Could not get projected schema, falling back to UnknownPartitioning."- );- return Partitioning::UnknownPartitioning(self.file_groups.len());- }- };-- // Build Column expressions for partition columns based on their- // position in the projected schema- let mut exprs: Vec<Arc<dyn PhysicalExpr>> = Vec::new();- for partition_col in partition_cols {- if let Some((idx, _)) = projected_schema- .fields()- .iter()- .enumerate()- .find(|(_, f)| f.name() == partition_col.name())- {- exprs.push(Arc::new(Column::new(partition_col.name(), idx)));- }- }+ let Some(output_partitioning) = self.output_partitioning.clone().or_else(|| {+ self.partitioned_by_file_group.then(|| {+ hash_partitioning_from_partition_fields(+ self.file_source.table_schema().table_schema(),+ self.table_partition_cols(),+ self.file_groups.len(),+ )+ })?+ }) else {+ return Partitioning::UnknownPartitioning(self.file_groups.len());+ };+ if output_partitioning.partition_count() != self.file_groups.len() {+ warn!(+ "Declared output partitioning has {} partitions, but file scan has {} file groups. Falling back to UnknownPartitioning.",+ output_partitioning.partition_count(),+ self.file_groups.len()+ );+ return Partitioning::UnknownPartitioning(self.file_groups.len());+ }- if exprs.len() == partition_cols.len() {- return Partitioning::Hash(exprs, self.file_groups.len());+ if let Some(projection) = self.file_source.projection() {+ let schema = self.file_source.table_schema().table_schema();+ return match projection.projection_mapping(schema) {+ Ok(mapping) => project_output_partitioning(+ &output_partitioning,+ &mapping,+ schema,+ self.file_groups.len(),+ ),+ Err(e) => {+ debug!(+ "Could not project output partitioning, falling back to UnknownPartitioning: {e}"+ );+ Partitioning::UnknownPartitioning(self.file_groups.len())}- }+ };}- Partitioning::UnknownPartitioning(self.file_groups.len())++ output_partitioning}/// Computes the effective equivalence properties of this file scan, taking@@ -1043,7 +1127,10 @@ impl DataSource for FileScanConfig {/// when file order must be preserved or the file groups define the output/// partitioning needed for the rest of the planfn create_sibling_state(&self) -> Option<Arc<dyn Any + Send + Sync>> {- if self.preserve_order || self.partitioned_by_file_group {+ if self.preserve_order+ || self.output_partitioning.is_some()+ || self.partitioned_by_file_group+ {return None;}@@ -2459,6 +2546,58 @@ mod tests {assert!(matches!(partitioning, Partitioning::UnknownPartitioning(_)));}+ #[test]+ fn test_declared_output_partitioning_projects_with_scan() {+ let file_schema = aggr_test_schema();+ let output_partitioning =+ Partitioning::Hash(vec![Arc::new(Column::new("c2", 1))], 4);++ let mut config = config_for_projection(+ Arc::clone(&file_schema),+ Some(vec![1, 2]),+ Statistics::new_unknown(&file_schema),+ vec![],+ );+ config.file_groups = vec![+ FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),+ FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),+ FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]),+ FileGroup::new(vec![PartitionedFile::new("f4.parquet".to_string(), 1024)]),+ ];+ config.output_partitioning = Some(output_partitioning);++ match config.output_partitioning() {+ Partitioning::Hash(exprs, num_partitions) => {+ assert_eq!(num_partitions, 4);+ assert_eq!(exprs.len(), 1);+ let column = exprs[0].downcast_ref::<Column>().unwrap();+ assert_eq!(column.name(), "c2");+ assert_eq!(column.index(), 0);+ }+ _ => panic!("Expected Hash partitioning"),+ }++ let mut config = config_for_projection(+ Arc::clone(&file_schema),+ Some(vec![2]),+ Statistics::new_unknown(&file_schema),+ vec![],+ );+ config.file_groups = vec![+ FileGroup::new(vec![PartitionedFile::new("f1.parquet".to_string(), 1024)]),+ FileGroup::new(vec![PartitionedFile::new("f2.parquet".to_string(), 1024)]),+ FileGroup::new(vec![PartitionedFile::new("f3.parquet".to_string(), 1024)]),+ FileGroup::new(vec![PartitionedFile::new("f4.parquet".to_string(), 1024)]),+ ];+ config.output_partitioning =+ Some(Partitioning::Hash(vec![Arc::new(Column::new("c2", 1))], 4));++ assert!(matches!(+ config.output_partitioning(),+ Partitioning::UnknownPartitioning(4)+ ));+ }+#[test]fn test_output_partitioning_no_partition_columns() {let file_schema = aggr_test_schema();diff --git a/datafusion/physical-expr/src/lib.rs b/datafusion/physical-expr/src/lib.rsindex b55bd70bdf..67419944cf 100644--- a/datafusion/physical-expr/src/lib.rs+++ b/datafusion/physical-expr/src/lib.rs@@ -66,8 +66,9 @@ pub use expressions::{DynamicFilterTracker, DynamicFilterTracking};pub use partitioning::{Distribution, Partitioning, RangePartitioning};pub use physical_expr::{add_offset_to_expr, add_offset_to_physical_sort_exprs, create_lex_ordering,- create_ordering, create_physical_sort_expr, create_physical_sort_exprs,- physical_exprs_bag_equal, physical_exprs_contains, physical_exprs_equal,+ create_ordering, create_physical_partitioning, create_physical_sort_expr,+ create_physical_sort_exprs, physical_exprs_bag_equal, physical_exprs_contains,+ physical_exprs_equal,};pub use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, PhysicalExprRef};diff --git a/datafusion/physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rsindex 77ede76e1d..6ff5be4e38 100644--- a/datafusion/physical-expr/src/physical_expr.rs+++ b/datafusion/physical-expr/src/physical_expr.rs@@ -21,15 +21,17 @@ use crate::expressions::{self, Column};use crate::{LexOrdering, PhysicalSortExpr, create_physical_expr};use arrow::compute::SortOptions;-use arrow::datatypes::{Schema, SchemaRef};+use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};-use datafusion_common::{DFSchema, HashMap};+use datafusion_common::{DFSchema, HashMap, ScalarValue, SplitPoint};use datafusion_common::{Result, plan_err};use datafusion_expr::execution_props::ExecutionProps;-use datafusion_expr::{Expr, SortExpr};+use datafusion_expr::{Expr, Partitioning as LogicalPartitioning, SortExpr};+use datafusion_expr_common::casts::try_cast_literal_to_type;use itertools::izip;// Exports:+use crate::{Partitioning, RangePartitioning};pub(crate) use datafusion_physical_expr_common::physical_expr::PhysicalExpr;/// Adds the `offset` value to `Column` indices inside `expr`. This function is@@ -216,6 +218,99 @@ pub fn create_physical_sort_exprs(.collect()}+/// Create physical partitioning from logical partitioning.+pub fn create_physical_partitioning(+ partitioning: &LogicalPartitioning,+ input_dfschema: &DFSchema,+ execution_props: &ExecutionProps,+) -> Result<Partitioning> {+ match partitioning {+ LogicalPartitioning::RoundRobinBatch(n) => Ok(Partitioning::RoundRobinBatch(*n)),+ LogicalPartitioning::Hash(exprs, partition_count) => {+ let exprs = exprs+ .iter()+ .map(|expr| create_physical_expr(expr, input_dfschema, execution_props))+ .collect::<Result<Vec<_>>>()?;+ Ok(Partitioning::Hash(exprs, *partition_count))+ }+ LogicalPartitioning::Range(range) => {+ let ordering = create_physical_sort_exprs(+ range.ordering(),+ input_dfschema,+ execution_props,+ )?;+ let Some(ordering) = LexOrdering::new(ordering) else {+ return plan_err!("Range partitioning requires non-empty ordering");+ };+ let split_points = normalize_range_split_points(+ &ordering,+ range.split_points(),+ input_dfschema.as_arrow(),+ )?;+ let range = RangePartitioning::try_new(ordering, split_points)?;+ Ok(Partitioning::Range(range))+ }+ LogicalPartitioning::DistributeBy(_) => {+ datafusion_common::not_impl_err!(+ "Physical plan does not support DistributeBy partitioning"+ )+ }+ }+}++fn normalize_range_split_points(+ ordering: &LexOrdering,+ split_points: &[SplitPoint],+ schema: &Schema,+) -> Result<Vec<SplitPoint>> {+ split_points+ .iter()+ .enumerate()+ .map(|(split_idx, split_point)| {+ let values = split_point+ .values()+ .iter()+ .zip(ordering.iter())+ .enumerate()+ .map(|(value_idx, (value, sort_expr))| {+ let target_type = sort_expr.expr.data_type(schema)?;+ normalize_range_split_point_value(+ value,+ &target_type,+ split_idx,+ value_idx,+ )+ })+ .collect::<Result<Vec<_>>>()?;+ Ok(SplitPoint::new(values))+ })+ .collect()+}++fn normalize_range_split_point_value(+ value: &ScalarValue,+ target_type: &DataType,+ split_idx: usize,+ value_idx: usize,+) -> Result<ScalarValue> {+ let value_type = value.data_type();+ if &value_type == target_type {+ return Ok(value.clone());+ }++ if let Some(casted) = try_cast_literal_to_type(value, target_type) {+ // Split points define physical partition boundaries, so normalization+ // must reject casts that would change the advertised boundary.+ if try_cast_literal_to_type(&casted, &value_type).as_ref() == Some(value) {+ return Ok(casted);+ }+ }++ plan_err!(+ "Range output partitioning split point {split_idx} value {value_idx} with type {value_type} cannot be represented exactly as ordering expression type {target_type}"+ )+}+pub fn add_offset_to_physical_sort_exprs(sort_exprs: impl IntoIterator<Item = PhysicalSortExpr>,offset: isize,diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.protoindex 8745100e55..d68973c44e 100644--- a/datafusion/proto-models/proto/datafusion.proto+++ b/datafusion/proto-models/proto/datafusion.proto@@ -1191,6 +1191,7 @@ message FileScanExecConf {optional ProjectionExprs projection_exprs = 13;optional bool partitioned_by_file_group = 14;+ optional Partitioning output_partitioning = 15;}message ParquetScanExecNode {diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rsindex a85f807bc8..f8e2103035 100644--- a/datafusion/proto-models/src/generated/pbjson.rs+++ b/datafusion/proto-models/src/generated/pbjson.rs@@ -6966,6 +6966,9 @@ impl serde::Serialize for FileScanExecConf {if self.partitioned_by_file_group.is_some() {len += 1;}+ if self.output_partitioning.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.FileScanExecConf", len)?;if !self.file_groups.is_empty() {struct_ser.serialize_field("fileGroups", &self.file_groups)?;@@ -7005,6 +7008,9 @@ impl serde::Serialize for FileScanExecConf {if let Some(v) = self.partitioned_by_file_group.as_ref() {struct_ser.serialize_field("partitionedByFileGroup", v)?;}+ if let Some(v) = self.output_partitioning.as_ref() {+ struct_ser.serialize_field("outputPartitioning", v)?;+ }struct_ser.end()}}@@ -7034,6 +7040,8 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {"projectionExprs","partitioned_by_file_group","partitionedByFileGroup",+ "output_partitioning",+ "outputPartitioning",];#[allow(clippy::enum_variant_names)]@@ -7050,6 +7058,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {BatchSize,ProjectionExprs,PartitionedByFileGroup,+ OutputPartitioning,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -7083,6 +7092,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {"batchSize" | "batch_size" => Ok(GeneratedField::BatchSize),"projectionExprs" | "projection_exprs" => Ok(GeneratedField::ProjectionExprs),"partitionedByFileGroup" | "partitioned_by_file_group" => Ok(GeneratedField::PartitionedByFileGroup),+ "outputPartitioning" | "output_partitioning" => Ok(GeneratedField::OutputPartitioning),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -7114,6 +7124,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {let mut batch_size__ = None;let mut projection_exprs__ = None;let mut partitioned_by_file_group__ = None;+ let mut output_partitioning__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::FileGroups => {@@ -7193,6 +7204,12 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {}partitioned_by_file_group__ = map_.next_value()?;}+ GeneratedField::OutputPartitioning => {+ if output_partitioning__.is_some() {+ return Err(serde::de::Error::duplicate_field("outputPartitioning"));+ }+ output_partitioning__ = map_.next_value()?;+ }}}Ok(FileScanExecConf {@@ -7208,6 +7225,7 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf {batch_size: batch_size__,projection_exprs: projection_exprs__,partitioned_by_file_group: partitioned_by_file_group__,+ output_partitioning: output_partitioning__,})}}diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rsindex 4da38881a8..675ead23f4 100644--- a/datafusion/proto-models/src/generated/prost.rs+++ b/datafusion/proto-models/src/generated/prost.rs@@ -1816,6 +1816,8 @@ pub struct FileScanExecConf {pub projection_exprs: ::core::option::Option<ProjectionExprs>,#[prost(bool, optional, tag = "14")]pub partitioned_by_file_group: ::core::option::Option<bool>,+ #[prost(message, optional, tag = "15")]+ pub output_partitioning: ::core::option::Option<Partitioning>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct ParquetScanExecNode {diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 7bd1a3dd66..53ff4a41d4 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -552,6 +552,12 @@ pub fn parse_protobuf_file_scan_config()?;output_ordering.extend(LexOrdering::new(sort_exprs));}+ let output_partitioning = parse_protobuf_partitioning(+ proto.output_partitioning.as_ref(),+ ctx,+ &schema,+ proto_converter,+ )?;// Parse projection expressions if present and apply to file sourcelet file_source = if let Some(proto_projection_exprs) = &proto.projection_exprs {@@ -580,15 +586,18 @@ pub fn parse_protobuf_file_scan_config(file_source};- let config = FileScanConfigBuilder::new(object_store_url, file_source)+ let mut config_builder = FileScanConfigBuilder::new(object_store_url, file_source).with_file_groups(file_groups).with_constraints(constraints).with_statistics(statistics).with_limit(proto.limit.as_ref().map(|sl| sl.limit as usize)).with_output_ordering(output_ordering)- .with_batch_size(proto.batch_size.map(|s| s as usize))- .with_partitioned_by_file_group(proto.partitioned_by_file_group.unwrap_or(false))- .build();+ .with_output_partitioning(output_partitioning)+ .with_batch_size(proto.batch_size.map(|s| s as usize));+ if proto.partitioned_by_file_group.unwrap_or(false) {+ config_builder = config_builder.with_partitioned_by_file_group(true);+ }+ let config = config_builder.build();Ok(config)}diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 7310c0928e..4614c4f002 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -498,6 +498,11 @@ pub fn serialize_file_scan_config(serialize_physical_sort_exprs(order.to_vec(), codec, proto_converter)?;output_orderings.push(ordering)}+ let output_partitioning = conf+ .output_partitioning+ .as_ref()+ .map(|partitioning| serialize_partitioning(partitioning, codec, proto_converter))+ .transpose()?;// Fields must be added to the schema so that they can persist in the protobuf,// and then they are to be removed from the schema in `parse_protobuf_file_scan_config`@@ -558,6 +563,7 @@ pub fn serialize_file_scan_config(batch_size: conf.batch_size.map(|s| s as u64),projection_exprs,partitioned_by_file_group: Some(conf.partitioned_by_file_group),+ output_partitioning,})}diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 8e80467788..2022857d4e 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -4080,10 +4080,26 @@ fn test_custom_node_with_dynamic_filter_dedup_roundtrip() -> Result<()> {Ok(())}+fn roundtrip_file_scan_config(scan_config: FileScanConfig) -> Result<FileScanConfig> {+ let exec_plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let proto_converter = DefaultPhysicalProtoConverter {};+ let result_plan =+ roundtrip_test_and_return(exec_plan, &ctx, &codec, &proto_converter)?;++ let data_source_exec = result_plan+ .downcast_ref::<DataSourceExec>()+ .expect("Expected DataSourceExec");+ let file_scan_config = data_source_exec+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .expect("Expected FileScanConfig");+ Ok(file_scan_config.clone())+}+#[test]fn roundtrip_parquet_exec_partitioned_by_file_group() -> Result<()> {- use datafusion::datasource::physical_plan::FileScanConfig;-let file_schema =Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));@@ -4096,34 +4112,65 @@ fn roundtrip_parquet_exec_partitioned_by_file_group() -> Result<()> {.with_partitioned_by_file_group(true).build();- assert!(scan_config.partitioned_by_file_group);+ assert!(roundtrip_file_scan_config(scan_config)?.partitioned_by_file_group);+ Ok(())+}- let exec_plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(scan_config);+#[test]+fn roundtrip_parquet_exec_output_partitioning() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let output_partitioning =+ Partitioning::Hash(vec![Arc::new(Column::new("col", 0))], 1);+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .with_output_partitioning(Some(output_partitioning.clone()))+ .build();- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan),- &codec,- &proto_converter,- )?;- let result_plan = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &proto_converter,- )?;+ assert_eq!(+ roundtrip_file_scan_config(scan_config)?.output_partitioning,+ Some(output_partitioning)+ );- let data_source_exec = result_plan- .downcast_ref::<DataSourceExec>()- .expect("Expected DataSourceExec");- let file_scan_config = data_source_exec- .data_source()- .downcast_ref::<FileScanConfig>()- .expect("Expected FileScanConfig");+ Ok(())+}++#[test]+fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> {+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)]));+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&file_schema)));+ let output_partitioning = Partitioning::Range(RangePartitioning::new(+ LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new(+ "col", 0,+ )))])+ .unwrap(),+ vec![SplitPoint::new(vec![ScalarValue::Int32(Some(10))])],+ ));+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-1.parquet".to_string(),+ 1024,+ )]),+ FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file-2.parquet".to_string(),+ 1024,+ )]),+ ])+ .with_output_partitioning(Some(output_partitioning.clone()))+ .build();- assert!(file_scan_config.partitioned_by_file_group);+ assert_eq!(+ roundtrip_file_scan_config(scan_config)?.output_partitioning,+ Some(output_partitioning)+ );Ok(())}diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.tomlindex a0c18c9086..13493d16c0 100644--- a/datafusion/sqllogictest/Cargo.toml+++ b/datafusion/sqllogictest/Cargo.toml@@ -47,7 +47,6 @@ bytes = { workspace = true, optional = true }chrono = { workspace = true, optional = true }clap = { version = "4.5.60", features = ["derive", "env"] }datafusion = { workspace = true, default-features = true, features = ["avro"] }-datafusion-datasource = { workspace = true }datafusion-spark = { workspace = true, features = ["core"] }datafusion-substrait = { workspace = true, default-features = true, optional = true }futures = { workspace = true }diff --git a/datafusion/sqllogictest/src/test_context/range_partitioning.rs b/datafusion/sqllogictest/src/test_context/range_partitioning.rsindex 88e49708ba..a3e16eefd8 100644--- a/datafusion/sqllogictest/src/test_context/range_partitioning.rs+++ b/datafusion/sqllogictest/src/test_context/range_partitioning.rs@@ -15,236 +15,94 @@// specific language governing permissions and limitations// under the License.-use std::fmt;+use std::fs::{create_dir_all, remove_dir_all, write};+use std::path::Path;use std::sync::Arc;-use arrow::array::Int32Array;-use arrow::compute::SortOptions;-use arrow::datatypes::{DataType, Field, Schema, SchemaRef};-use arrow::record_batch::RecordBatch;-use async_trait::async_trait;-use datafusion::catalog::Session;-use datafusion::common::{Result, ScalarValue, project_schema};-use datafusion::datasource::source::{DataSource, DataSourceExec};-use datafusion::datasource::{TableProvider, TableType};-use datafusion::execution::context::TaskContext;-use datafusion::logical_expr::Expr;-use datafusion::physical_expr::EquivalenceProperties;-use datafusion::physical_expr::expressions::col as physical_col;-use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr};-use datafusion::physical_plan::execution_plan::SchedulingType;-use datafusion::physical_plan::projection::ProjectionExprs;-use datafusion::physical_plan::{- DisplayFormatType, ExecutionPlan, Partitioning, RangePartitioning,- SendableRecordBatchStream, SplitPoint, Statistics,+use arrow::datatypes::{DataType, Field, Schema};+use datafusion::common::{ScalarValue, SplitPoint};+use datafusion::datasource::file_format::csv::CsvFormat;+use datafusion::datasource::listing::{+ ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,};+use datafusion::logical_expr::{Partitioning, RangePartitioning, col};use datafusion::prelude::SessionContext;-use datafusion_datasource::memory::MemorySourceConfig;// ==============================================================================// Range Partitioned Table (sqllogictest-only)// ==============================================================================-/// Simple range-partitioned table for testing before declaring such tables is-/// supported via SQL.-#[derive(Debug)]-struct RangePartitionedTable {- schema: SchemaRef,- partitions: Vec<Vec<RecordBatch>>,- range_column_index: usize,- split_points: Vec<SplitPoint>,-}--#[async_trait]-impl TableProvider for RangePartitionedTable {- fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)- }-- fn table_type(&self) -> TableType {- TableType::Base- }-- async fn scan(- &self,- state: &dyn Session,- projection: Option<&Vec<usize>>,- _filters: &[Expr],- _limit: Option<usize>,- ) -> Result<Arc<dyn ExecutionPlan>> {- let projected_schema = project_schema(&self.schema, projection)?;- let mut source = MemorySourceConfig::try_new(- &self.partitions,- Arc::clone(&self.schema),- projection.cloned(),- )?;- source = source.with_show_sizes(state.config_options().explain.show_sizes);-- let output_partitioning =- self.output_partitioning(projection, &projected_schema)?;- let source = RangePartitionedSource {- inner: source,- output_partitioning,- };-- Ok(DataSourceExec::from_data_source(source))- }-}--impl RangePartitionedTable {- fn output_partitioning(- &self,- projection: Option<&Vec<usize>>,- projected_schema: &SchemaRef,- ) -> Result<Partitioning> {- let Some(projected_range_index) =- projected_index(self.range_column_index, projection)- else {- return Ok(Partitioning::UnknownPartitioning(self.partitions.len()));- };-- let range_column = projected_schema.field(projected_range_index).name();- let ordering = LexOrdering::new(vec![PhysicalSortExpr::new(- physical_col(range_column, projected_schema)?,- SortOptions::default(),- )])- .expect("range ordering should not be empty");-- Ok(Partitioning::Range(RangePartitioning::try_new(- ordering,- self.split_points.clone(),- )?))- }-}--fn projected_index(- column_index: usize,- projection: Option<&Vec<usize>>,-) -> Option<usize> {- projection- .map(|projection| projection.iter().position(|idx| *idx == column_index))- .unwrap_or(Some(column_index))-}--#[derive(Clone, Debug)]-struct RangePartitionedSource {- inner: MemorySourceConfig,- output_partitioning: Partitioning,-}--impl DataSource for RangePartitionedSource {- fn open(- &self,- partition: usize,- context: Arc<TaskContext>,- ) -> Result<SendableRecordBatchStream> {- self.inner.open(partition, context)- }-- fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {- self.inner.fmt_as(t, f)?;- match t {- DisplayFormatType::Default | DisplayFormatType::Verbose => {- write!(f, ", output_partitioning={}", self.output_partitioning)- }- DisplayFormatType::TreeRender => Ok(()),- }- }-- fn output_partitioning(&self) -> Partitioning {- self.output_partitioning.clone()- }-- fn eq_properties(&self) -> EquivalenceProperties {- self.inner.eq_properties()- }-- fn scheduling_type(&self) -> SchedulingType {- self.inner.scheduling_type()- }-- fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {- self.inner.partition_statistics(partition)- }-- fn with_fetch(&self, limit: Option<usize>) -> Option<Arc<dyn DataSource>> {- Some(Arc::new(Self {- inner: self.inner.clone().with_limit(limit),- output_partitioning: self.output_partitioning.clone(),- }))- }-- fn fetch(&self) -> Option<usize> {- self.inner.fetch()- }-- fn try_swapping_with_projection(- &self,- _projection: &ProjectionExprs,- ) -> Result<Option<Arc<dyn DataSource>>> {- // Range partitioning metadata is projection-sensitive. This fixture- // computes it in TableProvider::scan, so do not rewrite later- // ProjectionExec nodes into the source.- Ok(None)- }-}-+/// Registers a simple range-partitioned listing table for testing before+/// declaring such tables is supported via SQL.pub(super) fn register_range_partitioned_table(ctx: &SessionContext) {let schema = Arc::new(Schema::new(vec![Field::new("range_key", DataType::Int32, false),Field::new("non_range_key", DataType::Int32, false),Field::new("value", DataType::Int32, false),]));- let partitions = vec![- vec![range_partition_batch(&schema, &[1, 5], &[1, 2], &[10, 50])],- vec![range_partition_batch(- &schema,- &[10, 15],- &[1, 2],- &[100, 150],- )],- vec![range_partition_batch(- &schema,- &[20, 25],- &[1, 2],- &[200, 250],- )],- vec![range_partition_batch(- &schema,- &[30, 35],- &[1, 2],- &[300, 350],- )],- ];- let split_points = vec![- SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),- SplitPoint::new(vec![ScalarValue::Int32(Some(20))]),- SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),- ];- let table = RangePartitionedTable {+ let output_partitioning = Partitioning::Range(+ RangePartitioning::try_new(+ vec![col("range_key").sort(true, true)],+ vec![+ SplitPoint::new(vec![ScalarValue::Int32(Some(10))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(20))]),+ SplitPoint::new(vec![ScalarValue::Int32(Some(30))]),+ ],+ )+ .expect("range partitioning should be valid"),+ );++ register_csv_listing_table(+ ctx,+ "range_partitioned",+ Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("test_files/scratch_range_partitioning/range_partitioned"),schema,- partitions,- range_column_index: 0,- split_points,- };-- ctx.register_table("range_partitioned", Arc::new(table))- .expect("range partitioned table registration should succeed");+ [+ "1,1,10\n5,2,50\n",+ "10,1,100\n15,2,150\n",+ "20,1,200\n25,2,250\n",+ "30,1,300\n35,2,350\n",+ ],+ Some(output_partitioning),+ );}-fn range_partition_batch(- schema: &SchemaRef,- range_key: &[i32],- non_range_key: &[i32],- value: &[i32],-) -> RecordBatch {- RecordBatch::try_new(- Arc::clone(schema),- vec![- Arc::new(Int32Array::from(range_key.to_vec())),- Arc::new(Int32Array::from(non_range_key.to_vec())),- Arc::new(Int32Array::from(value.to_vec())),- ],- )- .expect("range partition batch should be valid")+fn register_csv_listing_table(+ ctx: &SessionContext,+ name: &str,+ table_dir: impl AsRef<Path>,+ schema: Arc<Schema>,+ partitions: impl IntoIterator<Item = &'static str>,+ output_partitioning: Option<Partitioning>,+) {+ let table_dir = table_dir.as_ref();+ if table_dir.exists() {+ remove_dir_all(table_dir).expect("test table dir should be removable");+ }+ create_dir_all(table_dir).expect("test table dir should be created");+ for (idx, rows) in partitions.into_iter().enumerate() {+ write(table_dir.join(format!("part-{idx}.csv")), rows)+ .expect("test table csv partition should be written");+ }++ let table_path = format!(+ "{}/",+ table_dir+ .to_str()+ .expect("test table path should be valid utf8")+ );+ let table_url =+ ListingTableUrl::parse(&table_path).expect("test table url should parse");+ let options =+ ListingOptions::new(Arc::new(CsvFormat::default().with_has_header(false)))+ .with_output_partitioning(output_partitioning);+ let config = ListingTableConfig::new(table_url)+ .with_listing_options(options)+ .with_schema(schema);+ let table =+ ListingTable::try_new(config).expect("test listing table should be valid");++ ctx.register_table(name, Arc::new(table))+ .expect("test listing table registration should succeed");}diff --git a/datafusion/sqllogictest/test_files/range_partitioning.slt b/datafusion/sqllogictest/test_files/range_partitioning.sltindex a61f17a039..2b7a2cfdf4 100644--- a/datafusion/sqllogictest/test_files/range_partitioning.slt+++ b/datafusion/sqllogictest/test_files/range_partitioning.slt@@ -16,7 +16,7 @@# under the License.# The sqllogictest harness registers range_partitioned(range_key, non_range_key, value)-# as an in-memory source with four physical source partitions:+# as a CSV ListingTable with four declared range-partitioned file groups:## partition 0: range_key in [..., 10), rows (1, 1, 10), (5, 2, 50)# partition 1: range_key in [10, 20), rows (10, 1, 100), (15, 2, 150)@@ -40,7 +40,7 @@ physical_plan01)AggregateExec: mode=FinalPartitioned, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]02)--RepartitionExec: partitioning=Hash([range_key@0], 4), input_partitions=403)----AggregateExec: mode=Partial, gby=[range_key@0 as range_key], aggr=[sum(range_partitioned.value)]-04)------DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)+04)------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=falsequery IISELECT range_key, SUM(value) FROM range_partitioned GROUP BY range_key ORDER BY range_key;@@ -69,7 +69,7 @@ physical_plan01)AggregateExec: mode=FinalPartitioned, gby=[non_range_key@0 as non_range_key], aggr=[sum(range_partitioned.value)]02)--RepartitionExec: partitioning=Hash([non_range_key@0], 4), input_partitions=403)----AggregateExec: mode=Partial, gby=[non_range_key@0 as non_range_key], aggr=[sum(range_partitioned.value)]-04)------DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1], output_partitioning=UnknownPartitioning(4)+04)------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[non_range_key, value], output_partitioning=UnknownPartitioning(4), file_type=csv, has_header=falsequery IISELECT non_range_key, SUM(value) FROM range_partitioned GROUP BY non_range_key ORDER BY non_range_key;@@ -104,8 +104,8 @@ SELECT range_key, value FROM range_partitioned;----physical_plan01)UnionExec-02)--DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)-03)--DataSourceExec: partitions=4, partition_sizes=[1, 1, 1, 1], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4)+02)--DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=false+03)--DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-0.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-2.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch_range_partitioning/range_partitioned/part-3.csv]]}, projection=[range_key, value], output_partitioning=Range([range_key@0 ASC], [(10), (20), (30)], 4), file_type=csv, has_header=falsequery IISELECT range_key, value FROM range_partitioned