Apache DataFusion SQL Query Engine
perf: prune ListingTable files before grouping (#26441)
Rishabh Guptafd101e99c21e
diff --git a/Cargo.lock b/Cargo.lockindex 7f8b942657..e3bfa7f595 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1865,6 +1865,7 @@ dependencies = ["datafusion-physical-expr-adapter","datafusion-physical-expr-common","datafusion-physical-plan",+ "datafusion-pruning","futures","itertools 0.15.0","log",diff --git a/datafusion/catalog-listing/Cargo.toml b/datafusion/catalog-listing/Cargo.tomlindex a71791d430..d375e6c922 100644--- a/datafusion/catalog-listing/Cargo.toml+++ b/datafusion/catalog-listing/Cargo.toml@@ -29,6 +29,7 @@ datafusion-physical-expr = { workspace = true }datafusion-physical-expr-adapter = { workspace = true }datafusion-physical-expr-common = { workspace = true }datafusion-physical-plan = { workspace = true }+datafusion-pruning = { workspace = true }futures = { workspace = true }itertools = { workspace = true }log = { workspace = true }diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex a1c3f16d4e..6e3dfe0691 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -10,6 +10,7 @@ use crate::{ListingOptions, ListingTableConfig};use arrow::datatypes::{Field, Metadata, Schema, SchemaBuilder, SchemaRef};use async_trait::async_trait;use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};+use datafusion_common::pruning::PrunableStatistics;use datafusion_common::stats::{Precision, is_known_empty};use datafusion_common::{Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err,@@ -41,6 +42,7 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::empty::EmptyExec;use datafusion_physical_plan::limit::GlobalLimitExec;+use datafusion_pruning::PruningPredicateBuilder;use futures::future::BoxFuture;use futures::{Stream, StreamExt, TryStreamExt, future, stream};use object_store::ObjectStore;@@ -662,7 +664,12 @@ impl ListingTable {grouped_by_partition: partitioned_by_file_group,partition_split_points,} = self- .list_files_for_scan(state, &partition_filters, statistic_file_limit)+ .list_files_for_scan_inner(+ state,+ &partition_filters,+ &filters,+ statistic_file_limit,+ ).await?;// if no files need to be read, return an `EmptyExec`@@ -915,15 +922,28 @@ impl ListingTable {filters: &'a [Expr],limit: Option<usize>,) -> datafusion_common::Result<ListFilesResult> {+ self.list_files_for_scan_inner(ctx, filters, &[], limit)+ .await+ }++ async fn list_files_for_scan_inner<'a>(+ &'a self,+ ctx: &'a dyn Session,+ partition_filters: &'a [Expr],+ data_filters: &'a [Expr],+ limit: Option<usize>,+ ) -> datafusion_common::Result<ListFilesResult> {+ // Declared layouts must retain their partition identities, including empty groups.if let Some(output_partitioning) = self.options.output_partitioning.as_ref() {self.list_files_for_declared_output_partitioning(ctx,output_partitioning,- filters,+ partition_filters,).await} else {- self.list_files_for_regular_scan(ctx, filters, limit).await+ self.list_files_for_regular_scan(ctx, partition_filters, data_filters, limit)+ .await}}@@ -979,10 +999,51 @@ impl ListingTable {get_files_with_limit(files, file_limit, ctx.config().collect_statistics()).await}+ /// Build a predicate that can remove files whose collected statistics prove+ /// the filter cannot match. This performs no I/O and leaves the normal row+ /// filters in place.+ fn file_statistics_pruning_predicate(+ &self,+ ctx: &dyn Session,+ filters: &[Expr],+ ) -> Option<Arc<datafusion_pruning::PruningPredicate>> {+ if !ctx.config().collect_statistics() || self.expr_adapter_factory.is_some() {+ return None;+ }+ // Custom adapters can change values relative to file statistics, and volatile+ // expressions must not be evaluated while planning.+ let filter = filters+ .iter()+ .filter(|f| !f.is_volatile())+ .cloned()+ .reduce(Expr::and)?;+ let Ok(schema) = DFSchema::try_from(self.table_schema.as_ref().clone()) else {+ return None;+ };+ let Ok(predicate) = ctx.create_physical_expr(filter, &schema) else {+ return None;+ };+ PruningPredicateBuilder::new()+ .with_file_schema(Arc::clone(&self.table_schema))+ .build(predicate)+ }++ fn prune_files_by_statistics(+ &self,+ predicate: Option<&datafusion_pruning::PruningPredicate>,+ files: FileGroup,+ ) -> FileGroup {+ let Some(predicate) = predicate else {+ return files;+ };+ prune_collected_files(predicate, &self.file_schema, files)+ }+async fn list_files_for_regular_scan<'a>(&'a self,ctx: &'a dyn Session,filters: &'a [Expr],+ data_filters: &'a [Expr],limit: Option<usize>,) -> datafusion_common::Result<ListFilesResult> {let file_group_count = ctx.config().target_partitions();@@ -1005,6 +1066,7 @@ impl ListingTable {let (file_group, inexact_stats) = self.collect_files_for_scan(ctx, &store, filters, limit).await?;+ let pruning_predicate = self.file_statistics_pruning_predicate(ctx, data_filters);// Threshold: 0 = disabled, N > 0 = enabled when distinct_keys >= N//@@ -1018,17 +1080,30 @@ impl ListingTable {let (grouped, split_points) = file_group.group_by_partition_values_with_split_points(file_group_count);if grouped.len() >= threshold {+ // Preserve the groups and split points derived from all files so+ // pruning cannot change the declared range partitioning.+ let grouped = grouped+ .into_iter()+ .map(|group| {+ self.prune_files_by_statistics(+ pruning_predicate.as_deref(),+ group,+ )+ })+ .collect();(grouped, true, split_points)} else {let all_files: Vec<_> =grouped.into_iter().flat_map(|g| g.into_inner()).collect();- (- FileGroup::new(all_files).split_files(file_group_count),- false,- vec![],- )+ let file_group = self.prune_files_by_statistics(+ pruning_predicate.as_deref(),+ FileGroup::new(all_files),+ );+ (file_group.split_files(file_group_count), false, vec![])}} else {+ let file_group = self+ .prune_files_by_statistics(pruning_predicate.as_deref(), file_group);(file_group.split_files(file_group_count), false, vec![])};@@ -1260,6 +1335,39 @@ async fn get_files_with_limit(Ok((file_group, inexact_stats))}+/// Keep a file unless the pruning predicate proves it cannot contain a match.+fn prune_collected_files(+ predicate: &datafusion_pruning::PruningPredicate,+ file_schema: &SchemaRef,+ files: FileGroup,+) -> FileGroup {+ FileGroup::new(+ files+ .into_inner()+ .into_iter()+ .filter(|file| {+ // Same check as `FilePruner::try_new`: all-`Absent` statistics+ // cannot prune, so skip building `PrunableStatistics` for them.+ if !file.has_statistics() {+ return true;+ }+ let Some(statistics) = &file.statistics else {+ return true;+ };+ // One unknown file must not hide the exact bounds of another file.+ let statistics = PrunableStatistics::new(+ vec![Arc::clone(statistics)],+ Arc::clone(file_schema),+ );+ predicate+ .prune(&statistics)+ .map(|keep| keep != [false])+ .unwrap_or(true)+ })+ .collect(),+ )+}+#[cfg(test)]mod tests {use super::*;@@ -1421,4 +1529,137 @@ mod tests {let result = derive_common_ordering_from_files(&file_groups);assert_eq!(result, Some(ordering));}++ fn prune_test_files(expr: Expr, statistics: Vec<Option<Statistics>>) -> Vec<String> {+ use arrow::datatypes::DataType;+ let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));+ let physical = datafusion_physical_expr::create_physical_expr(+ &expr,+ &DFSchema::try_from(schema.as_ref().clone()).unwrap(),+ &ExecutionProps::new(),+ &PhysicalPlanningContext::default(),+ )+ .unwrap();+ let predicate = PruningPredicateBuilder::new()+ .with_file_schema(Arc::clone(&schema))+ .try_build(physical)+ .unwrap();+ let files = FileGroup::new(+ statistics+ .into_iter()+ .enumerate()+ .map(|(i, stats)| {+ let mut file = PartitionedFile::new(format!("f{i}"), 10);+ file.statistics = stats.map(Arc::new);+ file+ })+ .collect(),+ );+ prune_collected_files(&predicate, &schema, files)+ .iter()+ .map(|f| f.object_meta.location.to_string())+ .collect()+ }++ fn bounds(+ min: Precision<datafusion_common::ScalarValue>,+ max: Precision<datafusion_common::ScalarValue>,+ ) -> Statistics {+ Statistics {+ num_rows: Precision::Exact(10),+ column_statistics: vec![datafusion_common::ColumnStatistics {+ min_value: min,+ max_value: max,+ ..Default::default()+ }],+ ..Default::default()+ }+ }++ #[test]+ fn statistics_pruning_keeps_missing_and_inexact_bounds() {+ use datafusion_expr::{col, lit};+ let exact = |v: i64| Precision::Exact(v.into());+ let inexact = |v: i64| Precision::Inexact(v.into());+ assert_eq!(+ prune_test_files(+ col("d").gt_eq(lit(20_i64)),+ vec![+ None,+ Some(bounds(Precision::Absent, Precision::Absent)),+ Some(bounds(inexact(0), inexact(9))),+ Some(bounds(exact(0), exact(9))),+ Some(bounds(exact(20), exact(29)))+ ]+ ),+ ["f0", "f1", "f2", "f4"]+ );+ }++ #[test]+ fn statistics_pruning_uses_independent_bounds() {+ use datafusion_expr::{col, lit};+ assert_eq!(+ prune_test_files(+ col("d").lt(lit(20_i64)),+ vec![+ Some(bounds(Precision::Exact(30_i64.into()), Precision::Absent)),+ Some(bounds(Precision::Absent, Precision::Exact(10_i64.into())))+ ]+ ),+ ["f1"]+ );+ }++ #[test]+ fn statistics_pruning_preserves_null_semantics() {+ use datafusion_expr::{col, lit};+ let mut all_null = bounds(Precision::Absent, Precision::Absent);+ all_null.column_statistics[0].null_count = Precision::Exact(10);+ let mut no_null = bounds(+ Precision::Exact(0_i64.into()),+ Precision::Exact(9_i64.into()),+ );+ no_null.column_statistics[0].null_count = Precision::Exact(0);+ assert_eq!(+ prune_test_files(+ col("d").is_null(),+ vec![Some(all_null.clone()), Some(no_null.clone())]+ ),+ ["f0"]+ );+ assert_eq!(+ prune_test_files(+ col("d").is_not_null(),+ vec![Some(all_null.clone()), Some(no_null.clone())]+ ),+ ["f1"]+ );+ assert!(+ prune_test_files(col("d").eq(lit(5_i64)), vec![Some(all_null)]).is_empty()+ );+ assert_eq!(+ prune_test_files(+ col("d").eq(lit(5_i64)).or(col("d").is_null()),+ vec![Some(no_null)]+ ),+ ["f0"]+ );+ }++ #[test]+ fn statistics_pruning_keeps_file_on_evaluation_error() {+ use datafusion_common::ScalarValue;+ use datafusion_expr::{col, lit};+ // Binary bounds cannot be cast to Int64, so `prune` returns an error.+ // A Utf8 bound does not error, so it would not reach that branch.+ let binary = || Precision::Exact(ScalarValue::Binary(Some(vec![1])));+ assert_eq!(+ prune_test_files(+ col("d").gt(lit(10_i64)),+ vec![Some(bounds(binary(), binary()))]+ ),+ ["f0"]+ );+ }}diff --git a/datafusion/core/tests/catalog_listing/file_statistics_pruning.rs b/datafusion/core/tests/catalog_listing/file_statistics_pruning.rsnew file mode 100644index 0000000000..f672606afe--- /dev/null+++ b/datafusion/core/tests/catalog_listing/file_statistics_pruning.rs@@ -0,0 +1,429 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++use arrow::array::{Int64Array, RecordBatch};+use arrow::datatypes::{DataType, Field, Schema};+use datafusion::prelude::{SessionConfig, SessionContext};+use datafusion_catalog::TableProvider;+use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};+use datafusion_common::{Result, stats::Precision};+use datafusion_datasource::{+ ListingTableUrl, file_scan_config::FileScanConfig, source::DataSourceExec,+};+use datafusion_datasource_parquet::ParquetFormat;+use datafusion_expr::{Expr, Partitioning, col, lit};+use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;+use datafusion_physical_plan::{ExecutionPlan, empty::EmptyExec};+use parquet::arrow::ArrowWriter;+use parquet::file::properties::{EnabledStatistics, WriterProperties};+use std::fs::File;+use std::sync::Arc;+use tempfile::TempDir;++fn fixture(values: Vec<Vec<Option<i64>>>, hive: bool, statistics: bool) -> TempDir {+ let dir = tempfile::tempdir().unwrap();+ let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));+ for (i, values) in values.into_iter().enumerate() {+ let parent = if hive {+ dir.path().join(format!("p={}", i % 2))+ } else {+ dir.path().to_owned()+ };+ std::fs::create_dir_all(&parent).unwrap();+ let props = WriterProperties::builder()+ .set_statistics_enabled(if statistics {+ EnabledStatistics::Chunk+ } else {+ EnabledStatistics::None+ })+ .build();+ let mut writer = ArrowWriter::try_new(+ File::create(parent.join(format!("f{i}.parquet"))).unwrap(),+ Arc::clone(&schema),+ Some(props),+ )+ .unwrap();+ writer+ .write(+ &RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int64Array::from(values))],+ )+ .unwrap(),+ )+ .unwrap();+ writer.close().unwrap();+ }+ dir+}+fn eight_files() -> TempDir {+ fixture(+ (0..8)+ .map(|i| (i * 10..i * 10 + 10).map(Some).collect())+ .collect(),+ false,+ true,+ )+}+fn table(+ dir: &TempDir,+ partitioning: Option<Partitioning>,+ hive: bool,+ adapter: bool,+) -> ListingTable {+ let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));+ let mut options = ListingOptions::new(Arc::new(ParquetFormat::default()))+ .with_file_extension(".parquet")+ .with_output_partitioning(partitioning);+ if hive {+ options = options.with_table_partition_cols(vec![("p".into(), DataType::Int64)]);+ }+ let mut config = ListingTableConfig::new(+ ListingTableUrl::parse(dir.path().to_str().unwrap()).unwrap(),+ )+ .with_listing_options(options)+ .with_schema(schema);+ if adapter {+ config =+ config.with_expr_adapter_factory(Arc::new(DefaultPhysicalExprAdapterFactory));+ }+ ListingTable::try_new(config).unwrap()+}+fn context(statistics: bool) -> SessionContext {+ SessionContext::new_with_config(+ SessionConfig::new()+ .with_target_partitions(4)+ .with_collect_statistics(statistics),+ )+}++fn scan_config(plan: &Arc<dyn ExecutionPlan>) -> &FileScanConfig {+ plan.downcast_ref::<DataSourceExec>()+ .unwrap()+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .unwrap()+}++fn names(config: &FileScanConfig) -> Vec<Vec<String>> {+ config+ .file_groups+ .iter()+ .map(|g| {+ g.iter()+ .map(|f| f.object_meta.location.filename().unwrap().to_string())+ .collect()+ })+ .collect()+}++fn predicate() -> Expr {+ col("d").gt_eq(lit(20_i64)).and(col("d").lt(lit(40_i64)))+}++async fn rows(ctx: &SessionContext, query: &str) -> Result<usize> {+ Ok(ctx+ .sql(query)+ .await?+ .collect()+ .await?+ .iter()+ .map(RecordBatch::num_rows)+ .sum())+}++#[tokio::test]+async fn prunes_before_grouping_and_aggregates_survivor_statistics() -> Result<()> {+ let dir = eight_files();+ let ctx = context(true);+ let table = table(&dir, None, false, false);+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ let config = scan_config(&plan);+ assert_eq!(names(config), vec![vec!["f2.parquet"], vec!["f3.parquet"]]);+ assert_eq!(config.statistics().num_rows, Precision::Exact(20));+ assert_eq!(+ config.statistics().column_statistics[0].min_value,+ Precision::Exact(20_i64.into())+ );+ assert_eq!(+ config.statistics().column_statistics[0].max_value,+ Precision::Exact(39_i64.into())+ );+ ctx.register_table("t", Arc::new(table))?;+ let optimized = ctx+ .sql("select * from t where d >= 20 and d < 40")+ .await?+ .create_physical_plan()+ .await?;+ let filtered_statistics =+ datafusion_physical_plan::statistics::StatisticsContext::new().compute(+ optimized.as_ref(),+ &datafusion_physical_plan::statistics::StatisticsArgs::new(),+ )?;+ assert!(+ matches!(filtered_statistics.num_rows, Precision::Inexact(20)),+ "{filtered_statistics:?}"+ );+ assert_eq!(+ rows(&ctx, "select * from t where d >= 20 and d < 40").await?,+ 20+ );+ assert_eq!(+ rows(&ctx, "select * from t where d >= 25 and d < 35").await?,+ 10+ );+ Ok(())+}++#[tokio::test]+async fn disabled_statistics_keep_all_files() -> Result<()> {+ let dir = eight_files();+ let ctx = context(false);+ let table = table(&dir, None, false, false);+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ assert_eq!(+ scan_config(&plan)+ .file_groups+ .iter()+ .map(|g| g.len())+ .sum::<usize>(),+ 8+ );+ ctx.register_table("t", Arc::new(table))?;+ assert_eq!(+ rows(&ctx, "select * from t where d >= 20 and d < 40").await?,+ 20+ );+ Ok(())+}++#[tokio::test]+async fn missing_statistics_keep_files() -> Result<()> {+ let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, false);+ let ctx = context(true);+ let table = table(&dir, None, false, false);+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ assert_eq!(+ scan_config(&plan)+ .file_groups+ .iter()+ .map(|g| g.len())+ .sum::<usize>(),+ 2+ );+ ctx.register_table("t", Arc::new(table))?;+ assert_eq!(+ rows(&ctx, "select * from t where d >= 20 and d < 40").await?,+ 1+ );+ Ok(())+}++#[tokio::test]+async fn null_semantics_and_empty_scan() -> Result<()> {+ let dir = fixture(+ vec![+ vec![None, None],+ vec![Some(5), Some(6)],+ vec![None, Some(25)],+ ],+ false,+ true,+ );+ let ctx = context(true);+ let table = table(&dir, None, false, false);+ let plan = table+ .scan(&ctx.state(), None, &[col("d").gt(lit(100_i64))], None)+ .await?;+ assert!(plan.downcast_ref::<EmptyExec>().is_some());+ ctx.register_table("t", Arc::new(table))?;+ assert_eq!(rows(&ctx, "select * from t where d is null").await?, 3);+ assert_eq!(rows(&ctx, "select * from t where d is not null").await?, 3);+ assert_eq!(+ rows(&ctx, "select * from t where d >= 20 or d is null").await?,+ 4+ );+ assert_eq!(+ rows(&ctx, "select * from t where d not in (5, NULL)").await?,+ 0+ );+ Ok(())+}++#[tokio::test]+async fn declared_partitioning_keeps_groups_and_empty_trailing_groups() -> Result<()> {+ let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, true);+ let ctx = context(true);+ let table = table(+ &dir,+ Some(Partitioning::Hash(vec![col("d")], 4)),+ false,+ false,+ );+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ assert_eq!(+ names(scan_config(&plan)),+ vec![vec!["f0.parquet"], vec!["f1.parquet"], vec![], vec![]]+ );+ Ok(())+}++#[tokio::test]+async fn limits_offsets_and_projection_keep_row_filter() -> Result<()> {+ let dir = eight_files();+ let ctx = context(true);+ let table = table(&dir, None, false, false);+ ctx.register_table("t", Arc::new(table))?;+ assert_eq!(+ rows(+ &ctx,+ "select d from t where d >= 25 and d < 35 limit 3 offset 2"+ )+ .await?,+ 3+ );+ let batches = ctx+ .sql("select d from t where d >= 25 and d < 35 order by d limit 3 offset 2")+ .await?+ .collect()+ .await?;+ let values: Vec<_> = batches+ .iter()+ .flat_map(|b| {+ b.column(0)+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap()+ .values()+ .iter()+ .copied()+ })+ .collect();+ assert_eq!(values, [27, 28, 29]);+ Ok(())+}++#[tokio::test]+async fn custom_adapter_disables_statistics_pruning() -> Result<()> {+ let dir = eight_files();+ let ctx = context(true);+ let table = table(&dir, None, false, true);+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ assert_eq!(+ scan_config(&plan)+ .file_groups+ .iter()+ .map(|g| g.len())+ .sum::<usize>(),+ 8+ );+ Ok(())+}++#[tokio::test]+async fn hive_and_data_predicates_preserve_partition_groups() -> Result<()> {+ let dir = fixture(+ (0..8)+ .map(|i| (i * 10..i * 10 + 10).map(Some).collect())+ .collect(),+ true,+ true,+ );+ let mut config = SessionConfig::new()+ .with_target_partitions(4)+ .with_collect_statistics(true);+ config.options_mut().optimizer.preserve_file_partitions = 1;+ let ctx = SessionContext::new_with_config(config);+ let table = table(&dir, None, true, false);+ let filters = [col("p").eq(lit(0_i64)), predicate()];+ let plan = table.scan(&ctx.state(), None, &filters, None).await?;+ assert_eq!(names(scan_config(&plan)), vec![vec!["f2.parquet"]]);+ ctx.register_table("t", Arc::new(table))?;+ assert_eq!(+ rows(&ctx, "select * from t where p = 0 and d >= 20 and d < 40").await?,+ 10+ );+ assert_eq!(rows(&ctx, "select * from t where p = d").await?, 1);+ Ok(())+}++#[tokio::test]+async fn preserve_partitions_below_threshold_still_prunes() -> Result<()> {+ let dir = fixture(+ (0..8)+ .map(|i| (i * 10..i * 10 + 10).map(Some).collect())+ .collect(),+ true,+ true,+ );+ let mut config = SessionConfig::new()+ .with_target_partitions(4)+ .with_collect_statistics(true);+ // Two distinct `p` values are below the threshold, so files are split normally.+ config.options_mut().optimizer.preserve_file_partitions = 3;+ let ctx = SessionContext::new_with_config(config);+ let table = table(&dir, None, true, false);+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ assert_eq!(+ names(scan_config(&plan)),+ vec![vec!["f2.parquet"], vec!["f3.parquet"]]+ );+ Ok(())+}++#[tokio::test]+async fn volatile_predicate_is_not_evaluated_while_listing() -> Result<()> {+ let dir = eight_files();+ let ctx = context(true);+ let table = table(&dir, None, false, false);+ use datafusion_expr::{ColumnarValue, Volatility, create_udf};+ use std::sync::atomic::{AtomicUsize, Ordering};+ let calls = Arc::new(AtomicUsize::new(0));+ let observed = Arc::clone(&calls);+ let udf = create_udf(+ "volatile_threshold",+ vec![],+ DataType::Int64,+ Volatility::Volatile,+ Arc::new(move |_| {+ observed.fetch_add(1, Ordering::SeqCst);+ Ok(ColumnarValue::Scalar(1000_i64.into()))+ }),+ );+ let filters = [col("d").gt(udf.call(vec![]))];+ let plan = table.scan(&ctx.state(), None, &filters, None).await?;+ assert_eq!(+ scan_config(&plan)+ .file_groups+ .iter()+ .map(|g| g.len())+ .sum::<usize>(),+ 8+ );+ assert_eq!(calls.load(Ordering::SeqCst), 0);+ Ok(())+}++#[tokio::test]+async fn declared_range_partitioning_preserves_groups() -> Result<()> {+ use datafusion_common::SplitPoint;+ use datafusion_expr::RangePartitioning;+ let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, true);+ let ctx = context(true);+ let partitioning = Partitioning::Range(RangePartitioning::try_new(+ vec![col("d").sort(true, true)],+ vec![+ SplitPoint::new(vec![10_i64.into()]),+ SplitPoint::new(vec![30_i64.into()]),+ SplitPoint::new(vec![40_i64.into()]),+ ],+ )?);+ let table = table(&dir, Some(partitioning), false, false);+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;+ assert_eq!(+ names(scan_config(&plan)),+ vec![vec!["f0.parquet"], vec!["f1.parquet"], vec![], vec![]]+ );+ Ok(())+}diff --git a/datafusion/core/tests/catalog_listing/mod.rs b/datafusion/core/tests/catalog_listing/mod.rsindex d07fa3e8b8..49c0df520b 100644--- a/datafusion/core/tests/catalog_listing/mod.rs+++ b/datafusion/core/tests/catalog_listing/mod.rs@@ -3,3 +3,5 @@// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreementsmod pruned_partition_list;++mod file_statistics_pruning;diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rsindex f89879709a..3e4e1ce8af 100644--- a/datafusion/core/tests/datasource/object_store_access.rs+++ b/datafusion/core/tests/datasource/object_store_access.rs@@ -1642,3 +1642,50 @@ impl ObjectStore for RequestCountingObjectStore {unimplemented!()}}++#[tokio::test]+async fn listing_statistics_pruning_adds_no_object_store_requests() {+ let unfiltered = Test::new().with_single_file_parquet().await;+ unfiltered+ .session_context+ .sql("SET datafusion.execution.collect_statistics = true")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ unfiltered.object_store.clear_requests();+ unfiltered+ .session_context+ .sql("select * from parquet_table")+ .await+ .unwrap()+ .create_physical_plan()+ .await+ .unwrap();+ let baseline = unfiltered.requests();+ let filtered = Test::new().with_single_file_parquet().await;+ filtered+ .session_context+ .sql("SET datafusion.execution.collect_statistics = true")+ .await+ .unwrap()+ .collect()+ .await+ .unwrap();+ filtered.object_store.clear_requests();+ let plan = filtered+ .session_context+ .sql("select * from parquet_table where a > 1000")+ .await+ .unwrap()+ .create_physical_plan()+ .await+ .unwrap();+ let display = datafusion_physical_plan::displayable(plan.as_ref())+ .indent(true)+ .to_string();+ assert!(display.contains("EmptyExec"), "{display}");+ assert!(!display.contains("DataSourceExec"), "{display}");+ assert_eq!(filtered.requests(), baseline);+}diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rsindex 26d0829944..54482eaa5d 100644--- a/datafusion/core/tests/parquet/mod.rs+++ b/datafusion/core/tests/parquet/mod.rs@@ -416,7 +416,7 @@ impl ContextWithParquet {// find the parquet metricslet parquet_metrics =- MetricsFinder::find_metrics(physical_plan.as_ref()).unwrap();+ MetricsFinder::find_metrics(physical_plan.as_ref()).unwrap_or_default();let result_rows = results.iter().map(|b| b.num_rows()).sum();diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rsindex b002d1023f..f826a54dc0 100644--- a/datafusion/core/tests/parquet/row_group_pruning.rs+++ b/datafusion/core/tests/parquet/row_group_pruning.rs@@ -569,12 +569,12 @@ macro_rules! int_tests {RowGroupPruningTest::new().with_scenario(Scenario::Int).with_query(&format!("SELECT * FROM t where i{} in (100)", $bits))- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(1))- .with_matched_by_bloom_filter(Some(0))- .with_pruned_by_bloom_filter(Some(0))+ .with_expected_errors(None)+ .with_matched_by_stats(None)+ .with_pruned_by_stats(None)+ .with_pruned_files(None)+ .with_matched_by_bloom_filter(None)+ .with_pruned_by_bloom_filter(None).with_expected_rows(0).test_row_group_prune().await;@@ -1660,13 +1660,13 @@ async fn test_row_group_with_null_values() {RowGroupPruningTest::new().with_scenario(Scenario::WithNullValues).with_query("SELECT * FROM t WHERE \"i32\" > 7")- .with_expected_errors(Some(0))- .with_matched_by_stats(Some(0))- .with_pruned_by_stats(Some(0))- .with_pruned_files(Some(1))+ .with_expected_errors(None)+ .with_matched_by_stats(None)+ .with_pruned_by_stats(None)+ .with_pruned_files(None).with_expected_rows(0)- .with_matched_by_bloom_filter(Some(0))- .with_pruned_by_bloom_filter(Some(0))+ .with_matched_by_bloom_filter(None)+ .with_pruned_by_bloom_filter(None).test_row_group_prune().await;}diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rsindex 77c09afd16..8fe4519eba 100644--- a/datafusion/core/tests/sql/explain_analyze.rs+++ b/datafusion/core/tests/sql/explain_analyze.rs@@ -309,7 +309,12 @@ async fn explain_analyze_parquet_pruning_metrics() {let expected_metrics =format!("files_ranges_pruned_statistics={expected_pruning_metrics}");- assert_metrics!(&plan, "DataSourceExec", &expected_metrics);+ if l_orderkey == 10 {+ assert!(plan.contains("EmptyExec"), "{plan}");+ assert!(!plan.contains("DataSourceExec"), "{plan}");+ } else {+ assert_metrics!(&plan, "DataSourceExec", &expected_metrics);+ }}}diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.sltindex d89880ca33..fcebbf2850 100644--- a/datafusion/sqllogictest/test_files/clickbench.slt+++ b/datafusion/sqllogictest/test_files/clickbench.slt@@ -65,6 +65,9 @@ SELECT COUNT(*) FROM hits;10## Q1+statement ok+set datafusion.execution.collect_statistics = false;+query TTEXPLAIN SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0;----@@ -89,6 +92,9 @@ SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0;----0+statement ok+reset datafusion.execution.collect_statistics;+query TTEXPLAIN SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits;----@@ -193,6 +199,9 @@ SELECT MIN("EventDate"), MAX("EventDate") FROM hits;2013-07-15 2013-07-15## Q7+statement ok+set datafusion.execution.collect_statistics = false;+query TTEXPLAIN SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;----@@ -218,6 +227,9 @@ query IISELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;----+statement ok+reset datafusion.execution.collect_statistics;+## Q8query TTEXPLAIN SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10;@@ -970,6 +982,9 @@ SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c1615432634 1615432633 1615432632 1615432631 1## Q36+statement ok+set datafusion.execution.collect_statistics = false;+query TTEXPLAIN SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10;----@@ -1169,6 +1184,9 @@ query PISELECT DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000;----+statement ok+reset datafusion.execution.collect_statistics;+# Clickbench "Extended" queries that test count distinctquery TTdiff --git a/datafusion/sqllogictest/test_files/grouping_set_repartition.slt b/datafusion/sqllogictest/test_files/grouping_set_repartition.sltindex 1c1e509ae0..e78c79a28f 100644--- a/datafusion/sqllogictest/test_files/grouping_set_repartition.slt+++ b/datafusion/sqllogictest/test_files/grouping_set_repartition.slt@@ -138,19 +138,22 @@ physical_plan10)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=411)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]12)----------------------FilterExec: channel@0 = store, projection=[brand@1, amount@2]-13)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = store, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= store AND store <= channel_max@1, required_guarantees=[channel in (store)]-14)--------------ProjectionExec: expr=[web as channel, brand@0 as brand, sum(sales.amount)@1 as total]-15)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]-16)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4-17)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]-18)----------------------FilterExec: channel@0 = web, projection=[brand@1, amount@2]-19)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = web, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= web AND web <= channel_max@1, required_guarantees=[channel in (web)]-20)--------------ProjectionExec: expr=[catalog as channel, brand@0 as brand, sum(sales.amount)@1 as total]-21)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]-22)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4-23)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]-24)----------------------FilterExec: channel@0 = catalog, projection=[brand@1, amount@2]-25)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = catalog, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= catalog AND catalog <= channel_max@1, required_guarantees=[channel in (catalog)]+13)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3+14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = store, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= store AND store <= channel_max@1, required_guarantees=[channel in (store)]+15)--------------ProjectionExec: expr=[web as channel, brand@0 as brand, sum(sales.amount)@1 as total]+16)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]+17)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4+18)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]+19)----------------------FilterExec: channel@0 = web, projection=[brand@1, amount@2]+20)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2+21)--------------------------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = web, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= web AND web <= channel_max@1, required_guarantees=[channel in (web)]+22)--------------ProjectionExec: expr=[catalog as channel, brand@0 as brand, sum(sales.amount)@1 as total]+23)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]+24)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4+25)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]+26)----------------------FilterExec: channel@0 = catalog, projection=[brand@1, amount@2]+27)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2+28)--------------------------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = catalog, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= catalog AND catalog <= channel_max@1, required_guarantees=[channel in (catalog)]query TTI rowsortSELECT channel, brand, SUM(total) as grand_totaldiff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.sltindex 341b2784a5..767a5410a7 100644--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt@@ -361,9 +361,8 @@ logical_plan03)----Filter: t_pushdown.b > Int32(2) AND t_pushdown.a IS NOT NULL04)------TableScan: t_pushdown projection=[a, b], partial_filters=[t_pushdown.b > Int32(2), t_pushdown.a IS NOT NULL]physical_plan-01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]-02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]query I@@ -380,9 +379,8 @@ logical_plan03)----Filter: t_pushdown.a = Utf8View("bar")04)------TableScan: t_pushdown projection=[a, b], partial_filters=[t_pushdown.a = Utf8View("bar")]physical_plan-01)SortPreservingMergeExec: [b@0 ASC NULLS LAST]-02)--SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, sort_order_for_reorder=[b@0 ASC NULLS LAST], pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]+01)SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, sort_order_for_reorder=[b@0 ASC NULLS LAST], pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]# should not push down volatile predicates such as RANDOMdiff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex 208eabb832..3911180ec6 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -46,9 +46,9 @@ query TTEXPLAIN SELECT * FROM test_table WHERE column1 = 1;----physical_plan-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]# cleanupstatement ok@@ -71,9 +71,9 @@ query TTEXPLAIN SELECT * FROM test_table WHERE column1 = 1;----physical_plan-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]# cleanupstatement okdiff --git a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt b/datafusion/sqllogictest/test_files/preserve_file_partitioning.sltindex 209d050efc..21bef2bedf 100644--- a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt+++ b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt@@ -926,6 +926,163 @@ A 7 110.342857142857B 7 87.771428571429C 7 288.228571428571+##########+# TEST 18: File-statistics pruning preserves partition-value groups+# Both sides start with values A-D and split points [(B), (C)]. Pruning B from+# either side must leave an empty middle group without adding a repartition.+##########++query I+COPY (SELECT CAST(10 AS BIGINT) AS value)+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=A/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT CAST(20 AS BIGINT) AS value)+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=B/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT CAST(30 AS BIGINT) AS value)+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=C/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT CAST(40 AS BIGINT) AS value)+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=D/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT 'log' AS service)+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=A/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT 'trace' AS service)+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=B/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT 'log' AS service)+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=C/data.parquet'+STORED AS PARQUET;+----+1++query I+COPY (SELECT 'log' AS service)+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=D/data.parquet'+STORED AS PARQUET;+----+1++statement ok+CREATE EXTERNAL TABLE pruning_fact (value BIGINT)+STORED AS PARQUET+PARTITIONED BY (f_dkey STRING)+LOCATION 'test_files/scratch/preserve_file_partitioning/pruning_fact/';++statement ok+CREATE EXTERNAL TABLE pruning_dim (service STRING)+STORED AS PARQUET+PARTITIONED BY (d_dkey STRING)+LOCATION 'test_files/scratch/preserve_file_partitioning/pruning_dim/';++statement ok+set datafusion.execution.target_partitions = 3;++statement ok+set datafusion.optimizer.preserve_file_partitions = 1;++statement ok+set datafusion.optimizer.hash_join_single_partition_threshold = 0;++statement ok+set datafusion.optimizer.hash_join_single_partition_threshold_rows = 0;++# Pruning B from the dimension side keeps [A], [], [C, D].+query TT+EXPLAIN SELECT f.f_dkey, f.value+FROM pruning_fact f+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey+WHERE d.service = 'log'+ORDER BY f.f_dkey;+----+logical_plan+01)Sort: f.f_dkey ASC NULLS LAST+02)--Projection: f.f_dkey, f.value+03)----Inner Join: f.f_dkey = d.d_dkey+04)------SubqueryAlias: f+05)--------TableScan: pruning_fact projection=[value, f_dkey]+06)------SubqueryAlias: d+07)--------Projection: pruning_dim.d_dkey+08)----------Filter: pruning_dim.service = Utf8View("log")+09)------------TableScan: pruning_dim projection=[service, d_dkey], partial_filters=[pruning_dim.service = Utf8View("log")]+physical_plan+01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]+02)--SortExec: expr=[f_dkey@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(f_dkey@1, d_dkey@0)], projection=[f_dkey@1, value@0]+04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=D/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet+05)------FilterExec: service@0 = log, projection=[d_dkey@1]+06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=A/data.parquet], [], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=D/data.parquet]]}, projection=[service, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@0 = log AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]++query TI rowsort+SELECT f.f_dkey, f.value+FROM pruning_fact f+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey+WHERE d.service = 'log';+----+A 10+C 30+D 40++# Pruning B from the fact side keeps [A], [], [C, D].+query TT+EXPLAIN SELECT f.f_dkey, f.value+FROM pruning_fact f+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey+WHERE f.value <> 20+ORDER BY f.f_dkey;+----+logical_plan+01)Sort: f.f_dkey ASC NULLS LAST+02)--Projection: f.f_dkey, f.value+03)----Inner Join: f.f_dkey = d.d_dkey+04)------SubqueryAlias: f+05)--------Filter: pruning_fact.value != Int64(20)+06)----------TableScan: pruning_fact projection=[value, f_dkey], partial_filters=[pruning_fact.value != Int64(20)]+07)------SubqueryAlias: d+08)--------TableScan: pruning_dim projection=[d_dkey]+physical_plan+01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]+02)--SortExec: expr=[f_dkey@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(f_dkey@1, d_dkey@0)], projection=[f_dkey@1, value@0]+04)------FilterExec: value@0 != 20+05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=A/data.parquet], [], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=D/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=value@0 != 20, pruning_predicate=value_null_count@2 != row_count@3 AND (value_min@0 != 20 OR 20 != value_max@1), required_guarantees=[value not in (20)]+06)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=D/data.parquet]]}, projection=[d_dkey], output_partitioning=Range([d_dkey@0 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible++query TI rowsort+SELECT f.f_dkey, f.value+FROM pruning_fact f+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey+WHERE f.value <> 20;+----+A 10+C 30+D 40+########### CLEANUP##########@@ -963,3 +1120,9 @@ DROP TABLE dimension_table_skewed;statement okDROP TABLE high_cardinality_table;++statement ok+DROP TABLE pruning_fact;++statement ok+DROP TABLE pruning_dim;diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.sltindex 5f01f8e52e..aaef7746e2 100644--- a/datafusion/sqllogictest/test_files/projection.slt+++ b/datafusion/sqllogictest/test_files/projection.slt@@ -331,5 +331,4 @@ logical_plan03)----TableScan: t1 projection=[a], partial_filters=[t1.a > Int64(1)]physical_plan01)FilterExec: a@0 > 1, projection=[]-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]+02)--EmptyExecdiff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex eb698f6ef9..52dbb7875a 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -754,7 +754,7 @@ physical_plan03)----SortExec: expr=[id@1 ASC NULLS LAST], preserve_partitioning=[true]04)------FilterExec: id@1 > 205)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3-06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]+06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]# Verify correctnessquery IIdiff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex 3e5a35e0fe..92fac9936d 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -64,7 +64,7 @@ explain select * from test_filter_with_limit where value = 2 limit 1;----physical_plan01)CoalescePartitionsExec: fetch=1-02)--DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-2.parquet]]}, projection=[part_key, value], limit=1, file_type=parquet, predicate=value@1 = 2, pruning_predicate=value_null_count@2 != row_count@3 AND value_min@0 <= 2 AND 2 <= value_max@1, required_guarantees=[value in (2)]+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-1.parquet]]}, projection=[part_key, value], limit=1, file_type=parquet, predicate=value@1 = 2, pruning_predicate=value_null_count@2 != row_count@3 AND value_min@0 <= 2 AND 2 <= value_max@1, required_guarantees=[value in (2)]query IIselect * from test_filter_with_limit where value = 2 limit 1;@@ -101,19 +101,25 @@ LOCATION 'test_files/scratch/push_down_filter_parquet/t.parquet';query TTexplain select a from t where a = '100';-----physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/t.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 = 100, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= 100 AND 100 <= a_max@1, required_guarantees=[a in (100)]+physical_plan+01)FilterExec: a@0 = 100+02)--EmptyExec# The predicate should not have a column cast when the value is a valid i32query TTexplain select a from t where a != '100';-----physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/t.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 != 100, pruning_predicate=a_null_count@2 != row_count@3 AND (a_min@0 != 100 OR 100 != a_max@1), required_guarantees=[a not in (100)]+physical_plan+01)FilterExec: a@0 != 100+02)--EmptyExec# The predicate should not have a column cast when the operator is = or != and the literal can be round-trip casted without losing information.query TTexplain select a from t where cast(a as string) = '100';-----physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/t.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 = 100, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= 100 AND 100 <= a_max@1, required_guarantees=[a in (100)]+physical_plan+01)FilterExec: a@0 = 100+02)--EmptyExec# The predicate should still have the column cast when the literal alters its string representation after round-trip casting (leading zero lost).query TTdiff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.sltindex dd3ae49373..5e8f94fceb 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt@@ -133,7 +133,7 @@ physical_plan01)AggregateExec: mode=Final, gby=[], aggr=[max(agg_dyn_test.id)]02)--CoalescePartitionsExec03)----AggregateExec: mode=Partial, gby=[], aggr=[max(agg_dyn_test.id)]-04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-01/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-02/j5fUeSDQo22oPyPU.parquet], [WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-03/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-04/j5fUeSDQo22oPyPU.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 > 1 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-01/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-02/j5fUeSDQo22oPyPU.parquet], [WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-03/j5fUeSDQo22oPyPU.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 > 1 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]query Iselect max(id) from agg_dyn_test where id > 1;diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.sltindex aaf41e2c2f..86d5283a5b 100644--- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt+++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt@@ -366,7 +366,7 @@ physical_plan11)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]12)----------------------CoalescePartitionsExec13)------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]-14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible# Verify results without subset satisfaction@@ -461,7 +461,7 @@ physical_plan09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]10)------------------CoalescePartitionsExec11)--------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]-12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible# Verify results match with subset satisfaction