Skip to content

Apache DataFusion SQL Query Engine

perf: prune ListingTable files before grouping (#26441)

Rishabh Guptafd101e99c21e
diff --git a/Cargo.lock b/Cargo.lock
index 7f8b942657..e3bfa7f595 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1865,6 +1865,7 @@ dependencies = [
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-pruning",
"futures",
"itertools 0.15.0",
"log",
diff --git a/datafusion/catalog-listing/Cargo.toml b/datafusion/catalog-listing/Cargo.toml
index a71791d430..d375e6c922 100644
--- a/datafusion/catalog-listing/Cargo.toml
+++ b/datafusion/catalog-listing/Cargo.toml
@@ -29,6 +29,7 @@ datafusion-physical-expr = { workspace = true }
datafusion-physical-expr-adapter = { workspace = true }
datafusion-physical-expr-common = { workspace = true }
datafusion-physical-plan = { workspace = true }
+datafusion-pruning = { workspace = true }
futures = { workspace = true }
itertools = { workspace = true }
log = { workspace = true }
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index a1c3f16d4e..6e3dfe0691 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -10,6 +10,7 @@ use crate::{ListingOptions, ListingTableConfig};
use arrow::datatypes::{Field, Metadata, Schema, SchemaBuilder, SchemaRef};
use async_trait::async_trait;
use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};
+use datafusion_common::pruning::PrunableStatistics;
use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_common::{
Constraints, DFSchema, SchemaExt, SplitPoint, Statistics, internal_datafusion_err,
@@ -41,6 +42,7 @@ use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_plan::ExecutionPlan;
use datafusion_physical_plan::empty::EmptyExec;
use datafusion_physical_plan::limit::GlobalLimitExec;
+use datafusion_pruning::PruningPredicateBuilder;
use futures::future::BoxFuture;
use futures::{Stream, StreamExt, TryStreamExt, future, stream};
use object_store::ObjectStore;
@@ -662,7 +664,12 @@ impl ListingTable {
grouped_by_partition: partitioned_by_file_group,
partition_split_points,
} = self
- .list_files_for_scan(state, &partition_filters, statistic_file_limit)
+ .list_files_for_scan_inner(
+ state,
+ &partition_filters,
+ &filters,
+ statistic_file_limit,
+ )
.await?;
// if no files need to be read, return an `EmptyExec`
@@ -915,15 +922,28 @@ impl ListingTable {
filters: &'a [Expr],
limit: Option<usize>,
) -> datafusion_common::Result<ListFilesResult> {
+ self.list_files_for_scan_inner(ctx, filters, &[], limit)
+ .await
+ }
+
+ async fn list_files_for_scan_inner<'a>(
+ &'a self,
+ ctx: &'a dyn Session,
+ partition_filters: &'a [Expr],
+ data_filters: &'a [Expr],
+ limit: Option<usize>,
+ ) -> datafusion_common::Result<ListFilesResult> {
+ // Declared layouts must retain their partition identities, including empty groups.
if let Some(output_partitioning) = self.options.output_partitioning.as_ref() {
self.list_files_for_declared_output_partitioning(
ctx,
output_partitioning,
- filters,
+ partition_filters,
)
.await
} else {
- self.list_files_for_regular_scan(ctx, filters, limit).await
+ self.list_files_for_regular_scan(ctx, partition_filters, data_filters, limit)
+ .await
}
}
@@ -979,10 +999,51 @@ impl ListingTable {
get_files_with_limit(files, file_limit, ctx.config().collect_statistics()).await
}
+ /// Build a predicate that can remove files whose collected statistics prove
+ /// the filter cannot match. This performs no I/O and leaves the normal row
+ /// filters in place.
+ fn file_statistics_pruning_predicate(
+ &self,
+ ctx: &dyn Session,
+ filters: &[Expr],
+ ) -> Option<Arc<datafusion_pruning::PruningPredicate>> {
+ if !ctx.config().collect_statistics() || self.expr_adapter_factory.is_some() {
+ return None;
+ }
+ // Custom adapters can change values relative to file statistics, and volatile
+ // expressions must not be evaluated while planning.
+ let filter = filters
+ .iter()
+ .filter(|f| !f.is_volatile())
+ .cloned()
+ .reduce(Expr::and)?;
+ let Ok(schema) = DFSchema::try_from(self.table_schema.as_ref().clone()) else {
+ return None;
+ };
+ let Ok(predicate) = ctx.create_physical_expr(filter, &schema) else {
+ return None;
+ };
+ PruningPredicateBuilder::new()
+ .with_file_schema(Arc::clone(&self.table_schema))
+ .build(predicate)
+ }
+
+ fn prune_files_by_statistics(
+ &self,
+ predicate: Option<&datafusion_pruning::PruningPredicate>,
+ files: FileGroup,
+ ) -> FileGroup {
+ let Some(predicate) = predicate else {
+ return files;
+ };
+ prune_collected_files(predicate, &self.file_schema, files)
+ }
+
async fn list_files_for_regular_scan<'a>(
&'a self,
ctx: &'a dyn Session,
filters: &'a [Expr],
+ data_filters: &'a [Expr],
limit: Option<usize>,
) -> datafusion_common::Result<ListFilesResult> {
let file_group_count = ctx.config().target_partitions();
@@ -1005,6 +1066,7 @@ impl ListingTable {
let (file_group, inexact_stats) = self
.collect_files_for_scan(ctx, &store, filters, limit)
.await?;
+ let pruning_predicate = self.file_statistics_pruning_predicate(ctx, data_filters);
// Threshold: 0 = disabled, N > 0 = enabled when distinct_keys >= N
//
@@ -1018,17 +1080,30 @@ impl ListingTable {
let (grouped, split_points) = file_group
.group_by_partition_values_with_split_points(file_group_count);
if grouped.len() >= threshold {
+ // Preserve the groups and split points derived from all files so
+ // pruning cannot change the declared range partitioning.
+ let grouped = grouped
+ .into_iter()
+ .map(|group| {
+ self.prune_files_by_statistics(
+ pruning_predicate.as_deref(),
+ group,
+ )
+ })
+ .collect();
(grouped, true, split_points)
} else {
let all_files: Vec<_> =
grouped.into_iter().flat_map(|g| g.into_inner()).collect();
- (
- FileGroup::new(all_files).split_files(file_group_count),
- false,
- vec![],
- )
+ let file_group = self.prune_files_by_statistics(
+ pruning_predicate.as_deref(),
+ FileGroup::new(all_files),
+ );
+ (file_group.split_files(file_group_count), false, vec![])
}
} else {
+ let file_group = self
+ .prune_files_by_statistics(pruning_predicate.as_deref(), file_group);
(file_group.split_files(file_group_count), false, vec![])
};
@@ -1260,6 +1335,39 @@ async fn get_files_with_limit(
Ok((file_group, inexact_stats))
}
+/// Keep a file unless the pruning predicate proves it cannot contain a match.
+fn prune_collected_files(
+ predicate: &datafusion_pruning::PruningPredicate,
+ file_schema: &SchemaRef,
+ files: FileGroup,
+) -> FileGroup {
+ FileGroup::new(
+ files
+ .into_inner()
+ .into_iter()
+ .filter(|file| {
+ // Same check as `FilePruner::try_new`: all-`Absent` statistics
+ // cannot prune, so skip building `PrunableStatistics` for them.
+ if !file.has_statistics() {
+ return true;
+ }
+ let Some(statistics) = &file.statistics else {
+ return true;
+ };
+ // One unknown file must not hide the exact bounds of another file.
+ let statistics = PrunableStatistics::new(
+ vec![Arc::clone(statistics)],
+ Arc::clone(file_schema),
+ );
+ predicate
+ .prune(&statistics)
+ .map(|keep| keep != [false])
+ .unwrap_or(true)
+ })
+ .collect(),
+ )
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1421,4 +1529,137 @@ mod tests {
let result = derive_common_ordering_from_files(&file_groups);
assert_eq!(result, Some(ordering));
}
+
+ fn prune_test_files(expr: Expr, statistics: Vec<Option<Statistics>>) -> Vec<String> {
+ use arrow::datatypes::DataType;
+ let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));
+ let physical = datafusion_physical_expr::create_physical_expr(
+ &expr,
+ &DFSchema::try_from(schema.as_ref().clone()).unwrap(),
+ &ExecutionProps::new(),
+ &PhysicalPlanningContext::default(),
+ )
+ .unwrap();
+ let predicate = PruningPredicateBuilder::new()
+ .with_file_schema(Arc::clone(&schema))
+ .try_build(physical)
+ .unwrap();
+ let files = FileGroup::new(
+ statistics
+ .into_iter()
+ .enumerate()
+ .map(|(i, stats)| {
+ let mut file = PartitionedFile::new(format!("f{i}"), 10);
+ file.statistics = stats.map(Arc::new);
+ file
+ })
+ .collect(),
+ );
+ prune_collected_files(&predicate, &schema, files)
+ .iter()
+ .map(|f| f.object_meta.location.to_string())
+ .collect()
+ }
+
+ fn bounds(
+ min: Precision<datafusion_common::ScalarValue>,
+ max: Precision<datafusion_common::ScalarValue>,
+ ) -> Statistics {
+ Statistics {
+ num_rows: Precision::Exact(10),
+ column_statistics: vec![datafusion_common::ColumnStatistics {
+ min_value: min,
+ max_value: max,
+ ..Default::default()
+ }],
+ ..Default::default()
+ }
+ }
+
+ #[test]
+ fn statistics_pruning_keeps_missing_and_inexact_bounds() {
+ use datafusion_expr::{col, lit};
+ let exact = |v: i64| Precision::Exact(v.into());
+ let inexact = |v: i64| Precision::Inexact(v.into());
+ assert_eq!(
+ prune_test_files(
+ col("d").gt_eq(lit(20_i64)),
+ vec![
+ None,
+ Some(bounds(Precision::Absent, Precision::Absent)),
+ Some(bounds(inexact(0), inexact(9))),
+ Some(bounds(exact(0), exact(9))),
+ Some(bounds(exact(20), exact(29)))
+ ]
+ ),
+ ["f0", "f1", "f2", "f4"]
+ );
+ }
+
+ #[test]
+ fn statistics_pruning_uses_independent_bounds() {
+ use datafusion_expr::{col, lit};
+ assert_eq!(
+ prune_test_files(
+ col("d").lt(lit(20_i64)),
+ vec![
+ Some(bounds(Precision::Exact(30_i64.into()), Precision::Absent)),
+ Some(bounds(Precision::Absent, Precision::Exact(10_i64.into())))
+ ]
+ ),
+ ["f1"]
+ );
+ }
+
+ #[test]
+ fn statistics_pruning_preserves_null_semantics() {
+ use datafusion_expr::{col, lit};
+ let mut all_null = bounds(Precision::Absent, Precision::Absent);
+ all_null.column_statistics[0].null_count = Precision::Exact(10);
+ let mut no_null = bounds(
+ Precision::Exact(0_i64.into()),
+ Precision::Exact(9_i64.into()),
+ );
+ no_null.column_statistics[0].null_count = Precision::Exact(0);
+ assert_eq!(
+ prune_test_files(
+ col("d").is_null(),
+ vec![Some(all_null.clone()), Some(no_null.clone())]
+ ),
+ ["f0"]
+ );
+ assert_eq!(
+ prune_test_files(
+ col("d").is_not_null(),
+ vec![Some(all_null.clone()), Some(no_null.clone())]
+ ),
+ ["f1"]
+ );
+ assert!(
+ prune_test_files(col("d").eq(lit(5_i64)), vec![Some(all_null)]).is_empty()
+ );
+ assert_eq!(
+ prune_test_files(
+ col("d").eq(lit(5_i64)).or(col("d").is_null()),
+ vec![Some(no_null)]
+ ),
+ ["f0"]
+ );
+ }
+
+ #[test]
+ fn statistics_pruning_keeps_file_on_evaluation_error() {
+ use datafusion_common::ScalarValue;
+ use datafusion_expr::{col, lit};
+ // Binary bounds cannot be cast to Int64, so `prune` returns an error.
+ // A Utf8 bound does not error, so it would not reach that branch.
+ let binary = || Precision::Exact(ScalarValue::Binary(Some(vec![1])));
+ assert_eq!(
+ prune_test_files(
+ col("d").gt(lit(10_i64)),
+ vec![Some(bounds(binary(), binary()))]
+ ),
+ ["f0"]
+ );
+ }
}
diff --git a/datafusion/core/tests/catalog_listing/file_statistics_pruning.rs b/datafusion/core/tests/catalog_listing/file_statistics_pruning.rs
new file mode 100644
index 0000000000..f672606afe
--- /dev/null
+++ b/datafusion/core/tests/catalog_listing/file_statistics_pruning.rs
@@ -0,0 +1,429 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
+
+use arrow::array::{Int64Array, RecordBatch};
+use arrow::datatypes::{DataType, Field, Schema};
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_catalog::TableProvider;
+use datafusion_catalog_listing::{ListingOptions, ListingTable, ListingTableConfig};
+use datafusion_common::{Result, stats::Precision};
+use datafusion_datasource::{
+ ListingTableUrl, file_scan_config::FileScanConfig, source::DataSourceExec,
+};
+use datafusion_datasource_parquet::ParquetFormat;
+use datafusion_expr::{Expr, Partitioning, col, lit};
+use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;
+use datafusion_physical_plan::{ExecutionPlan, empty::EmptyExec};
+use parquet::arrow::ArrowWriter;
+use parquet::file::properties::{EnabledStatistics, WriterProperties};
+use std::fs::File;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+fn fixture(values: Vec<Vec<Option<i64>>>, hive: bool, statistics: bool) -> TempDir {
+ let dir = tempfile::tempdir().unwrap();
+ let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));
+ for (i, values) in values.into_iter().enumerate() {
+ let parent = if hive {
+ dir.path().join(format!("p={}", i % 2))
+ } else {
+ dir.path().to_owned()
+ };
+ std::fs::create_dir_all(&parent).unwrap();
+ let props = WriterProperties::builder()
+ .set_statistics_enabled(if statistics {
+ EnabledStatistics::Chunk
+ } else {
+ EnabledStatistics::None
+ })
+ .build();
+ let mut writer = ArrowWriter::try_new(
+ File::create(parent.join(format!("f{i}.parquet"))).unwrap(),
+ Arc::clone(&schema),
+ Some(props),
+ )
+ .unwrap();
+ writer
+ .write(
+ &RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int64Array::from(values))],
+ )
+ .unwrap(),
+ )
+ .unwrap();
+ writer.close().unwrap();
+ }
+ dir
+}
+fn eight_files() -> TempDir {
+ fixture(
+ (0..8)
+ .map(|i| (i * 10..i * 10 + 10).map(Some).collect())
+ .collect(),
+ false,
+ true,
+ )
+}
+fn table(
+ dir: &TempDir,
+ partitioning: Option<Partitioning>,
+ hive: bool,
+ adapter: bool,
+) -> ListingTable {
+ let schema = Arc::new(Schema::new(vec![Field::new("d", DataType::Int64, true)]));
+ let mut options = ListingOptions::new(Arc::new(ParquetFormat::default()))
+ .with_file_extension(".parquet")
+ .with_output_partitioning(partitioning);
+ if hive {
+ options = options.with_table_partition_cols(vec![("p".into(), DataType::Int64)]);
+ }
+ let mut config = ListingTableConfig::new(
+ ListingTableUrl::parse(dir.path().to_str().unwrap()).unwrap(),
+ )
+ .with_listing_options(options)
+ .with_schema(schema);
+ if adapter {
+ config =
+ config.with_expr_adapter_factory(Arc::new(DefaultPhysicalExprAdapterFactory));
+ }
+ ListingTable::try_new(config).unwrap()
+}
+fn context(statistics: bool) -> SessionContext {
+ SessionContext::new_with_config(
+ SessionConfig::new()
+ .with_target_partitions(4)
+ .with_collect_statistics(statistics),
+ )
+}
+
+fn scan_config(plan: &Arc<dyn ExecutionPlan>) -> &FileScanConfig {
+ plan.downcast_ref::<DataSourceExec>()
+ .unwrap()
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .unwrap()
+}
+
+fn names(config: &FileScanConfig) -> Vec<Vec<String>> {
+ config
+ .file_groups
+ .iter()
+ .map(|g| {
+ g.iter()
+ .map(|f| f.object_meta.location.filename().unwrap().to_string())
+ .collect()
+ })
+ .collect()
+}
+
+fn predicate() -> Expr {
+ col("d").gt_eq(lit(20_i64)).and(col("d").lt(lit(40_i64)))
+}
+
+async fn rows(ctx: &SessionContext, query: &str) -> Result<usize> {
+ Ok(ctx
+ .sql(query)
+ .await?
+ .collect()
+ .await?
+ .iter()
+ .map(RecordBatch::num_rows)
+ .sum())
+}
+
+#[tokio::test]
+async fn prunes_before_grouping_and_aggregates_survivor_statistics() -> Result<()> {
+ let dir = eight_files();
+ let ctx = context(true);
+ let table = table(&dir, None, false, false);
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ let config = scan_config(&plan);
+ assert_eq!(names(config), vec![vec!["f2.parquet"], vec!["f3.parquet"]]);
+ assert_eq!(config.statistics().num_rows, Precision::Exact(20));
+ assert_eq!(
+ config.statistics().column_statistics[0].min_value,
+ Precision::Exact(20_i64.into())
+ );
+ assert_eq!(
+ config.statistics().column_statistics[0].max_value,
+ Precision::Exact(39_i64.into())
+ );
+ ctx.register_table("t", Arc::new(table))?;
+ let optimized = ctx
+ .sql("select * from t where d >= 20 and d < 40")
+ .await?
+ .create_physical_plan()
+ .await?;
+ let filtered_statistics =
+ datafusion_physical_plan::statistics::StatisticsContext::new().compute(
+ optimized.as_ref(),
+ &datafusion_physical_plan::statistics::StatisticsArgs::new(),
+ )?;
+ assert!(
+ matches!(filtered_statistics.num_rows, Precision::Inexact(20)),
+ "{filtered_statistics:?}"
+ );
+ assert_eq!(
+ rows(&ctx, "select * from t where d >= 20 and d < 40").await?,
+ 20
+ );
+ assert_eq!(
+ rows(&ctx, "select * from t where d >= 25 and d < 35").await?,
+ 10
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn disabled_statistics_keep_all_files() -> Result<()> {
+ let dir = eight_files();
+ let ctx = context(false);
+ let table = table(&dir, None, false, false);
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ assert_eq!(
+ scan_config(&plan)
+ .file_groups
+ .iter()
+ .map(|g| g.len())
+ .sum::<usize>(),
+ 8
+ );
+ ctx.register_table("t", Arc::new(table))?;
+ assert_eq!(
+ rows(&ctx, "select * from t where d >= 20 and d < 40").await?,
+ 20
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn missing_statistics_keep_files() -> Result<()> {
+ let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, false);
+ let ctx = context(true);
+ let table = table(&dir, None, false, false);
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ assert_eq!(
+ scan_config(&plan)
+ .file_groups
+ .iter()
+ .map(|g| g.len())
+ .sum::<usize>(),
+ 2
+ );
+ ctx.register_table("t", Arc::new(table))?;
+ assert_eq!(
+ rows(&ctx, "select * from t where d >= 20 and d < 40").await?,
+ 1
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn null_semantics_and_empty_scan() -> Result<()> {
+ let dir = fixture(
+ vec![
+ vec![None, None],
+ vec![Some(5), Some(6)],
+ vec![None, Some(25)],
+ ],
+ false,
+ true,
+ );
+ let ctx = context(true);
+ let table = table(&dir, None, false, false);
+ let plan = table
+ .scan(&ctx.state(), None, &[col("d").gt(lit(100_i64))], None)
+ .await?;
+ assert!(plan.downcast_ref::<EmptyExec>().is_some());
+ ctx.register_table("t", Arc::new(table))?;
+ assert_eq!(rows(&ctx, "select * from t where d is null").await?, 3);
+ assert_eq!(rows(&ctx, "select * from t where d is not null").await?, 3);
+ assert_eq!(
+ rows(&ctx, "select * from t where d >= 20 or d is null").await?,
+ 4
+ );
+ assert_eq!(
+ rows(&ctx, "select * from t where d not in (5, NULL)").await?,
+ 0
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn declared_partitioning_keeps_groups_and_empty_trailing_groups() -> Result<()> {
+ let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, true);
+ let ctx = context(true);
+ let table = table(
+ &dir,
+ Some(Partitioning::Hash(vec![col("d")], 4)),
+ false,
+ false,
+ );
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ assert_eq!(
+ names(scan_config(&plan)),
+ vec![vec!["f0.parquet"], vec!["f1.parquet"], vec![], vec![]]
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn limits_offsets_and_projection_keep_row_filter() -> Result<()> {
+ let dir = eight_files();
+ let ctx = context(true);
+ let table = table(&dir, None, false, false);
+ ctx.register_table("t", Arc::new(table))?;
+ assert_eq!(
+ rows(
+ &ctx,
+ "select d from t where d >= 25 and d < 35 limit 3 offset 2"
+ )
+ .await?,
+ 3
+ );
+ let batches = ctx
+ .sql("select d from t where d >= 25 and d < 35 order by d limit 3 offset 2")
+ .await?
+ .collect()
+ .await?;
+ let values: Vec<_> = batches
+ .iter()
+ .flat_map(|b| {
+ b.column(0)
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .unwrap()
+ .values()
+ .iter()
+ .copied()
+ })
+ .collect();
+ assert_eq!(values, [27, 28, 29]);
+ Ok(())
+}
+
+#[tokio::test]
+async fn custom_adapter_disables_statistics_pruning() -> Result<()> {
+ let dir = eight_files();
+ let ctx = context(true);
+ let table = table(&dir, None, false, true);
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ assert_eq!(
+ scan_config(&plan)
+ .file_groups
+ .iter()
+ .map(|g| g.len())
+ .sum::<usize>(),
+ 8
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn hive_and_data_predicates_preserve_partition_groups() -> Result<()> {
+ let dir = fixture(
+ (0..8)
+ .map(|i| (i * 10..i * 10 + 10).map(Some).collect())
+ .collect(),
+ true,
+ true,
+ );
+ let mut config = SessionConfig::new()
+ .with_target_partitions(4)
+ .with_collect_statistics(true);
+ config.options_mut().optimizer.preserve_file_partitions = 1;
+ let ctx = SessionContext::new_with_config(config);
+ let table = table(&dir, None, true, false);
+ let filters = [col("p").eq(lit(0_i64)), predicate()];
+ let plan = table.scan(&ctx.state(), None, &filters, None).await?;
+ assert_eq!(names(scan_config(&plan)), vec![vec!["f2.parquet"]]);
+ ctx.register_table("t", Arc::new(table))?;
+ assert_eq!(
+ rows(&ctx, "select * from t where p = 0 and d >= 20 and d < 40").await?,
+ 10
+ );
+ assert_eq!(rows(&ctx, "select * from t where p = d").await?, 1);
+ Ok(())
+}
+
+#[tokio::test]
+async fn preserve_partitions_below_threshold_still_prunes() -> Result<()> {
+ let dir = fixture(
+ (0..8)
+ .map(|i| (i * 10..i * 10 + 10).map(Some).collect())
+ .collect(),
+ true,
+ true,
+ );
+ let mut config = SessionConfig::new()
+ .with_target_partitions(4)
+ .with_collect_statistics(true);
+ // Two distinct `p` values are below the threshold, so files are split normally.
+ config.options_mut().optimizer.preserve_file_partitions = 3;
+ let ctx = SessionContext::new_with_config(config);
+ let table = table(&dir, None, true, false);
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ assert_eq!(
+ names(scan_config(&plan)),
+ vec![vec!["f2.parquet"], vec!["f3.parquet"]]
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn volatile_predicate_is_not_evaluated_while_listing() -> Result<()> {
+ let dir = eight_files();
+ let ctx = context(true);
+ let table = table(&dir, None, false, false);
+ use datafusion_expr::{ColumnarValue, Volatility, create_udf};
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed = Arc::clone(&calls);
+ let udf = create_udf(
+ "volatile_threshold",
+ vec![],
+ DataType::Int64,
+ Volatility::Volatile,
+ Arc::new(move |_| {
+ observed.fetch_add(1, Ordering::SeqCst);
+ Ok(ColumnarValue::Scalar(1000_i64.into()))
+ }),
+ );
+ let filters = [col("d").gt(udf.call(vec![]))];
+ let plan = table.scan(&ctx.state(), None, &filters, None).await?;
+ assert_eq!(
+ scan_config(&plan)
+ .file_groups
+ .iter()
+ .map(|g| g.len())
+ .sum::<usize>(),
+ 8
+ );
+ assert_eq!(calls.load(Ordering::SeqCst), 0);
+ Ok(())
+}
+
+#[tokio::test]
+async fn declared_range_partitioning_preserves_groups() -> Result<()> {
+ use datafusion_common::SplitPoint;
+ use datafusion_expr::RangePartitioning;
+ let dir = fixture(vec![vec![Some(1)], vec![Some(25)]], false, true);
+ let ctx = context(true);
+ let partitioning = Partitioning::Range(RangePartitioning::try_new(
+ vec![col("d").sort(true, true)],
+ vec![
+ SplitPoint::new(vec![10_i64.into()]),
+ SplitPoint::new(vec![30_i64.into()]),
+ SplitPoint::new(vec![40_i64.into()]),
+ ],
+ )?);
+ let table = table(&dir, Some(partitioning), false, false);
+ let plan = table.scan(&ctx.state(), None, &[predicate()], None).await?;
+ assert_eq!(
+ names(scan_config(&plan)),
+ vec![vec!["f0.parquet"], vec!["f1.parquet"], vec![], vec![]]
+ );
+ Ok(())
+}
diff --git a/datafusion/core/tests/catalog_listing/mod.rs b/datafusion/core/tests/catalog_listing/mod.rs
index d07fa3e8b8..49c0df520b 100644
--- a/datafusion/core/tests/catalog_listing/mod.rs
+++ b/datafusion/core/tests/catalog_listing/mod.rs
@@ -3,3 +3,5 @@
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
mod pruned_partition_list;
+
+mod file_statistics_pruning;
diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rs
index f89879709a..3e4e1ce8af 100644
--- a/datafusion/core/tests/datasource/object_store_access.rs
+++ b/datafusion/core/tests/datasource/object_store_access.rs
@@ -1642,3 +1642,50 @@ impl ObjectStore for RequestCountingObjectStore {
unimplemented!()
}
}
+
+#[tokio::test]
+async fn listing_statistics_pruning_adds_no_object_store_requests() {
+ let unfiltered = Test::new().with_single_file_parquet().await;
+ unfiltered
+ .session_context
+ .sql("SET datafusion.execution.collect_statistics = true")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ unfiltered.object_store.clear_requests();
+ unfiltered
+ .session_context
+ .sql("select * from parquet_table")
+ .await
+ .unwrap()
+ .create_physical_plan()
+ .await
+ .unwrap();
+ let baseline = unfiltered.requests();
+ let filtered = Test::new().with_single_file_parquet().await;
+ filtered
+ .session_context
+ .sql("SET datafusion.execution.collect_statistics = true")
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ filtered.object_store.clear_requests();
+ let plan = filtered
+ .session_context
+ .sql("select * from parquet_table where a > 1000")
+ .await
+ .unwrap()
+ .create_physical_plan()
+ .await
+ .unwrap();
+ let display = datafusion_physical_plan::displayable(plan.as_ref())
+ .indent(true)
+ .to_string();
+ assert!(display.contains("EmptyExec"), "{display}");
+ assert!(!display.contains("DataSourceExec"), "{display}");
+ assert_eq!(filtered.requests(), baseline);
+}
diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs
index 26d0829944..54482eaa5d 100644
--- a/datafusion/core/tests/parquet/mod.rs
+++ b/datafusion/core/tests/parquet/mod.rs
@@ -416,7 +416,7 @@ impl ContextWithParquet {
// find the parquet metrics
let parquet_metrics =
- MetricsFinder::find_metrics(physical_plan.as_ref()).unwrap();
+ MetricsFinder::find_metrics(physical_plan.as_ref()).unwrap_or_default();
let result_rows = results.iter().map(|b| b.num_rows()).sum();
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index b002d1023f..f826a54dc0 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -569,12 +569,12 @@ macro_rules! int_tests {
RowGroupPruningTest::new()
.with_scenario(Scenario::Int)
.with_query(&format!("SELECT * FROM t where i{} in (100)", $bits))
- .with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
- .with_pruned_by_stats(Some(0))
- .with_pruned_files(Some(1))
- .with_matched_by_bloom_filter(Some(0))
- .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_errors(None)
+ .with_matched_by_stats(None)
+ .with_pruned_by_stats(None)
+ .with_pruned_files(None)
+ .with_matched_by_bloom_filter(None)
+ .with_pruned_by_bloom_filter(None)
.with_expected_rows(0)
.test_row_group_prune()
.await;
@@ -1660,13 +1660,13 @@ async fn test_row_group_with_null_values() {
RowGroupPruningTest::new()
.with_scenario(Scenario::WithNullValues)
.with_query("SELECT * FROM t WHERE \"i32\" > 7")
- .with_expected_errors(Some(0))
- .with_matched_by_stats(Some(0))
- .with_pruned_by_stats(Some(0))
- .with_pruned_files(Some(1))
+ .with_expected_errors(None)
+ .with_matched_by_stats(None)
+ .with_pruned_by_stats(None)
+ .with_pruned_files(None)
.with_expected_rows(0)
- .with_matched_by_bloom_filter(Some(0))
- .with_pruned_by_bloom_filter(Some(0))
+ .with_matched_by_bloom_filter(None)
+ .with_pruned_by_bloom_filter(None)
.test_row_group_prune()
.await;
}
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs
index 77c09afd16..8fe4519eba 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -309,7 +309,12 @@ async fn explain_analyze_parquet_pruning_metrics() {
let expected_metrics =
format!("files_ranges_pruned_statistics={expected_pruning_metrics}");
- assert_metrics!(&plan, "DataSourceExec", &expected_metrics);
+ if l_orderkey == 10 {
+ assert!(plan.contains("EmptyExec"), "{plan}");
+ assert!(!plan.contains("DataSourceExec"), "{plan}");
+ } else {
+ assert_metrics!(&plan, "DataSourceExec", &expected_metrics);
+ }
}
}
diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.slt
index d89880ca33..fcebbf2850 100644
--- a/datafusion/sqllogictest/test_files/clickbench.slt
+++ b/datafusion/sqllogictest/test_files/clickbench.slt
@@ -65,6 +65,9 @@ SELECT COUNT(*) FROM hits;
10
## Q1
+statement ok
+set datafusion.execution.collect_statistics = false;
+
query TT
EXPLAIN SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0;
----
@@ -89,6 +92,9 @@ SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0;
----
0
+statement ok
+reset datafusion.execution.collect_statistics;
+
query TT
EXPLAIN SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits;
----
@@ -193,6 +199,9 @@ SELECT MIN("EventDate"), MAX("EventDate") FROM hits;
2013-07-15 2013-07-15
## Q7
+statement ok
+set datafusion.execution.collect_statistics = false;
+
query TT
EXPLAIN SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;
----
@@ -218,6 +227,9 @@ query II
SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;
----
+statement ok
+reset datafusion.execution.collect_statistics;
+
## Q8
query TT
EXPLAIN SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10;
@@ -970,6 +982,9 @@ SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c
1615432634 1615432633 1615432632 1615432631 1
## Q36
+statement ok
+set datafusion.execution.collect_statistics = false;
+
query TT
EXPLAIN SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10;
----
@@ -1169,6 +1184,9 @@ query PI
SELECT DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000;
----
+statement ok
+reset datafusion.execution.collect_statistics;
+
# Clickbench "Extended" queries that test count distinct
query TT
diff --git a/datafusion/sqllogictest/test_files/grouping_set_repartition.slt b/datafusion/sqllogictest/test_files/grouping_set_repartition.slt
index 1c1e509ae0..e78c79a28f 100644
--- a/datafusion/sqllogictest/test_files/grouping_set_repartition.slt
+++ b/datafusion/sqllogictest/test_files/grouping_set_repartition.slt
@@ -138,19 +138,22 @@ physical_plan
10)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4
11)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
12)----------------------FilterExec: channel@0 = store, projection=[brand@1, amount@2]
-13)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = store, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= store AND store <= channel_max@1, required_guarantees=[channel in (store)]
-14)--------------ProjectionExec: expr=[web as channel, brand@0 as brand, sum(sales.amount)@1 as total]
-15)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
-16)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4
-17)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
-18)----------------------FilterExec: channel@0 = web, projection=[brand@1, amount@2]
-19)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = web, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= web AND web <= channel_max@1, required_guarantees=[channel in (web)]
-20)--------------ProjectionExec: expr=[catalog as channel, brand@0 as brand, sum(sales.amount)@1 as total]
-21)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
-22)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4
-23)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
-24)----------------------FilterExec: channel@0 = catalog, projection=[brand@1, amount@2]
-25)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = catalog, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= catalog AND catalog <= channel_max@1, required_guarantees=[channel in (catalog)]
+13)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3
+14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=1/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = store, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= store AND store <= channel_max@1, required_guarantees=[channel in (store)]
+15)--------------ProjectionExec: expr=[web as channel, brand@0 as brand, sum(sales.amount)@1 as total]
+16)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
+17)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4
+18)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
+19)----------------------FilterExec: channel@0 = web, projection=[brand@1, amount@2]
+20)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
+21)--------------------------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=2/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = web, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= web AND web <= channel_max@1, required_guarantees=[channel in (web)]
+22)--------------ProjectionExec: expr=[catalog as channel, brand@0 as brand, sum(sales.amount)@1 as total]
+23)----------------AggregateExec: mode=FinalPartitioned, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
+24)------------------RepartitionExec: partitioning=Hash([brand@0], 4), input_partitions=4
+25)--------------------AggregateExec: mode=Partial, gby=[brand@0 as brand], aggr=[sum(sales.amount)]
+26)----------------------FilterExec: channel@0 = catalog, projection=[brand@1, amount@2]
+27)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
+28)--------------------------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=3/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/grouping_set_repartition/part=4/data.parquet]]}, projection=[channel, brand, amount], file_type=parquet, predicate=channel@0 = catalog, pruning_predicate=channel_null_count@2 != row_count@3 AND channel_min@0 <= catalog AND catalog <= channel_max@1, required_guarantees=[channel in (catalog)]
query TTI rowsort
SELECT channel, brand, SUM(total) as grand_total
diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
index 341b2784a5..767a5410a7 100644
--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
@@ -361,9 +361,8 @@ logical_plan
03)----Filter: t_pushdown.b > Int32(2) AND t_pushdown.a IS NOT NULL
04)------TableScan: t_pushdown projection=[a, b], partial_filters=[t_pushdown.b > Int32(2), t_pushdown.a IS NOT NULL]
physical_plan
-01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]
-02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]
query I
@@ -380,9 +379,8 @@ logical_plan
03)----Filter: t_pushdown.a = Utf8View("bar")
04)------TableScan: t_pushdown projection=[a, b], partial_filters=[t_pushdown.a = Utf8View("bar")]
physical_plan
-01)SortPreservingMergeExec: [b@0 ASC NULLS LAST]
-02)--SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, sort_order_for_reorder=[b@0 ASC NULLS LAST], pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]
+01)SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, sort_order_for_reorder=[b@0 ASC NULLS LAST], pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]
# should not push down volatile predicates such as RANDOM
diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt
index 208eabb832..3911180ec6 100644
--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt
@@ -46,9 +46,9 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]
-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]
# cleanup
statement ok
@@ -71,9 +71,9 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]
-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]
+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(3), Bytes=Inexact(24), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(3)) Null=Inexact(0) ScanBytes=Inexact(24))]]
# cleanup
statement ok
diff --git a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt
index 209d050efc..21bef2bedf 100644
--- a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt
+++ b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt
@@ -926,6 +926,163 @@ A 7 110.342857142857
B 7 87.771428571429
C 7 288.228571428571
+##########
+# TEST 18: File-statistics pruning preserves partition-value groups
+# Both sides start with values A-D and split points [(B), (C)]. Pruning B from
+# either side must leave an empty middle group without adding a repartition.
+##########
+
+query I
+COPY (SELECT CAST(10 AS BIGINT) AS value)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=A/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT CAST(20 AS BIGINT) AS value)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=B/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT CAST(30 AS BIGINT) AS value)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=C/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT CAST(40 AS BIGINT) AS value)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=D/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT 'log' AS service)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=A/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT 'trace' AS service)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=B/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT 'log' AS service)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=C/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+query I
+COPY (SELECT 'log' AS service)
+TO 'test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=D/data.parquet'
+STORED AS PARQUET;
+----
+1
+
+statement ok
+CREATE EXTERNAL TABLE pruning_fact (value BIGINT)
+STORED AS PARQUET
+PARTITIONED BY (f_dkey STRING)
+LOCATION 'test_files/scratch/preserve_file_partitioning/pruning_fact/';
+
+statement ok
+CREATE EXTERNAL TABLE pruning_dim (service STRING)
+STORED AS PARQUET
+PARTITIONED BY (d_dkey STRING)
+LOCATION 'test_files/scratch/preserve_file_partitioning/pruning_dim/';
+
+statement ok
+set datafusion.execution.target_partitions = 3;
+
+statement ok
+set datafusion.optimizer.preserve_file_partitions = 1;
+
+statement ok
+set datafusion.optimizer.hash_join_single_partition_threshold = 0;
+
+statement ok
+set datafusion.optimizer.hash_join_single_partition_threshold_rows = 0;
+
+# Pruning B from the dimension side keeps [A], [], [C, D].
+query TT
+EXPLAIN SELECT f.f_dkey, f.value
+FROM pruning_fact f
+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey
+WHERE d.service = 'log'
+ORDER BY f.f_dkey;
+----
+logical_plan
+01)Sort: f.f_dkey ASC NULLS LAST
+02)--Projection: f.f_dkey, f.value
+03)----Inner Join: f.f_dkey = d.d_dkey
+04)------SubqueryAlias: f
+05)--------TableScan: pruning_fact projection=[value, f_dkey]
+06)------SubqueryAlias: d
+07)--------Projection: pruning_dim.d_dkey
+08)----------Filter: pruning_dim.service = Utf8View("log")
+09)------------TableScan: pruning_dim projection=[service, d_dkey], partial_filters=[pruning_dim.service = Utf8View("log")]
+physical_plan
+01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]
+02)--SortExec: expr=[f_dkey@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(f_dkey@1, d_dkey@0)], projection=[f_dkey@1, value@0]
+04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=D/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet
+05)------FilterExec: service@0 = log, projection=[d_dkey@1]
+06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=A/data.parquet], [], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=D/data.parquet]]}, projection=[service, d_dkey], output_partitioning=Range([d_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@0 = log AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]
+
+query TI rowsort
+SELECT f.f_dkey, f.value
+FROM pruning_fact f
+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey
+WHERE d.service = 'log';
+----
+A 10
+C 30
+D 40
+
+# Pruning B from the fact side keeps [A], [], [C, D].
+query TT
+EXPLAIN SELECT f.f_dkey, f.value
+FROM pruning_fact f
+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey
+WHERE f.value <> 20
+ORDER BY f.f_dkey;
+----
+logical_plan
+01)Sort: f.f_dkey ASC NULLS LAST
+02)--Projection: f.f_dkey, f.value
+03)----Inner Join: f.f_dkey = d.d_dkey
+04)------SubqueryAlias: f
+05)--------Filter: pruning_fact.value != Int64(20)
+06)----------TableScan: pruning_fact projection=[value, f_dkey], partial_filters=[pruning_fact.value != Int64(20)]
+07)------SubqueryAlias: d
+08)--------TableScan: pruning_dim projection=[d_dkey]
+physical_plan
+01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]
+02)--SortExec: expr=[f_dkey@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(f_dkey@1, d_dkey@0)], projection=[f_dkey@1, value@0]
+04)------FilterExec: value@0 != 20
+05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=A/data.parquet], [], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_fact/f_dkey=D/data.parquet]]}, projection=[value, f_dkey], output_partitioning=Range([f_dkey@1 ASC], [(B), (C)], 3), file_type=parquet, predicate=value@0 != 20, pruning_predicate=value_null_count@2 != row_count@3 AND (value_min@0 != 20 OR 20 != value_max@1), required_guarantees=[value not in (20)]
+06)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/pruning_dim/d_dkey=D/data.parquet]]}, projection=[d_dkey], output_partitioning=Range([d_dkey@0 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
+
+query TI rowsort
+SELECT f.f_dkey, f.value
+FROM pruning_fact f
+INNER JOIN pruning_dim d ON f.f_dkey = d.d_dkey
+WHERE f.value <> 20;
+----
+A 10
+C 30
+D 40
+
##########
# CLEANUP
##########
@@ -963,3 +1120,9 @@ DROP TABLE dimension_table_skewed;
statement ok
DROP TABLE high_cardinality_table;
+
+statement ok
+DROP TABLE pruning_fact;
+
+statement ok
+DROP TABLE pruning_dim;
diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.slt
index 5f01f8e52e..aaef7746e2 100644
--- a/datafusion/sqllogictest/test_files/projection.slt
+++ b/datafusion/sqllogictest/test_files/projection.slt
@@ -331,5 +331,4 @@ logical_plan
03)----TableScan: t1 projection=[a], partial_filters=[t1.a > Int64(1)]
physical_plan
01)FilterExec: a@0 > 1, projection=[]
-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[]
+02)--EmptyExec
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index eb698f6ef9..52dbb7875a 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -754,7 +754,7 @@ physical_plan
03)----SortExec: expr=[id@1 ASC NULLS LAST], preserve_partitioning=[true]
04)------FilterExec: id@1 > 2
05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3
-06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]
+06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]
# Verify correctness
query II
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index 3e5a35e0fe..92fac9936d 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -64,7 +64,7 @@ explain select * from test_filter_with_limit where value = 2 limit 1;
----
physical_plan
01)CoalescePartitionsExec: fetch=1
-02)--DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-2.parquet]]}, projection=[part_key, value], limit=1, file_type=parquet, predicate=value@1 = 2, pruning_predicate=value_null_count@2 != row_count@3 AND value_min@0 <= 2 AND 2 <= value_max@1, required_guarantees=[value in (2)]
+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/test_filter_with_limit/part-1.parquet]]}, projection=[part_key, value], limit=1, file_type=parquet, predicate=value@1 = 2, pruning_predicate=value_null_count@2 != row_count@3 AND value_min@0 <= 2 AND 2 <= value_max@1, required_guarantees=[value in (2)]
query II
select * from test_filter_with_limit where value = 2 limit 1;
@@ -101,19 +101,25 @@ LOCATION 'test_files/scratch/push_down_filter_parquet/t.parquet';
query TT
explain select a from t where a = '100';
----
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/t.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 = 100, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= 100 AND 100 <= a_max@1, required_guarantees=[a in (100)]
+physical_plan
+01)FilterExec: a@0 = 100
+02)--EmptyExec
# The predicate should not have a column cast when the value is a valid i32
query TT
explain select a from t where a != '100';
----
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/t.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 != 100, pruning_predicate=a_null_count@2 != row_count@3 AND (a_min@0 != 100 OR 100 != a_max@1), required_guarantees=[a not in (100)]
+physical_plan
+01)FilterExec: a@0 != 100
+02)--EmptyExec
# The predicate should not have a column cast when the operator is = or != and the literal can be round-trip casted without losing information.
query TT
explain select a from t where cast(a as string) = '100';
----
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/t.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 = 100, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= 100 AND 100 <= a_max@1, required_guarantees=[a in (100)]
+physical_plan
+01)FilterExec: a@0 = 100
+02)--EmptyExec
# The predicate should still have the column cast when the literal alters its string representation after round-trip casting (leading zero lost).
query TT
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
index dd3ae49373..5e8f94fceb 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt
@@ -133,7 +133,7 @@ physical_plan
01)AggregateExec: mode=Final, gby=[], aggr=[max(agg_dyn_test.id)]
02)--CoalescePartitionsExec
03)----AggregateExec: mode=Partial, gby=[], aggr=[max(agg_dyn_test.id)]
-04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-01/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-02/j5fUeSDQo22oPyPU.parquet], [WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-03/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-04/j5fUeSDQo22oPyPU.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 > 1 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-01/j5fUeSDQo22oPyPU.parquet, WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-02/j5fUeSDQo22oPyPU.parquet], [WORKSPACE_ROOT/datafusion/core/tests/data/test_statistics_per_partition/date=2025-03-03/j5fUeSDQo22oPyPU.parquet]]}, projection=[id], file_type=parquet, predicate=id@0 > 1 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
query I
select max(id) from agg_dyn_test where id > 1;
diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt
index aaf41e2c2f..86d5283a5b 100644
--- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt
+++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt
@@ -366,7 +366,7 @@ physical_plan
11)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]
12)----------------------CoalescePartitionsExec
13)------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]
-14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]
+14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]
15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
# Verify results without subset satisfaction
@@ -461,7 +461,7 @@ physical_plan
09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]
10)------------------CoalescePartitionsExec
11)--------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]
-12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]
+12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], output_partitioning=Range([d_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]
13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Range([f_dkey@2 ASC], [(B), (C)], 3), file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
# Verify results match with subset satisfaction