Apache DataFusion SQL Query Engine
Move statistics handling into FileScanConfig (#18721)
Adrian Garcia Badaracco8a91db56b38a
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rsindex 0d2b573b47..30fd864405 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs@@ -18,7 +18,7 @@use arrow::datatypes::SchemaRef;use arrow::{array::RecordBatch, compute::concat_batches};use datafusion::{datasource::object_store::ObjectStoreUrl, physical_plan::PhysicalExpr};-use datafusion_common::{config::ConfigOptions, internal_err, Result, Statistics};+use datafusion_common::{config::ConfigOptions, internal_err, Result};use datafusion_datasource::{file::FileSource, file_scan_config::FileScanConfig,file_scan_config::FileScanConfigBuilder, file_stream::FileOpenFuture,@@ -108,7 +108,6 @@ impl FileOpener for TestOpener {pub struct TestSource {support: bool,predicate: Option<Arc<dyn PhysicalExpr>>,- statistics: Option<Statistics>,batch_size: Option<usize>,batches: Vec<RecordBatch>,schema: SchemaRef,@@ -128,7 +127,6 @@ impl TestSource {metrics: ExecutionPlanMetricsSet::new(),batches,predicate: None,- statistics: None,batch_size: None,projection: None,schema_adapter_factory: None,@@ -175,25 +173,10 @@ impl FileSource for TestSource {})}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- Arc::new(TestSource {- statistics: Some(statistics),- ..self.clone()- })- }-fn metrics(&self) -> &ExecutionPlanMetricsSet {&self.metrics}- fn statistics(&self) -> Result<Statistics> {- Ok(self- .statistics- .as_ref()- .expect("statistics not set")- .clone())- }-fn file_type(&self) -> &str {"test"}diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex 60fec22436..e410c495c8 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -131,10 +131,7 @@ pub(crate) fn parquet_exec_with_stats(file_size: u64) -> Arc<DataSourceExec> {.with_statistics(statistics).build();- assert_eq!(- config.file_source.statistics().unwrap().num_rows,- Precision::Inexact(10000)- );+ assert_eq!(config.statistics().num_rows, Precision::Inexact(10000));DataSourceExec::from_data_source(config)}diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex 5e4d7c4606..3132d8a10d 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -40,7 +40,7 @@ use datafusion_datasource::{as_file_source, TableSchema};use arrow::buffer::Buffer;use arrow::ipc::reader::{FileDecoder, FileReader, StreamReader};use datafusion_common::error::Result;-use datafusion_common::{exec_datafusion_err, Statistics};+use datafusion_common::exec_datafusion_err;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::PartitionedFile;@@ -250,7 +250,6 @@ pub struct ArrowSource {format: ArrowFormat,table_schema: TableSchema,metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}@@ -261,7 +260,6 @@ impl ArrowSource {format: ArrowFormat::File,table_schema: table_schema.into(),metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,}}@@ -272,7 +270,6 @@ impl ArrowSource {format: ArrowFormat::Stream,table_schema: table_schema.into(),metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,}}@@ -305,12 +302,6 @@ impl FileSource for ArrowSource {Arc::new(Self { ..self.clone() })}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {Arc::new(Self { ..self.clone() })}@@ -319,13 +310,6 @@ impl FileSource for ArrowSource {&self.metrics}- fn statistics(&self) -> Result<Statistics> {- let statistics = &self.projected_statistics;- Ok(statistics- .clone()- .expect("projected_statistics must be set"))- }-fn file_type(&self) -> &str {match self.format {ArrowFormat::File => "arrow",diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rsindex 600dbed21c..e83113f40e 100644--- a/datafusion/datasource-avro/src/source.rs+++ b/datafusion/datasource-avro/src/source.rs@@ -23,7 +23,6 @@ use std::sync::Arc;use crate::avro_to_arrow::Reader as AvroReader;use datafusion_common::error::Result;-use datafusion_common::Statistics;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::file_stream::FileOpener;@@ -41,7 +40,6 @@ pub struct AvroSource {batch_size: Option<usize>,projection: Option<Vec<String>>,metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}@@ -53,7 +51,6 @@ impl AvroSource {batch_size: None,projection: None,metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,}}@@ -95,12 +92,6 @@ impl FileSource for AvroSource {Arc::new(conf)}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.projection = config.projected_file_column_names();@@ -111,13 +102,6 @@ impl FileSource for AvroSource {&self.metrics}- fn statistics(&self) -> Result<Statistics> {- let statistics = &self.projected_statistics;- Ok(statistics- .clone()- .expect("projected_statistics must be set"))- }-fn file_type(&self) -> &str {"avro"}diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex 94c6b3810a..b39f5a63a6 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -34,7 +34,7 @@ use datafusion_datasource::{use arrow::csv;use datafusion_common::config::CsvOptions;-use datafusion_common::{DataFusionError, Result, Statistics};+use datafusion_common::{DataFusionError, Result};use datafusion_common_runtime::JoinSet;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -90,7 +90,6 @@ pub struct CsvSource {table_schema: TableSchema,file_projection: Option<Vec<usize>>,metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}@@ -103,7 +102,6 @@ impl CsvSource {batch_size: None,file_projection: None,metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,}}@@ -266,12 +264,6 @@ impl FileSource for CsvSource {Arc::new(conf)}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource> {let mut conf = self.clone();conf.file_projection = config.file_column_projection_indices();@@ -281,12 +273,7 @@ impl FileSource for CsvSource {fn metrics(&self) -> &ExecutionPlanMetricsSet {&self.metrics}- fn statistics(&self) -> Result<Statistics> {- let statistics = &self.projected_statistics;- Ok(statistics- .clone()- .expect("projected_statistics must be set"))- }+fn file_type(&self) -> &str {"csv"}diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 44b71ce680..fd382efc75 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -37,7 +37,6 @@ use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties};use arrow::json::ReaderBuilder;use arrow::{datatypes::SchemaRef, json};-use datafusion_common::Statistics;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_execution::TaskContext;@@ -79,7 +78,6 @@ pub struct JsonSource {table_schema: datafusion_datasource::TableSchema,batch_size: Option<usize>,metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,}@@ -90,7 +88,6 @@ impl JsonSource {table_schema: table_schema.into(),batch_size: None,metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,}}@@ -133,12 +130,6 @@ impl FileSource for JsonSource {Arc::new(conf)}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {Arc::new(Self { ..self.clone() })}@@ -147,13 +138,6 @@ impl FileSource for JsonSource {&self.metrics}- fn statistics(&self) -> Result<Statistics> {- let statistics = &self.projected_statistics;- Ok(statistics- .clone()- .expect("projected_statistics must be set to call"))- }-fn file_type(&self) -> &str {"json"}diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 7c07b7b68c..da7bc125d2 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -37,7 +37,7 @@ use datafusion_datasource::schema_adapter::{use arrow::datatypes::TimeUnit;use datafusion_common::config::TableParquetOptions;-use datafusion_common::{DataFusionError, Statistics};+use datafusion_common::DataFusionError;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;use datafusion_datasource::TableSchema;@@ -286,7 +286,6 @@ pub struct ParquetSource {pub(crate) batch_size: Option<usize>,/// Optional hint for the size of the parquet metadatapub(crate) metadata_size_hint: Option<usize>,- pub(crate) projected_statistics: Option<Statistics>,#[cfg(feature = "parquet_encryption")]pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,}@@ -307,7 +306,6 @@ impl ParquetSource {schema_adapter_factory: None,batch_size: None,metadata_size_hint: None,- projected_statistics: None,#[cfg(feature = "parquet_encryption")]encryption_factory: None,}@@ -625,12 +623,6 @@ impl FileSource for ParquetSource {Arc::new(conf)}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut conf = self.clone();- conf.projected_statistics = Some(statistics);- Arc::new(conf)- }-fn with_projection(&self, _config: &FileScanConfig) -> Arc<dyn FileSource> {Arc::new(Self { ..self.clone() })}@@ -639,23 +631,6 @@ impl FileSource for ParquetSource {&self.metrics}- fn statistics(&self) -> datafusion_common::Result<Statistics> {- let statistics = &self.projected_statistics;- let statistics = statistics- .clone()- .expect("projected_statistics must be set");- // When filters are pushed down, we have no way of knowing the exact statistics.- // Note that pruning predicate is also a kind of filter pushdown.- // (bloom filters use `pruning_predicate` too).- // Because filter pushdown may happen dynamically as long as there is a predicate- // if we have *any* predicate applied, we can't guarantee the statistics are exact.- if self.filter().is_some() {- Ok(statistics.to_inexact())- } else {- Ok(statistics)- }- }-fn file_type(&self) -> &str {"parquet"}diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex 9245f60e23..9ec34b5dda 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -27,7 +27,7 @@ use crate::file_scan_config::FileScanConfig;use crate::file_stream::FileOpener;use crate::schema_adapter::SchemaAdapterFactory;use datafusion_common::config::ConfigOptions;-use datafusion_common::{not_impl_err, Result, Statistics};+use datafusion_common::{not_impl_err, Result};use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use datafusion_physical_plan::filter_pushdown::{FilterPushdownPropagation, PushedDown};use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;@@ -68,16 +68,12 @@ pub trait FileSource: Send + Sync {fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>;/// Initialize new instance with projection informationfn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource>;- /// Initialize new instance with projected statistics- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource>;/// Returns the filter expression that will be applied during the file scan.fn filter(&self) -> Option<Arc<dyn PhysicalExpr>> {None}/// Return execution plan metricsfn metrics(&self) -> &ExecutionPlanMetricsSet;- /// Return projected statistics- fn statistics(&self) -> Result<Statistics>;/// String representation of file source such as "csv", "json", "parquet"fn file_type(&self) -> &str;/// Format FileType specific informationdiff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex 897206c3d5..4387996a29 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -104,7 +104,6 @@ use log::{debug, warn};/// #[derive(Clone)]/// # struct ParquetSource {/// # table_schema: TableSchema,-/// # projected_statistics: Option<Statistics>,/// # schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>/// # };/// # impl FileSource for ParquetSource {@@ -113,15 +112,13 @@ use log::{debug, warn};/// # fn table_schema(&self) -> &TableSchema { &self.table_schema }/// # fn with_batch_size(&self, _: usize) -> Arc<dyn FileSource> { unimplemented!() }/// # fn with_projection(&self, _: &FileScanConfig) -> Arc<dyn FileSource> { unimplemented!() }-/// # fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> { Arc::new(Self {table_schema: self.table_schema.clone(), projected_statistics: Some(statistics), schema_adapter_factory: self.schema_adapter_factory.clone()} ) }/// # fn metrics(&self) -> &ExecutionPlanMetricsSet { unimplemented!() }-/// # fn statistics(&self) -> Result<Statistics> { Ok(self.projected_statistics.clone().expect("projected_statistics should be set")) }/// # fn file_type(&self) -> &str { "parquet" }-/// # fn with_schema_adapter_factory(&self, factory: Arc<dyn SchemaAdapterFactory>) -> Result<Arc<dyn FileSource>> { Ok(Arc::new(Self {table_schema: self.table_schema.clone(), projected_statistics: self.projected_statistics.clone(), schema_adapter_factory: Some(factory)} )) }+/// # fn with_schema_adapter_factory(&self, factory: Arc<dyn SchemaAdapterFactory>) -> Result<Arc<dyn FileSource>> { Ok(Arc::new(Self {table_schema: self.table_schema.clone(), schema_adapter_factory: Some(factory)} )) }/// # fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> { self.schema_adapter_factory.clone() }/// # }/// # impl ParquetSource {-/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into(), projected_statistics: None, schema_adapter_factory: None} }+/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into(), schema_adapter_factory: None} }/// # }/// // create FileScan config for reading parquet files from file:///// let object_store_url = ObjectStoreUrl::local_filesystem();@@ -192,6 +189,13 @@ pub struct FileScanConfig {/// Expression adapter used to adapt filters and projections that are pushed down into the scan/// from the logical schema to the physical schema of the file.pub expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,+ /// Unprojected statistics for the table (file schema + partition columns).+ /// These are projected on-demand via `projected_stats()`.+ ///+ /// Note that this field is pub(crate) because accessing it directly from outside+ /// would be incorrect if there are filters being applied, thus this should be accessed+ /// via [`FileScanConfig::statistics`].+ pub(crate) statistics: Statistics,}/// A builder for [`FileScanConfig`]'s.@@ -437,10 +441,9 @@ impl FileScanConfigBuilder {let constraints = constraints.unwrap_or_default();let statistics = statistics.unwrap_or_else(|| {- Statistics::new_unknown(file_source.table_schema().file_schema())+ Statistics::new_unknown(file_source.table_schema().table_schema())});- let file_source = file_source.with_statistics(statistics.clone());let file_compression_type =file_compression_type.unwrap_or(FileCompressionType::UNCOMPRESSED);let new_lines_in_values = new_lines_in_values.unwrap_or(false);@@ -466,6 +469,7 @@ impl FileScanConfigBuilder {new_lines_in_values,batch_size,expr_adapter_factory: expr_adapter,+ statistics,}}}@@ -476,7 +480,7 @@ impl From<FileScanConfig> for FileScanConfigBuilder {object_store_url: config.object_store_url,file_source: Arc::<dyn FileSource>::clone(&config.file_source),file_groups: config.file_groups,- statistics: config.file_source.statistics().ok(),+ statistics: Some(config.statistics),output_ordering: config.output_ordering,file_compression_type: Some(config.file_compression_type),new_lines_in_values: Some(config.new_lines_in_values),@@ -736,8 +740,21 @@ impl FileScanConfig {}}- pub fn projected_stats(&self) -> Statistics {- let statistics = self.file_source.statistics().unwrap();+ /// Returns the unprojected table statistics, marking them as inexact if filters are present.+ ///+ /// When filters are pushed down (including pruning predicates and bloom filters),+ /// we can't guarantee the statistics are exact because we don't know how many+ /// rows will be filtered out.+ pub fn statistics(&self) -> Statistics {+ if self.file_source.filter().is_some() {+ self.statistics.clone().to_inexact()+ } else {+ self.statistics.clone()+ }+ }++ fn projected_stats(&self) -> Statistics {+ let statistics = self.statistics();let table_cols_stats = self.projection_indices()@@ -830,7 +847,7 @@ impl FileScanConfig {return (Arc::clone(self.file_schema()),self.constraints.clone(),- self.file_source.statistics().unwrap().clone(),+ self.statistics().clone(),self.output_ordering.clone(),);}@@ -1064,11 +1081,7 @@ impl Debug for FileScanConfig {write!(f, "FileScanConfig {{")?;write!(f, "object_store_url={:?}, ", self.object_store_url)?;- write!(- f,- "statistics={:?}, ",- self.file_source.statistics().unwrap()- )?;+ write!(f, "statistics={:?}, ", self.statistics())?;DisplayAs::fmt_as(self, DisplayFormatType::Verbose, f)?;write!(f, "}}")@@ -1647,7 +1660,7 @@ mod tests {to_partition_cols(partition_cols.clone()),);- let source_statistics = conf.file_source.statistics().unwrap();+ let source_statistics = conf.statistics();let conf_stats = conf.partition_statistics(None).unwrap();// projection should be reflected in the file source statistics@@ -2347,24 +2360,13 @@ mod tests {assert!(config.constraints.is_empty());// Verify statistics are set to unknown+ assert_eq!(config.statistics().num_rows, Precision::Absent);+ assert_eq!(config.statistics().total_byte_size, Precision::Absent);assert_eq!(- config.file_source.statistics().unwrap().num_rows,- Precision::Absent- );- assert_eq!(- config.file_source.statistics().unwrap().total_byte_size,- Precision::Absent- );- assert_eq!(- config- .file_source- .statistics()- .unwrap()- .column_statistics- .len(),+ config.statistics().column_statistics.len(),file_schema.fields().len());- for stat in config.file_source.statistics().unwrap().column_statistics {+ for stat in config.statistics().column_statistics {assert_eq!(stat.distinct_count, Precision::Absent);assert_eq!(stat.min_value, Precision::Absent);assert_eq!(stat.max_value, Precision::Absent);diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rsindex 78ba593f22..5d5b277dcf 100644--- a/datafusion/datasource/src/test_util.rs+++ b/datafusion/datasource/src/test_util.rs@@ -23,7 +23,7 @@ use crate::{use std::sync::Arc;use arrow::datatypes::Schema;-use datafusion_common::{Result, Statistics};+use datafusion_common::Result;use datafusion_physical_expr::{expressions::Column, PhysicalExpr};use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use object_store::ObjectStore;@@ -32,7 +32,6 @@ use object_store::ObjectStore;#[derive(Clone)]pub(crate) struct MockSource {metrics: ExecutionPlanMetricsSet,- projected_statistics: Option<Statistics>,schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,filter: Option<Arc<dyn PhysicalExpr>>,table_schema: crate::table_schema::TableSchema,@@ -42,7 +41,6 @@ impl Default for MockSource {fn default() -> Self {Self {metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,filter: None,table_schema: crate::table_schema::TableSchema::new(@@ -57,7 +55,6 @@ impl MockSource {pub fn new(table_schema: impl Into<crate::table_schema::TableSchema>) -> Self {Self {metrics: ExecutionPlanMetricsSet::new(),- projected_statistics: None,schema_adapter_factory: None,filter: None,table_schema: table_schema.into(),@@ -96,24 +93,10 @@ impl FileSource for MockSource {Arc::new(Self { ..self.clone() })}- fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {- let mut source = self.clone();- source.projected_statistics = Some(statistics);- Arc::new(source)- }-fn metrics(&self) -> &ExecutionPlanMetricsSet {&self.metrics}- fn statistics(&self) -> Result<Statistics> {- Ok(self- .projected_statistics- .as_ref()- .expect("projected_statistics must be set")- .clone())- }-fn file_type(&self) -> &str {"mock"}diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 820628ff70..1ae85618b9 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -530,7 +530,7 @@ pub fn serialize_file_scan_config(Ok(protobuf::FileScanExecConf {file_groups,- statistics: Some((&conf.file_source.statistics().unwrap()).into()),+ statistics: Some((&conf.statistics()).into()),limit: conf.limit.map(|l| protobuf::ScanLimit { limit: l as u32 }),projection: conf.projection_exprsdiff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 73f39eaa7b..c50f41625c 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -53,7 +53,7 @@ use datafusion::datasource::listing::{use datafusion::datasource::object_store::ObjectStoreUrl;use datafusion::datasource::physical_plan::{wrap_partition_type_in_dict, wrap_partition_value_in_dict, FileGroup,- FileScanConfigBuilder, FileSinkConfig, FileSource, ParquetSource,+ FileScanConfigBuilder, FileSinkConfig, ParquetSource,};use datafusion::datasource::sink::DataSinkExec;use datafusion::datasource::source::DataSourceExec;@@ -1805,8 +1805,7 @@ async fn roundtrip_projection_source() -> Result<()> {let statistics = Statistics::new_unknown(&schema);- let file_source =- ParquetSource::new(Arc::clone(&schema)).with_statistics(statistics.clone());+ let file_source = Arc::new(ParquetSource::new(Arc::clone(&schema)));let scan_config =FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source).with_file_groups(vec+### Statistics handling moved from `FileSource` to `FileScanConfig`++Statistics are now managed directly by `FileScanConfig` instead of being delegated to `FileSource` implementations. This simplifies the `FileSource` trait and provides more consistent statistics handling across all file formats.++**Who is affected:**++- Users who have implemented custom `FileSource` implementations++**Breaking changes:**++Two methods have been removed from the `FileSource` trait:++- `with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource>`+- `statistics(&self) -> Result<Statistics>`++**Migration guide:**++If you have a custom `FileSource` implementation, you need to:++1. Remove the `with_statistics` method implementation+2. Remove the `statistics` method implementation+3. Remove any internal state that was storing statistics++**Before:**++```rust,ignore+#[derive(Clone)]+struct MyCustomSource {+ table_schema: TableSchema,+ projected_statistics: Option<Statistics>,+ // other fields...+}++impl FileSource for MyCustomSource {+ fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource> {+ Arc::new(Self {+ table_schema: self.table_schema.clone(),+ projected_statistics: Some(statistics),+ // other fields...+ })+ }++ fn statistics(&self) -> Result<Statistics> {+ Ok(self.projected_statistics.clone().unwrap_or_else(||+ Statistics::new_unknown(self.table_schema.file_schema())+ ))+ }++ // other methods...+}+```++**After:**++```rust,ignore+#[derive(Clone)]+struct MyCustomSource {+ table_schema: TableSchema,+ // projected_statistics field removed+ // other fields...+}++impl FileSource for MyCustomSource {+ // with_statistics method removed+ // statistics method removed++ // other methods...+}+```++**Accessing statistics:**++Statistics are now accessed through `FileScanConfig` instead of `FileSource`:++```diff+- let stats = config.file_source.statistics()?;++ let stats = config.statistics();+```++Note that `FileScanConfig::statistics()` automatically marks statistics as inexact when filters are present, ensuring correctness when filters are pushed down.+### Planner now requires explicit opt-in for WITHIN GROUP syntaxThe SQL planner now enforces the aggregate UDF contract more strictly: the