Apache DataFusion SQL Query Engine
use ProjectionExprs:project_statistics in FileScanConfig (#19094)
Adrian Garcia Badaracco03c1a8df9e82
diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rsindex cf0143ebf1..1cd579365a 100644--- a/datafusion/common/src/stats.rs+++ b/datafusion/common/src/stats.rs@@ -317,6 +317,31 @@ impl Statistics {}}+ /// Calculates `total_byte_size` based on the schema and `num_rows`.+ /// If any of the columns has non-primitive width, `total_byte_size` is set to inexact.+ pub fn calculate_total_byte_size(&mut self, schema: &Schema) {+ let mut row_size = Some(0);+ for field in schema.fields() {+ match field.data_type().primitive_width() {+ Some(width) => {+ row_size = row_size.map(|s| s + width);+ }+ None => {+ row_size = None;+ break;+ }+ }+ }+ match row_size {+ None => {+ self.total_byte_size = self.total_byte_size.to_inexact();+ }+ Some(size) => {+ self.total_byte_size = self.num_rows.multiply(&Precision::Exact(size));+ }+ }+ }+/// Returns an unbounded `ColumnStatistics` for each field in the schema.pub fn unknown_column(schema: &Schema) -> Vec<ColumnStatistics> {schemadiff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rsindex 52c5393e10..87131e0824 100644--- a/datafusion/core/src/datasource/file_format/parquet.rs+++ b/datafusion/core/src/datasource/file_format/parquet.rs@@ -724,7 +724,7 @@ mod tests {// TODO correct byte size: https://github.com/apache/datafusion/issues/14936assert_eq!(exec.partition_statistics(None)?.total_byte_size,- Precision::Exact(671)+ Precision::Absent,);Ok(())@@ -770,10 +770,9 @@ mod tests {exec.partition_statistics(None)?.num_rows,Precision::Exact(8));- // TODO correct byte size: https://github.com/apache/datafusion/issues/14936assert_eq!(exec.partition_statistics(None)?.total_byte_size,- Precision::Exact(671)+ Precision::Absent,);let batches = collect(exec, task_ctx).await?;assert_eq!(1, batches.len());diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rsindex 3333b70676..fbb9ed7a3d 100644--- a/datafusion/core/src/datasource/listing/table.rs+++ b/datafusion/core/src/datasource/listing/table.rs@@ -257,7 +257,7 @@ mod tests {);assert_eq!(exec.partition_statistics(None)?.total_byte_size,- Precision::Exact(671)+ Precision::Absent,);Ok(())@@ -1397,7 +1397,7 @@ mod tests {// TODO correct byte size: https://github.com/apache/datafusion/issues/14936assert_eq!(exec_enabled.partition_statistics(None)?.total_byte_size,- Precision::Exact(671)+ Precision::Absent,);Ok(())diff --git a/datafusion/core/tests/parquet/file_statistics.rs b/datafusion/core/tests/parquet/file_statistics.rsindex 22f015d802..cda8c2c448 100644--- a/datafusion/core/tests/parquet/file_statistics.rs+++ b/datafusion/core/tests/parquet/file_statistics.rs@@ -128,8 +128,9 @@ async fn load_table_stats_with_session_level_cache() {);assert_eq!(exec1.partition_statistics(None).unwrap().total_byte_size,- // TODO correct byte size: https://github.com/apache/datafusion/issues/14936- Precision::Exact(671),+ // Byte size is absent because we cannot estimate the output size+ // of the Arrow data since there are variable length columns.+ Precision::Absent,);assert_eq!(get_static_cache_size(&state1), 1);@@ -143,8 +144,8 @@ async fn load_table_stats_with_session_level_cache() {);assert_eq!(exec2.partition_statistics(None).unwrap().total_byte_size,- // TODO correct byte size: https://github.com/apache/datafusion/issues/14936- Precision::Exact(671),+ // Absent because the data contains variable length columns+ Precision::Absent,);assert_eq!(get_static_cache_size(&state2), 1);@@ -158,8 +159,8 @@ async fn load_table_stats_with_session_level_cache() {);assert_eq!(exec3.partition_statistics(None).unwrap().total_byte_size,- // TODO correct byte size: https://github.com/apache/datafusion/issues/14936- Precision::Exact(671),+ // Absent because the data contains variable length columns+ Precision::Absent,);// List same file no increaseassert_eq!(get_static_cache_size(&state1), 1);diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex 12c31b3945..173cbad8ad 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -67,7 +67,7 @@ mod test {/// - Each partition has an "id" column (INT) with the following values:/// - First partition: [3, 4]/// - Second partition: [1, 2]- /// - Each row is 110 bytes in size+ /// - Each partition has 16 bytes total (Int32 id: 4 bytes × 2 rows + Date32 date: 4 bytes × 2 rows)////// @param create_table_sql Optional parameter to set the create table SQL/// @param target_partition Optional parameter to set the target partitions@@ -215,9 +215,9 @@ mod test {.map(|idx| scan.partition_statistics(Some(idx))).collect::<Result<Vec<_>>>()?;let expected_statistic_partition_1 =- create_partition_statistics(2, 110, 3, 4, true);+ create_partition_statistics(2, 16, 3, 4, true);let expected_statistic_partition_2 =- create_partition_statistics(2, 110, 1, 2, true);+ create_partition_statistics(2, 16, 1, 2, true);// Check the statistics of each partitionassert_eq!(statistics.len(), 2);assert_eq!(statistics[0], expected_statistic_partition_1);@@ -277,8 +277,7 @@ mod test {let statistics = (0..sort_exec.output_partitioning().partition_count()).map(|idx| sort_exec.partition_statistics(Some(idx))).collect::<Result<Vec<_>>>()?;- let expected_statistic_partition =- create_partition_statistics(4, 220, 1, 4, true);+ let expected_statistic_partition = create_partition_statistics(4, 32, 1, 4, true);assert_eq!(statistics.len(), 1);assert_eq!(statistics[0], expected_statistic_partition);// Check the statistics_by_partition with real results@@ -292,9 +291,9 @@ mod test {SortExec::new(ordering.into(), scan_2).with_preserve_partitioning(true),);let expected_statistic_partition_1 =- create_partition_statistics(2, 110, 3, 4, true);+ create_partition_statistics(2, 16, 3, 4, true);let expected_statistic_partition_2 =- create_partition_statistics(2, 110, 1, 2, true);+ create_partition_statistics(2, 16, 1, 2, true);let statistics = (0..sort_exec.output_partitioning().partition_count()).map(|idx| sort_exec.partition_statistics(Some(idx))).collect::<Result<Vec<_>>>()?;@@ -366,9 +365,9 @@ mod test {// Check that we have 4 partitions (2 from each scan)assert_eq!(statistics.len(), 4);let expected_statistic_partition_1 =- create_partition_statistics(2, 110, 3, 4, true);+ create_partition_statistics(2, 16, 3, 4, true);let expected_statistic_partition_2 =- create_partition_statistics(2, 110, 1, 2, true);+ create_partition_statistics(2, 16, 1, 2, true);// Verify first partition (from first scan)assert_eq!(statistics[0], expected_statistic_partition_1);// Verify second partition (from first scan)@@ -418,7 +417,7 @@ mod test {let expected_stats = Statistics {num_rows: Precision::Inexact(4),- total_byte_size: Precision::Inexact(220),+ total_byte_size: Precision::Inexact(32),column_statistics: vec![ColumnStatistics::new_unknown(),ColumnStatistics::new_unknown(),@@ -462,7 +461,7 @@ mod test {// Check that we have 2 partitionsassert_eq!(statistics.len(), 2);let mut expected_statistic_partition_1 =- create_partition_statistics(8, 48400, 1, 4, true);+ create_partition_statistics(8, 512, 1, 4, true);expected_statistic_partition_1.column_statistics.push(ColumnStatistics {@@ -473,7 +472,7 @@ mod test {distinct_count: Precision::Absent,});let mut expected_statistic_partition_2 =- create_partition_statistics(8, 48400, 1, 4, true);+ create_partition_statistics(8, 512, 1, 4, true);expected_statistic_partition_2.column_statistics.push(ColumnStatistics {@@ -501,9 +500,9 @@ mod test {let coalesce_batches: Arc<dyn ExecutionPlan> =Arc::new(CoalesceBatchesExec::new(scan, 2));let expected_statistic_partition_1 =- create_partition_statistics(2, 110, 3, 4, true);+ create_partition_statistics(2, 16, 3, 4, true);let expected_statistic_partition_2 =- create_partition_statistics(2, 110, 1, 2, true);+ create_partition_statistics(2, 16, 1, 2, true);let statistics = (0..coalesce_batches.output_partitioning().partition_count()).map(|idx| coalesce_batches.partition_statistics(Some(idx))).collect::<Result<Vec<_>>>()?;@@ -525,8 +524,7 @@ mod test {let scan = create_scan_exec_with_statistics(None, Some(2)).await;let coalesce_partitions: Arc<dyn ExecutionPlan> =Arc::new(CoalescePartitionsExec::new(scan));- let expected_statistic_partition =- create_partition_statistics(4, 220, 1, 4, true);+ let expected_statistic_partition = create_partition_statistics(4, 32, 1, 4, true);let statistics = (0..coalesce_partitions.output_partitioning().partition_count()).map(|idx| coalesce_partitions.partition_statistics(Some(idx))).collect::<Result<Vec<_>>>()?;@@ -575,8 +573,7 @@ mod test {.map(|idx| global_limit.partition_statistics(Some(idx))).collect::<Result<Vec<_>>>()?;assert_eq!(statistics.len(), 1);- let expected_statistic_partition =- create_partition_statistics(2, 110, 3, 4, true);+ let expected_statistic_partition = create_partition_statistics(2, 16, 3, 4, true);assert_eq!(statistics[0], expected_statistic_partition);Ok(())}@@ -627,7 +624,11 @@ mod test {let expected_p0_statistics = Statistics {num_rows: Precision::Inexact(2),- total_byte_size: Precision::Inexact(110),+ // Each row produces 8 bytes of data:+ // - id column: Int32 (4 bytes) × 2 rows = 8 bytes+ // - id + 1 column: Int32 (4 bytes) × 2 rows = 8 bytes+ // AggregateExec cannot yet derive byte sizes for the COUNT(c) column+ total_byte_size: Precision::Inexact(16),column_statistics: vec![ColumnStatistics {null_count: Precision::Absent,@@ -645,7 +646,11 @@ mod test {let expected_p1_statistics = Statistics {num_rows: Precision::Inexact(2),- total_byte_size: Precision::Inexact(110),+ // Each row produces 8 bytes of data:+ // - id column: Int32 (4 bytes) × 2 rows = 8 bytes+ // - id + 1 column: Int32 (4 bytes) × 2 rows = 8 bytes+ // AggregateExec cannot yet derive byte sizes for the COUNT(c) column+ total_byte_size: Precision::Inexact(16),column_statistics: vec![ColumnStatistics {null_count: Precision::Absent,@@ -851,7 +856,7 @@ mod test {let expected_stats = Statistics {num_rows: Precision::Inexact(1),- total_byte_size: Precision::Inexact(73),+ total_byte_size: Precision::Inexact(10),column_statistics: vec![ColumnStatistics::new_unknown(),ColumnStatistics::new_unknown(),@@ -955,7 +960,7 @@ mod test {let expected_stats = Statistics {num_rows: Precision::Inexact(2),- total_byte_size: Precision::Inexact(110),+ total_byte_size: Precision::Inexact(16),column_statistics: vec![ColumnStatistics::new_unknown(),ColumnStatistics::new_unknown(),diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rsindex fcd3a22dcf..0640b19aee 100644--- a/datafusion/datasource-parquet/src/metadata.rs+++ b/datafusion/datasource-parquet/src/metadata.rs@@ -247,10 +247,8 @@ impl<'a> DFParquetMetadata<'a> {let mut statistics = Statistics::new_unknown(table_schema);let mut has_statistics = false;let mut num_rows = 0_usize;- let mut total_byte_size = 0_usize;for row_group_meta in row_groups_metadata {num_rows += row_group_meta.num_rows() as usize;- total_byte_size += row_group_meta.total_byte_size() as usize;if !has_statistics {has_statistics = row_group_meta@@ -260,7 +258,7 @@ impl<'a> DFParquetMetadata<'a> {}}statistics.num_rows = Precision::Exact(num_rows);- statistics.total_byte_size = Precision::Exact(total_byte_size);+ statistics.calculate_total_byte_size(table_schema);let file_metadata = metadata.file_metadata();let mut file_schema = parquet_to_arrow_schema(diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex 925caee162..8976b4efc4 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -28,8 +28,7 @@ use arrow::datatypes::FieldRef;use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;use datafusion_common::{- internal_datafusion_err, internal_err, ColumnStatistics, Constraints, Result,- ScalarValue, Statistics,+ internal_datafusion_err, internal_err, Constraints, Result, ScalarValue, Statistics,};use datafusion_execution::{object_store::ObjectStoreUrl, SendableRecordBatchStream, TaskContext,@@ -647,32 +646,26 @@ impl DataSource for FileScanConfig {if let Some(file_group) = self.file_groups.get(partition) {if let Some(stat) = file_group.file_statistics(None) {// Project the statistics based on the projection- let table_cols_stats = self- .projection_indices()- .into_iter()- .map(|idx| {- if idx < self.file_schema().fields().len() {- stat.column_statistics[idx].clone()- } else {- // TODO provide accurate stat for partition column- // See https://github.com/apache/datafusion/issues/1186- ColumnStatistics::new_unknown()- }- })- .collect();-- return Ok(Statistics {- num_rows: stat.num_rows,- total_byte_size: stat.total_byte_size,- column_statistics: table_cols_stats,- });+ let output_schema = self.projected_schema()?;+ return if let Some(projection) = self.file_source.projection() {+ projection.project_statistics(stat.clone(), &output_schema)+ } else {+ Ok(stat.clone())+ };}}// If no statistics available for this partition, return unknownOk(Statistics::new_unknown(self.projected_schema()?.as_ref()))} else {// Return aggregate statistics across all partitions- Ok(self.projected_stats())+ let statistics = self.statistics();+ let projection = self.file_source.projection();+ let output_schema = self.projected_schema()?;+ if let Some(projection) = &projection {+ projection.project_statistics(statistics.clone(), &output_schema)+ } else {+ Ok(statistics)+ }}}@@ -777,15 +770,6 @@ impl FileScanConfig {self.file_source.table_schema().table_partition_cols()}- fn projection_indices(&self) -> Vec<usize> {- match self.file_source.projection() {- Some(proj) => proj.ordered_column_indices(),- None => (0..self.file_schema().fields().len()- + self.table_partition_cols().len())- .collect(),- }- }-/// Returns the unprojected table statistics, marking them as inexact if filters are present.////// When filters are pushed down (including pruning predicates and bloom filters),@@ -799,30 +783,6 @@ impl FileScanConfig {}}- fn projected_stats(&self) -> Statistics {- let statistics = self.statistics();-- let table_cols_stats = self- .projection_indices()- .into_iter()- .map(|idx| {- if idx < self.file_schema().fields().len() {- statistics.column_statistics[idx].clone()- } else {- // TODO provide accurate stat for partition column (#1186)- ColumnStatistics::new_unknown()- }- })- .collect();-- Statistics {- num_rows: statistics.num_rows,- // TODO correct byte size: https://github.com/apache/datafusion/issues/14936- total_byte_size: statistics.total_byte_size,- column_statistics: table_cols_stats,- }- }-pub fn projected_schema(&self) -> Result<Arc<Schema>> {let schema = self.file_source.table_schema().table_schema();match self.file_source.projection() {@@ -857,9 +817,13 @@ impl FileScanConfig {Ok(())}+ #[deprecated(+ since = "53.0.0",+ note = "This method is no longer used, use eq_properties instead. It will be removed in 58.0.0."+ )]pub fn projected_constraints(&self) -> Constraints {- let indexes = self.projection_indices();- self.constraints.project(&indexes).unwrap_or_default()+ let props = self.eq_properties();+ props.constraints().clone()}/// Specifies whether newlines in (quoted) values are supported.@@ -873,7 +837,12 @@ impl FileScanConfig {self.new_lines_in_values}+ #[deprecated(+ since = "53.0.0",+ note = "Use file_column_projection_indices instead. This method will be removed in 58.0.0."+ )]pub fn file_column_projection_indices(&self) -> Option<Vec<usize>> {+ #[expect(deprecated)]self.file_source.projection().as_ref().map(|p| {p.ordered_column_indices().into_iter()@@ -1090,6 +1059,24 @@ impl DisplayAs for FileScanConfig {}}+/// Get the indices of columns in a projection if the projection is a simple+/// list of columns.+/// If there are any expressions other than columns, returns None.+fn ordered_column_indices_from_projection(+ projection: &ProjectionExprs,+) -> Option<Vec<usize>> {+ projection+ .expr_iter()+ .map(|e| {+ let index = e+ .as_any()+ .downcast_ref::<datafusion_physical_expr::expressions::Column>()?+ .index();+ Some(index)+ })+ .collect::<Option<Vec<usize>>>()+}+/// The various listing tables does not attempt to read all files/// concurrently, instead they will read files in sequence within a/// partition. This is an important property as it allows plans to@@ -1165,11 +1152,15 @@ fn get_projected_output_ordering(return false;}- let indices = base_config+ let Some(indices) = base_config.file_source.projection().as_ref()- .map(|p| p.ordered_column_indices());+ .map(|p| ordered_column_indices_from_projection(p))+ else {+ // Can't determine if ordered without a simple projection+ return true;+ };let statistics = match MinMaxStatistics::new_from_files(&new_ordering,@@ -1234,8 +1225,8 @@ mod tests {};use arrow::datatypes::Field;- use datafusion_common::internal_err;use datafusion_common::stats::Precision;+ use datafusion_common::{internal_err, ColumnStatistics};use datafusion_expr::{Operator, SortExpr};use datafusion_physical_expr::create_physical_sort_expr;use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal};@@ -1271,76 +1262,6 @@ mod tests {);}- #[test]- fn test_projected_file_schema_with_partition_col() {- let schema = aggr_test_schema();- let partition_cols = vec![- (- "part1".to_owned(),- wrap_partition_type_in_dict(DataType::Utf8),- ),- (- "part2".to_owned(),- wrap_partition_type_in_dict(DataType::Utf8),- ),- ];-- // Projected file schema for config with projection including partition column- let config = config_for_projection(- schema.clone(),- Some(vec![0, 3, 5, schema.fields().len()]),- Statistics::new_unknown(&schema),- to_partition_cols(partition_cols),- );- let projection = projected_file_schema(&config);-- // Assert partition column filtered out in projected file schema- let expected_columns = vec!["c1", "c4", "c6"];- let actual_columns = projection- .fields()- .iter()- .map(|f| f.name().clone())- .collect::<Vec<_>>();- assert_eq!(expected_columns, actual_columns);- }-- /// Projects only file schema, ignoring partition columns- fn projected_file_schema(config: &FileScanConfig) -> SchemaRef {- let file_schema = config.file_source.table_schema().file_schema();- if let Some(file_indices) = config.file_column_projection_indices() {- Arc::new(file_schema.project(&file_indices).unwrap())- } else {- Arc::clone(file_schema)- }- }-- #[test]- fn test_projected_file_schema_without_projection() {- let schema = aggr_test_schema();- let partition_cols = vec![- (- "part1".to_owned(),- wrap_partition_type_in_dict(DataType::Utf8),- ),- (- "part2".to_owned(),- wrap_partition_type_in_dict(DataType::Utf8),- ),- ];-- // Projected file schema for config without projection- let config = config_for_projection(- schema.clone(),- None,- Statistics::new_unknown(&schema),- to_partition_cols(partition_cols),- );- let projection = projected_file_schema(&config);-- // Assert projected file schema is equal to file schema- assert_eq!(projection.fields(), schema.fields());- }-#[test]fn test_split_groups_by_statistics() -> Result<()> {use chrono::TimeZone;@@ -1667,14 +1588,6 @@ mod tests {.build()}- /// Convert partition columns from Vec<String DataType> to Vec<Field>- fn to_partition_cols(table_partition_cols: Vec<(String, DataType)>) -> Vec<Field> {- table_partition_cols- .iter()- .map(|(name, dtype)| Field::new(name, dtype.clone(), false))- .collect::<Vec<_>>()- }-#[test]fn test_file_scan_config_builder() {let file_schema = aggr_test_schema();@@ -2154,8 +2067,8 @@ mod tests {"Second projected column should be col2 with 10 nulls");- // Verify row count and byte size are preserved+ // Verify row count and byte sizeassert_eq!(partition_stats.num_rows, Precision::Exact(100));- assert_eq!(partition_stats.total_byte_size, Precision::Exact(1024));+ assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));}}diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex 4688ac0e1b..0f8929bb06 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -24,7 +24,7 @@ use crate::PhysicalExpr;use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::datatypes::{Field, Schema, SchemaRef};-use datafusion_common::stats::{ColumnStatistics, Precision};+use datafusion_common::stats::ColumnStatistics;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{assert_or_internal_err, internal_datafusion_err, plan_err, Result,@@ -407,6 +407,14 @@ impl ProjectionExprs {////// Use [`column_indices()`](Self::column_indices) instead if the projection may contain/// non-column expressions or if you need a deduplicated sorted list.+ ///+ /// # Panics+ ///+ /// Panics if any expression in the projection is not a simple column reference.+ #[deprecated(+ since = "53.0.0",+ note = "Use column_indices() instead. This method will be removed in 58.0.0."+ )]pub fn ordered_column_indices(&self) -> Vec<usize> {self.exprs.iter()@@ -471,37 +479,120 @@ impl ProjectionExprs {/// Project statistics according to this projection./// For example, for a projection `SELECT a AS x, b + 1 AS y`, where `a` is at index 0 and `b` is at index 1,/// if the input statistics has column statistics for columns `a`, `b`, and `c`, the output statistics would have column statistics for columns `x` and `y`.+ ///+ /// # Example+ ///+ /// ```rust+ /// use arrow::datatypes::{DataType, Field, Schema};+ /// use datafusion_common::stats::{ColumnStatistics, Precision, Statistics};+ /// use datafusion_physical_expr::projection::ProjectionExprs;+ /// use datafusion_common::Result;+ /// use datafusion_common::ScalarValue;+ /// use std::sync::Arc;+ ///+ /// fn main() -> Result<()> {+ /// // Input schema: a: Int32, b: Int32, c: Int32+ /// let input_schema = Arc::new(Schema::new(vec![+ /// Field::new("a", DataType::Int32, false),+ /// Field::new("b", DataType::Int32, false),+ /// Field::new("c", DataType::Int32, false),+ /// ]));+ ///+ /// // Input statistics with column stats for a, b, c+ /// let input_stats = Statistics {+ /// num_rows: Precision::Exact(100),+ /// total_byte_size: Precision::Exact(1200),+ /// column_statistics: vec![+ /// // Column a stats+ /// ColumnStatistics {+ /// null_count: Precision::Exact(0),+ /// distinct_count: Precision::Exact(100),+ /// min_value: Precision::Exact(ScalarValue::Int32(Some(0))),+ /// max_value: Precision::Exact(ScalarValue::Int32(Some(100))),+ /// sum_value: Precision::Absent,+ /// },+ /// // Column b stats+ /// ColumnStatistics {+ /// null_count: Precision::Exact(0),+ /// distinct_count: Precision::Exact(50),+ /// min_value: Precision::Exact(ScalarValue::Int32(Some(10))),+ /// max_value: Precision::Exact(ScalarValue::Int32(Some(60))),+ /// sum_value: Precision::Absent,+ /// },+ /// // Column c stats+ /// ColumnStatistics {+ /// null_count: Precision::Exact(5),+ /// distinct_count: Precision::Exact(25),+ /// min_value: Precision::Exact(ScalarValue::Int32(Some(-10))),+ /// max_value: Precision::Exact(ScalarValue::Int32(Some(200))),+ /// sum_value: Precision::Absent,+ /// },+ /// ],+ /// };+ ///+ /// // Create a projection that selects columns c and a (indices 2 and 0)+ /// let projection = ProjectionExprs::from_indices(&[2, 0], &input_schema);+ ///+ /// // Compute output schema+ /// let output_schema = projection.project_schema(&input_schema)?;+ ///+ /// // Project the statistics+ /// let output_stats = projection.project_statistics(input_stats, &output_schema)?;+ ///+ /// // The output should have 2 column statistics (for c and a, in that order)+ /// assert_eq!(output_stats.column_statistics.len(), 2);+ ///+ /// // First column in output is c (was at index 2)+ /// assert_eq!(+ /// output_stats.column_statistics[0].min_value,+ /// Precision::Exact(ScalarValue::Int32(Some(-10)))+ /// );+ /// assert_eq!(+ /// output_stats.column_statistics[0].null_count,+ /// Precision::Exact(5)+ /// );+ ///+ /// // Second column in output is a (was at index 0)+ /// assert_eq!(+ /// output_stats.column_statistics[1].min_value,+ /// Precision::Exact(ScalarValue::Int32(Some(0)))+ /// );+ /// assert_eq!(+ /// output_stats.column_statistics[1].distinct_count,+ /// Precision::Exact(100)+ /// );+ ///+ /// // Total byte size is recalculated based on projected columns+ /// assert_eq!(+ /// output_stats.total_byte_size,+ /// Precision::Exact(800), // each Int32 column is 4 bytes * 100 rows * 2 columns+ /// );+ ///+ /// // Number of rows remains the same+ /// assert_eq!(output_stats.num_rows, Precision::Exact(100));+ ///+ /// Ok(())+ /// }+ /// ```pub fn project_statistics(&self,mut stats: datafusion_common::Statistics,- input_schema: &Schema,+ output_schema: &Schema,) -> Result<datafusion_common::Statistics> {- let mut primitive_row_size = 0;- let mut primitive_row_size_possible = true;let mut column_statistics = vec![];for proj_expr in &self.exprs {let expr = &proj_expr.expr;let col_stats = if let Some(col) = expr.as_any().downcast_ref::<Column>() {- stats.column_statistics[col.index()].clone()+ std::mem::take(&mut stats.column_statistics[col.index()])} else {// TODO stats: estimate more statistics from expressions// (expressions should compute their statistics themselves)ColumnStatistics::new_unknown()};column_statistics.push(col_stats);- let data_type = expr.data_type(input_schema)?;- if let Some(value) = data_type.primitive_width() {- primitive_row_size += value;- continue;- }- primitive_row_size_possible = false;- }-- if primitive_row_size_possible {- stats.total_byte_size =- Precision::Exact(primitive_row_size).multiply(&stats.num_rows);}+ stats.calculate_total_byte_size(output_schema);stats.column_statistics = column_statistics;Ok(stats)}@@ -908,6 +999,7 @@ pub(crate) mod tests {use arrow::compute::SortOptions;use arrow::datatypes::{DataType, Field, Schema, TimeUnit};use datafusion_common::config::ConfigOptions;+ use datafusion_common::stats::Precision;use datafusion_common::{ScalarValue, Statistics};use datafusion_expr::{Operator, ScalarUDF};use insta::assert_snapshot;@@ -1816,11 +1908,15 @@ pub(crate) mod tests {},]);- let result = projection.project_statistics(source, &schema).unwrap();+ let result = projection+ .project_statistics(source, &projection.project_schema(&schema).unwrap())+ .unwrap();let expected = Statistics {num_rows: Precision::Exact(5),- total_byte_size: Precision::Exact(23),+ // Because there is a variable length Utf8 column we cannot calculate exact byte size after projection+ // Thus we set it to Inexact (originally it was Exact(23))+ total_byte_size: Precision::Inexact(23),column_statistics: vec![ColumnStatistics {distinct_count: Precision::Exact(1),@@ -1858,7 +1954,9 @@ pub(crate) mod tests {},]);- let result = projection.project_statistics(source, &schema).unwrap();+ let result = projection+ .project_statistics(source, &projection.project_schema(&schema).unwrap())+ .unwrap();let expected = Statistics {num_rows: Precision::Exact(5),@@ -2343,7 +2441,10 @@ pub(crate) mod tests {},]);- let output_stats = projection.project_statistics(input_stats, &input_schema)?;+ let output_stats = projection.project_statistics(+ input_stats,+ &projection.project_schema(&input_schema)?,+ )?;// Row count should be preservedassert_eq!(output_stats.num_rows, Precision::Exact(5));@@ -2395,7 +2496,10 @@ pub(crate) mod tests {},]);- let output_stats = projection.project_statistics(input_stats, &input_schema)?;+ let output_stats = projection.project_statistics(+ input_stats,+ &projection.project_schema(&input_schema)?,+ )?;// Row count should be preservedassert_eq!(output_stats.num_rows, Precision::Exact(5));@@ -2439,7 +2543,10 @@ pub(crate) mod tests {},]);- let output_stats = projection.project_statistics(input_stats, &input_schema)?;+ let output_stats = projection.project_statistics(+ input_stats,+ &projection.project_schema(&input_schema)?,+ )?;// Row count should be preservedassert_eq!(output_stats.num_rows, Precision::Exact(5));@@ -2461,7 +2568,10 @@ pub(crate) mod tests {let projection = ProjectionExprs::new(vec![]);- let output_stats = projection.project_statistics(input_stats, &input_schema)?;+ let output_stats = projection.project_statistics(+ input_stats,+ &projection.project_schema(&input_schema)?,+ )?;// Row count should be preservedassert_eq!(output_stats.num_rows, Precision::Exact(5));diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex c256bcbcd6..022842bbe7 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -305,9 +305,10 @@ impl ExecutionPlan for ProjectionExec {fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {let input_stats = self.input.partition_statistics(partition)?;+ let output_schema = self.schema();self.projector.projection()- .project_statistics(input_stats, &self.input.schema())+ .project_statistics(input_stats, &output_schema)}fn supports_limit_pushdown(&self) -> bool {diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex 918c01b561..de4e5325d6 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -287,22 +287,22 @@ CREATE EXTERNAL TABLE alltypes_plain STORED AS PARQUET LOCATION '../../parquet-tquery TTEXPLAIN SELECT * FROM alltypes_plain limit 10;-----physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]# explain verbose with both collect & show statistics onquery TTEXPLAIN VERBOSE SELECT * FROM alltypes_plain limit 10;----initial_physical_plan-01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]initial_physical_plan_with_schema01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N]physical_plan after OutputRequirements-01)OutputRequirementExec: order_by=[], dist_by=Unspecified, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]-02)--GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+01)OutputRequirementExec: order_by=[], dist_by=Unspecified, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+02)--GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]physical_plan after aggregate_statistics SAME TEXT AS ABOVEphysical_plan after join_selection SAME TEXT AS ABOVEphysical_plan after LimitedDistinctAggregation SAME TEXT AS ABOVE@@ -314,16 +314,16 @@ physical_plan after OptimizeAggregateOrder SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVEphysical_plan after coalesce_batches SAME TEXT AS ABOVEphysical_plan after OutputRequirements-01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]physical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVE-physical_plan after LimitPushdown DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+physical_plan after LimitPushdown DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]physical_plan after ProjectionPushdown SAME TEXT AS ABOVEphysical_plan after EnsureCooperative SAME TEXT AS ABOVEphysical_plan after FilterPushdown(Post) SAME TEXT AS ABOVEphysical_plan after SanityCheckPlan SAME TEXT AS ABOVE-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]physical_plan_with_schema DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N]@@ -338,8 +338,8 @@ initial_physical_plan01)GlobalLimitExec: skip=0, fetch=1002)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquetinitial_physical_plan_with_stats-01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]initial_physical_plan_with_schema01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N]@@ -368,7 +368,7 @@ physical_plan after EnsureCooperative SAME TEXT AS ABOVEphysical_plan after FilterPushdown(Post) SAME TEXT AS ABOVEphysical_plan after SanityCheckPlan SAME TEXT AS ABOVEphysical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet-physical_plan_with_stats DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]+physical_plan_with_stats DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]]physical_plan_with_schema DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N]diff --git a/datafusion/sqllogictest/test_files/listing_table_statistics.slt b/datafusion/sqllogictest/test_files/listing_table_statistics.sltindex 37daf551c2..233c2ff589 100644--- a/datafusion/sqllogictest/test_files/listing_table_statistics.slt+++ b/datafusion/sqllogictest/test_files/listing_table_statistics.slt@@ -35,7 +35,7 @@ query TTexplain format indent select * from t;----logical_plan TableScan: t projection=[int_col, str_col]-physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/2.parquet]]}, projection=[int_col, str_col], file_type=parquet, statistics=[Rows=Exact(4), Bytes=Exact(212), [(Col[0]: Min=Exact(Int64(-1)) Max=Exact(Int64(3)) Null=Exact(0)),(Col[1]: Min=Exact(Utf8View("a")) Max=Exact(Utf8View("d")) Null=Exact(0))]]+physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/2.parquet]]}, projection=[int_col, str_col], file_type=parquet, statistics=[Rows=Exact(4), Bytes=Absent, [(Col[0]: Min=Exact(Int64(-1)) Max=Exact(Int64(3)) Null=Exact(0)),(Col[1]: Min=Exact(Utf8View("a")) Max=Exact(Utf8View("d")) Null=Exact(0))]]statement okdrop table t;diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex 14cf4b2802..b3796cd551 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -59,9 +59,9 @@ query TTEXPLAIN SELECT * FROM test_table WHERE column1 = 1;----physical_plan-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(31), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0))]]-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(121), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(121), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0))]]+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]# cleanupstatement ok@@ -84,9 +84,9 @@ query TTEXPLAIN SELECT * FROM test_table WHERE column1 = 1;----physical_plan-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(31), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0))]]-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(121), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(121), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0))]]+02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]]# cleanupstatement ok