Apache DataFusion SQL Query Engine
fix: only propagate cast statistics through safe conversions (#25227)
Huaijinccfe70480630
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rsindex 8e49b6ff48..9a2e4ab648 100644--- a/datafusion/physical-expr/src/expressions/cast.rs+++ b/datafusion/physical-expr/src/expressions/cast.rs@@ -248,8 +248,12 @@ impl CastExpr {})}- /// Check if casting from the specified source type to the target type is a- /// widening cast (e.g. from `Int8` to `Int16`).+ /// Check if casting from the source type to the target type is known to be+ /// lossless and strictly order-preserving for all source values, preserving nulls.+ /// This includes widening casts (e.g. `Int8` to `Int16`) and representation+ /// conversions such as `Int32` to `Date32`, which interprets the same integer+ /// as days since the epoch, or `Int64` to `Date64`, which interprets the same+ /// integer as milliseconds since the epoch.pub fn check_bigger_cast(cast_type: &DataType, src: &DataType) -> bool {if cast_type.eq(src) {return true;@@ -259,6 +263,10 @@ impl CastExpr {(Int8, Int16 | Int32 | Int64)| (Int16, Int32 | Int64)| (Int32, Int64)+ | (Int32, Date32)+ | (Date32, Int32)+ | (Int64, Date64)+ | (Date64, Int64)| (UInt8, UInt16 | UInt32 | UInt64)| (UInt16, UInt32 | UInt64)| (UInt32, UInt64)@@ -268,7 +276,8 @@ impl CastExpr {)}- /// Check if the cast is a widening cast (e.g. from `Int8` to `Int16`).+ /// Check if the cast is lossless and strictly order-preserving for all source+ /// values, preserving nulls. See [`Self::check_bigger_cast`].pub fn is_bigger_cast(&self, src: &DataType) -> bool {Self::check_bigger_cast(self.cast_type(), src)}@@ -289,8 +298,8 @@ pub(crate) fn cast_expr_properties() -> Result<ExprProperties> {let unbounded = Interval::make_unbounded(target_type)?;let source_type = child.range.data_type();- // A widening cast is additionally one-to-one, so it is strictly- // order-preserving; a narrowing cast may collapse distinct values,+ // A lossless cast recognized by check_bigger_cast is one-to-one, so it is+ // strictly order-preserving; a narrowing cast may collapse distinct values,// breaking the ordering of subsequent sort keys.let bigger_cast = CastExpr::check_bigger_cast(target_type, &source_type);if is_order_preserving_cast_family(&source_type, target_type) || bigger_cast {@@ -1505,6 +1514,60 @@ mod tests {Ok(())}+ #[test]+ fn test_integer_date_cast_preserves_values_and_ordering() {+ use arrow::array::{Date32Array, Date64Array};+ use arrow::compute::SortOptions;+ use datafusion_expr_common::sort_properties::SortProperties;++ let values = vec![None, Some(i32::MIN), Some(-1), Some(0), Some(i32::MAX)];+ let integers: ArrayRef = Arc::new(Int32Array::from(values.clone()));+ let dates: ArrayRef = Arc::new(Date32Array::from(values));+ let values = vec![None, Some(i64::MIN), Some(-1), Some(0), Some(i64::MAX)];+ let integers64: ArrayRef = Arc::new(Int64Array::from(values.clone()));+ let dates64: ArrayRef = Arc::new(Date64Array::from(values));+ for (input, expected) in [+ (Arc::clone(&integers), Arc::clone(&dates)),+ (dates, integers),+ (Arc::clone(&integers64), Arc::clone(&dates64)),+ (dates64, integers64),+ ] {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "a",+ input.data_type().clone(),+ true,+ )]));+ let expr = CastExpr::new(+ col("a", &schema).expect("column exists"),+ expected.data_type().clone(),+ None,+ );+ assert!(expr.is_bigger_cast(input.data_type()));+ let child = ExprProperties::new_unknown()+ .with_range(+ Interval::make_unbounded(input.data_type())+ .expect("supported interval type"),+ )+ .with_order(SortProperties::Ordered(SortOptions::default()))+ .with_strictly_order_preserving(true);+ let properties = expr+ .get_properties(std::slice::from_ref(&child))+ .expect("cast properties");+ assert_eq!(properties.sort_properties, child.sort_properties);+ assert!(properties.strictly_order_preserving);+ assert_eq!(properties.range.data_type(), *expected.data_type());++ let batch =+ RecordBatch::try_new(schema, vec![input]).expect("valid input batch");+ let actual = expr+ .evaluate(&batch)+ .expect("cast succeeds")+ .into_array(batch.num_rows())+ .expect("array result");+ assert_eq!(actual.as_ref(), expected.as_ref());+ }+ }+#[test]fn test_check_bigger_cast_precision_loss() {use DataType::*;diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex 0e8876f017..6a038b429b 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -872,15 +872,39 @@ fn project_column_statistics_through_expr(return inner_stats;}+ let min_value = inner_stats+ .min_value+ .cast_to(target_type)+ .unwrap_or(Precision::Absent);+ let max_value = inner_stats+ .max_value+ .cast_to(target_type)+ .unwrap_or(Precision::Absent);+ let source_type = inner_stats+ .min_value+ .get_value()+ .or_else(|| inner_stats.max_value.get_value())+ .map(ScalarValue::data_type);+ // Copy extrema only for casts that preserve order and cannot discard values+ // or fail within the input domain. Copying string endpoints into a numeric+ // domain, for example, does not bound the converted column. Merely casting+ // a failing endpoint to NULL also cannot establish the remaining extrema.+ let preserves_values = source_type.is_some_and(|source_type| {+ CastExpr::check_bigger_cast(target_type, &source_type)+ || is_within_extrema(+ &inner_stats.min_value,+ &inner_stats.max_value,+ &min_value,+ &max_value,+ )+ });+ if !preserves_values {+ return ColumnStatistics::new_unknown();+ }+ColumnStatistics {- min_value: inner_stats- .min_value- .cast_to(target_type)- .unwrap_or(Precision::Absent),- max_value: inner_stats- .max_value- .cast_to(target_type)- .unwrap_or(Precision::Absent),+ min_value,+ max_value,null_count: inner_stats.null_count,distinct_count: inner_stats.distinct_count,sum_value: Precision::Absent,@@ -888,6 +912,23 @@ fn project_column_statistics_through_expr(}}+/// Whether an integer cast preserves every value between the source extrema.+/// The converted extrema must come from casting the corresponding source bounds.+/// Exact, non-null integer bounds that both cast successfully prove the entire+/// range fits in the target type. For example, Int64 [-128, 127] fits in Int8,+/// whereas [-129, 127] does not. Inexact bounds cannot establish exact extrema.+fn is_within_extrema(+ lower: &Precision<ScalarValue>,+ upper: &Precision<ScalarValue>,+ min: &Precision<ScalarValue>,+ max: &Precision<ScalarValue>,+) -> bool {+ [lower, upper, min, max].into_iter().all(|bound| {+ matches!(bound, Precision::Exact(value)+ if value.data_type().is_integer() && !value.is_null())+ })+}+fn column_statistics_at(column_stats: &[ColumnStatistics],index: usize,@@ -2249,6 +2290,197 @@ pub(crate) mod tests {Ok(())}+ #[test]+ fn test_project_statistics_non_monotonic_cast() {+ let input_schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]);+ let mut stats = Statistics::new_unknown(&input_schema);+ stats.num_rows = Precision::Exact(3);+ stats.column_statistics[0].min_value = Precision::Exact(ScalarValue::from("1"));+ stats.column_statistics[0].max_value = Precision::Exact(ScalarValue::from("2"));+ let expr: Arc<dyn PhysicalExpr> = Arc::new(CastExpr::new(+ Arc::new(Column::new("a", 0)),+ DataType::Int32,+ None,+ ));+ let projection = ProjectionExprs::new(vec![ProjectionExpr {+ expr: Arc::clone(&expr),+ alias: "x".to_string(),+ }]);+ let out = projection+ .project_statistics(+ stats,+ &projection+ .project_schema(&input_schema)+ .expect("valid projection schema"),+ )+ .expect("statistics projection succeeds");+ let batch = RecordBatch::try_new(+ Arc::new(input_schema),+ vec![Arc::new(arrow::array::StringArray::from(vec![+ "1", "100", "2",+ ]))],+ )+ .expect("valid input batch");+ let actual = expr+ .evaluate(&batch)+ .expect("cast succeeds")+ .into_array(3)+ .expect("array result");+ assert_eq!(out.column_statistics[0].max_value, Precision::Absent);+ assert_eq!(out.column_statistics[0].min_value, Precision::Absent);+ assert_eq!(+ ScalarValue::try_from_array(&actual, 1).expect("valid scalar value"),+ ScalarValue::Int32(Some(100))+ );+ }++ #[test]+ fn test_is_within_extrema() {+ use Precision::{Absent, Exact, Inexact};+ use ScalarValue::{Int8, Int64, UInt8};++ for (lower, upper, target_type, expected) in [+ (+ Exact(Int64(Some(-128))),+ Exact(Int64(Some(127))),+ DataType::Int8,+ true,+ ),+ (+ Exact(Int64(Some(-129))),+ Exact(Int64(Some(127))),+ DataType::Int8,+ false,+ ),+ (+ Exact(Int64(Some(-128))),+ Exact(Int64(Some(128))),+ DataType::Int8,+ false,+ ),+ (+ Exact(Int64(Some(0))),+ Exact(Int64(Some(255))),+ DataType::UInt8,+ true,+ ),+ (+ Exact(Int64(Some(-1))),+ Exact(Int64(Some(255))),+ DataType::UInt8,+ false,+ ),+ (+ Exact(UInt8(Some(0))),+ Exact(UInt8(Some(127))),+ DataType::Int8,+ true,+ ),+ (+ Exact(UInt8(Some(0))),+ Exact(UInt8(Some(128))),+ DataType::Int8,+ false,+ ),+ (+ Exact(ScalarValue::from("1")),+ Exact(ScalarValue::from("2")),+ DataType::Int8,+ false,+ ),+ (+ Exact(Int64(Some(1))),+ Exact(Int64(Some(2))),+ DataType::Utf8,+ false,+ ),+ ] {+ let min = lower.cast_to(&target_type).unwrap_or(Absent);+ let max = upper.cast_to(&target_type).unwrap_or(Absent);+ assert_eq!(+ is_within_extrema(&lower, &upper, &min, &max),+ expected,+ "{lower:?}..{upper:?} -> {target_type:?}",+ );+ }++ // Every bound must be exact and non-null, including converted endpoints.+ let bounds = [+ Exact(Int64(Some(-128))),+ Exact(Int64(Some(127))),+ Exact(Int8(Some(-128))),+ Exact(Int8(Some(127))),+ ];+ for index in 0..bounds.len() {+ for invalid in [+ Absent,+ Inexact(bounds[index].get_value().unwrap().clone()),+ Exact(+ ScalarValue::try_from(+ &bounds[index].get_value().unwrap().data_type(),+ )+ .unwrap(),+ ),+ ] {+ let mut bounds = bounds.clone();+ bounds[index] = invalid;+ assert!(!is_within_extrema(+ &bounds[0], &bounds[1], &bounds[2], &bounds[3],+ ));+ }+ }+ }++ #[test]+ fn test_project_statistics_narrowing_cast_requires_safe_bounds() {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ for (lower, upper, exact, safe) in [+ (-100, 100, true, true),+ (-200, 100, true, false),+ (-100, 200, true, false),+ (-100, 100, false, false),+ ] {+ let precision = |v| {+ if exact {+ Precision::Exact(ScalarValue::Int32(Some(v)))+ } else {+ Precision::Inexact(ScalarValue::Int32(Some(v)))+ }+ };+ let mut stats = Statistics::new_unknown(&schema);+ stats.column_statistics[0].min_value = precision(lower);+ stats.column_statistics[0].max_value = precision(upper);+ let projection = ProjectionExprs::new(vec![ProjectionExpr::new(+ Arc::new(CastExpr::new(+ Arc::new(Column::new("a", 0)),+ DataType::Int8,+ None,+ )),+ "x",+ )]);+ let output = projection+ .project_statistics(+ stats,+ &projection+ .project_schema(&schema)+ .expect("valid projection schema"),+ )+ .expect("statistics projection succeeds");+ if safe {+ assert_eq!(+ output.column_statistics[0].min_value,+ Precision::Exact(ScalarValue::Int8(Some(-100)))+ );+ assert_eq!(+ output.column_statistics[0].max_value,+ Precision::Exact(ScalarValue::Int8(Some(100)))+ );+ } else {+ assert_eq!(output.column_statistics[0], ColumnStatistics::new_unknown());+ }+ }+ }+fn get_stats() -> Statistics {Statistics {num_rows: Precision::Exact(5),diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex ade96e128c..2c9a29c74b 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -186,6 +186,93 @@ physical_planstatement okDROP TABLE typed_table;+######+# Aggregate statistics through casts+######++# String extrema do not bound the numeric values after a CAST. These aggregates+# must not be replaced with casts of the Parquet string min/max statistics.+statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+COPY (SELECT * FROM (VALUES ('1'), ('100'), ('2')) AS t(a))+TO 'test_files/scratch/parquet_statistics/cast_strings.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE cast_stats_strings STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/cast_strings.parquet';++query II+SELECT MIN(CAST(a AS INT)), MAX(CAST(a AS INT)) FROM cast_stats_strings;+----+1 100++query II+SELECT MIN(CAST(a AS BIGINT)), MAX(CAST(a AS BIGINT)) FROM cast_stats_strings;+----+1 100++statement ok+DROP TABLE cast_stats_strings;++# Exact Int64 extrema that fit in Int8 preserve the aggregate optimization.+statement ok+COPY (SELECT CAST(a AS BIGINT) AS a FROM (VALUES (-128), (0), (127), (NULL)) AS t(a))+TO 'test_files/scratch/parquet_statistics/cast_ints.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE cast_stats_ints STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/cast_ints.parquet';++query II+SELECT MIN(CAST(a AS TINYINT)), MAX(CAST(a AS TINYINT)) FROM cast_stats_ints;+----+-128 127++statement ok+SET datafusion.explain.show_statistics = false;++query TT+EXPLAIN SELECT MIN(CAST(a AS TINYINT)), MAX(CAST(a AS TINYINT)) FROM cast_stats_ints;+----+physical_plan+01)ProjectionExec: expr=[-128 as min(cast_stats_ints.a), 127 as max(cast_stats_ints.a)]+02)--PlaceholderRowExec++statement ok+DROP TABLE cast_stats_ints;++# An out-of-range minimum or maximum must prevent aggregate folding and preserve+# the runtime cast error, even when the other endpoint fits in Int8.+statement ok+COPY (SELECT CAST(a AS BIGINT) AS a, CAST(b AS BIGINT) AS b+ FROM (VALUES (-129, -128), (0, 0), (127, 128), (NULL, NULL)) AS t(a, b))+TO 'test_files/scratch/parquet_statistics/cast_ints_overflow.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE cast_stats_ints_overflow STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/cast_ints_overflow.parquet';++query error Cast error: Can't cast value -129 to type Int8+SELECT MIN(CAST(a AS TINYINT)), MAX(CAST(a AS TINYINT)) FROM cast_stats_ints_overflow;++query error Cast error: Can't cast value 128 to type Int8+SELECT MIN(CAST(b AS TINYINT)), MAX(CAST(b AS TINYINT)) FROM cast_stats_ints_overflow;++query TT+EXPLAIN SELECT MIN(CAST(a AS TINYINT)), MAX(CAST(a AS TINYINT)) FROM cast_stats_ints_overflow;+----+physical_plan+01)AggregateExec: mode=Single, gby=[], aggr=[min(cast_stats_ints_overflow.a), max(cast_stats_ints_overflow.a)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/cast_ints_overflow.parquet]]}, projection=[CAST(a@0 AS Int8) as __common_expr_1], file_type=parquet++statement ok+DROP TABLE cast_stats_ints_overflow;++statement ok+SET datafusion.execution.target_partitions = 4;+# Config resetstatement okRESET datafusion.execution.collect_statistics;