Apache DataFusion SQL Query Engine
feat: extend interval analysis support for temporal types (#21520)
Burak Şen6cd7e8395820
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 4511d8db90..ab58bf4a61 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -2526,6 +2526,25 @@ impl ScalarValue {(Self::Float64(Some(l)), Self::Float64(Some(r))) => {Some((l - r).abs().round() as _)}+ (Self::Date32(Some(l)), Self::Date32(Some(r))) => Some(l.abs_diff(*r) as _),+ (Self::Date64(Some(l)), Self::Date64(Some(r))) => Some(l.abs_diff(*r) as _),+ // Timestamp values are stored as epoch ticks regardless of timezone+ // annotation, so the distance is tz-independent (tz is display metadata).+ (Self::TimestampSecond(Some(l), _), Self::TimestampSecond(Some(r), _)) => {+ Some(l.abs_diff(*r) as _)+ }+ (+ Self::TimestampMillisecond(Some(l), _),+ Self::TimestampMillisecond(Some(r), _),+ ) => Some(l.abs_diff(*r) as _),+ (+ Self::TimestampMicrosecond(Some(l), _),+ Self::TimestampMicrosecond(Some(r), _),+ ) => Some(l.abs_diff(*r) as _),+ (+ Self::TimestampNanosecond(Some(l), _),+ Self::TimestampNanosecond(Some(r), _),+ ) => Some(l.abs_diff(*r) as _),(Self::Decimal128(Some(l), lprecision, lscale),Self::Decimal128(Some(r), rprecision, rscale),@@ -8766,6 +8785,42 @@ mod tests {ScalarValue::Decimal256(Some(10.into()), 1, 0),5,),+ // Temporal types+ (+ ScalarValue::Date32(Some(0)),+ ScalarValue::Date32(Some(10)),+ 10,+ ),+ (+ ScalarValue::Date32(Some(10)),+ ScalarValue::Date32(Some(0)),+ 10,+ ),+ (+ ScalarValue::Date64(Some(1000)),+ ScalarValue::Date64(Some(5000)),+ 4000,+ ),+ (+ ScalarValue::TimestampSecond(Some(100), None),+ ScalarValue::TimestampSecond(Some(200), None),+ 100,+ ),+ (+ ScalarValue::TimestampMillisecond(Some(1000), None),+ ScalarValue::TimestampMillisecond(Some(5000), None),+ 4000,+ ),+ (+ ScalarValue::TimestampMicrosecond(Some(0), None),+ ScalarValue::TimestampMicrosecond(Some(1_000_000), None),+ 1_000_000,+ ),+ (+ ScalarValue::TimestampNanosecond(Some(1_000_000_000), None),+ ScalarValue::TimestampNanosecond(Some(2_000_000_000), None),+ 1_000_000_000,+ ),];for (lhs, rhs, expected) in cases.iter() {let distance = lhs.distance(rhs).unwrap();@@ -8828,8 +8883,6 @@ mod tests {ScalarValue::Boolean(Some(true)),ScalarValue::Boolean(Some(false)),),- (ScalarValue::Date32(Some(0)), ScalarValue::Date32(Some(1))),- (ScalarValue::Date64(Some(0)), ScalarValue::Date64(Some(1))),(ScalarValue::Decimal128(Some(123), 5, 5),ScalarValue::Decimal128(Some(120), 5, 3),diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex c8024af30d..f84d79146b 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -150,15 +150,16 @@ mod test {// - null_count = 0 (partition values from paths are never null)// - min/max are the merged partition values across files in the group// - byte_size = num_rows * 4 (Date32 is 4 bytes per row)- // - distinct_count = Inexact(1) per partition file (single partition value per file),- // preserved via max() when merging stats across partitions+ // - distinct_count = Inexact(max_date - min_date + 1), derived from the+ // date range via interval analysis for temporal typeslet date32_byte_size = num_rows * 4;+ let distinct_dates = (max_date - min_date + 1) as usize;column_stats.push(ColumnStatistics {null_count: Precision::Exact(0),max_value: Precision::Exact(ScalarValue::Date32(Some(max_date))),min_value: Precision::Exact(ScalarValue::Date32(Some(min_date))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(distinct_dates),byte_size: Precision::Exact(date32_byte_size),});}@@ -579,7 +580,7 @@ mod test {max_value: Precision::Exact(ScalarValue::Date32(Some(20151))),min_value: Precision::Exact(ScalarValue::Date32(Some(20148))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(4),byte_size: Precision::Absent,},// column 2: right.id (Int32, file column from t2) - right partition 0: ids [3,4]@@ -613,7 +614,7 @@ mod test {max_value: Precision::Exact(ScalarValue::Date32(Some(20151))),min_value: Precision::Exact(ScalarValue::Date32(Some(20148))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(4),byte_size: Precision::Absent,},// column 2: right.id (Int32, file column from t2) - right partition 1: ids [1,2]@@ -1252,7 +1253,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(2),byte_size: Precision::Exact(8),},ColumnStatistics::new_unknown(), // window column@@ -1280,7 +1281,7 @@ mod test {DATE_2025_03_03,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(2),byte_size: Precision::Exact(8),},ColumnStatistics::new_unknown(), // window column@@ -1417,8 +1418,7 @@ mod test {byte_size: Precision::Exact(16),},// Left date column: all partitions (2025-03-01..2025-03-04)- // NDV is Inexact(1) because each Hive partition has exactly 1 distinct date value,- // and merging takes max as a conservative lower bound+ // NDV is Inexact(4) derived from the date range via interval analysisColumnStatistics {null_count: Precision::Exact(0),max_value: Precision::Exact(ScalarValue::Date32(Some(@@ -1428,7 +1428,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(4),byte_size: Precision::Exact(16),},// Right id column: partition 0 only (id 3..4)@@ -1441,7 +1441,7 @@ mod test {byte_size: Precision::Exact(8),},// Right date column: partition 0 only (2025-03-01..2025-03-02)- // NDV is Inexact(1) from the single Hive partition's date value+ // NDV is Inexact(2) derived from the date range via interval analysisColumnStatistics {null_count: Precision::Exact(0),max_value: Precision::Exact(ScalarValue::Date32(Some(@@ -1451,7 +1451,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(2),byte_size: Precision::Exact(8),},],@@ -1503,7 +1503,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(2),byte_size: Precision::Exact(8),},// Right id column: partition 0 only (id 3..4)@@ -1525,7 +1525,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(2),byte_size: Precision::Exact(8),},],@@ -1577,7 +1577,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(4),byte_size: Precision::Exact(16),},// Right id column: all partitions (id 1..4)@@ -1599,7 +1599,7 @@ mod test {DATE_2025_03_01,))),sum_value: Precision::Absent,- distinct_count: Precision::Inexact(1),+ distinct_count: Precision::Inexact(4),byte_size: Precision::Exact(16),},],diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rsindex c0cecad4a3..71b150eb92 100644--- a/datafusion/expr-common/src/interval_arithmetic.rs+++ b/datafusion/expr-common/src/interval_arithmetic.rs@@ -929,7 +929,12 @@ impl Interval {/// when the calculated cardinality does not fit in an `u64`.pub fn cardinality(&self) -> Option<u64> {let data_type = self.data_type();- if data_type.is_integer() {+ if data_type.is_integer()+ || matches!(+ data_type,+ DataType::Date32 | DataType::Date64 | DataType::Timestamp(_, _)+ )+ {self.upper.distance(&self.lower).map(|diff| diff as u64)} else if data_type.is_floating() {// Negative numbers are sorted in the reverse order. To@@ -3958,6 +3963,31 @@ mod tests {)?;assert_eq!(interval.cardinality().unwrap(), 2);+ // Temporal types+ let interval = Interval::try_new(+ ScalarValue::Date32(Some(0)),+ ScalarValue::Date32(Some(10)),+ )?;+ assert_eq!(interval.cardinality().unwrap(), 11);++ let interval = Interval::try_new(+ ScalarValue::Date64(Some(1000)),+ ScalarValue::Date64(Some(5000)),+ )?;+ assert_eq!(interval.cardinality().unwrap(), 4001);++ let interval = Interval::try_new(+ ScalarValue::TimestampSecond(Some(100), None),+ ScalarValue::TimestampSecond(Some(200), None),+ )?;+ assert_eq!(interval.cardinality().unwrap(), 101);++ let interval = Interval::try_new(+ ScalarValue::TimestampNanosecond(Some(1_000_000_000), None),+ ScalarValue::TimestampNanosecond(Some(2_000_000_000), None),+ )?;+ assert_eq!(interval.cardinality().unwrap(), 1_000_000_001);+Ok(())}diff --git a/datafusion/physical-expr/src/intervals/utils.rs b/datafusion/physical-expr/src/intervals/utils.rsindex fb039a6e5f..1090660a6b 100644--- a/datafusion/physical-expr/src/intervals/utils.rs+++ b/datafusion/physical-expr/src/intervals/utils.rs@@ -103,6 +103,9 @@ pub fn is_datatype_supported(data_type: &DataType) -> bool {| &DataType::UInt8| &DataType::Float64| &DataType::Float32+ | &DataType::Date32+ | &DataType::Date64+ | &DataType::Timestamp(_, _))}diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex ed24033d69..50efe8f509 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -2892,6 +2892,52 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_filter_statistics_equality_timestamp_ndv() -> Result<()> {+ // ts: min=1_000_000_000, max=2_000_000_000, ndv=500+ let schema = Schema::new(vec![Field::new(+ "ts",+ DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, None),+ false,+ )]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows: Precision::Inexact(1000),+ total_byte_size: Precision::Inexact(8000),+ column_statistics: vec![ColumnStatistics {+ min_value: Precision::Inexact(ScalarValue::TimestampNanosecond(+ Some(1_000_000_000),+ None,+ )),+ max_value: Precision::Inexact(ScalarValue::TimestampNanosecond(+ Some(2_000_000_000),+ None,+ )),+ distinct_count: Precision::Inexact(500),+ ..Default::default()+ }],+ },+ schema.clone(),+ ));++ let predicate = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("ts", 0)),+ Operator::Eq,+ Arc::new(Literal::new(ScalarValue::TimestampNanosecond(+ Some(1_500_000_000),+ None,+ ))),+ ));+ let filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(predicate, input)?);+ let statistics = filter.partition_statistics(None)?;+ assert_eq!(+ statistics.column_statistics[0].distinct_count,+ Precision::Exact(1)+ );+ Ok(())+ }+#[test]fn test_collect_equality_columns() {use std::collections::HashSet;