Apache DataFusion SQL Query Engine
fix: block timestamp precision narrowing unwrap (#22837)
discord97ced0ac5fb54
diff --git a/datafusion/expr-common/src/casts.rs b/datafusion/expr-common/src/casts.rsindex d18c3d4f04..320f7cec79 100644--- a/datafusion/expr-common/src/casts.rs+++ b/datafusion/expr-common/src/casts.rs@@ -103,6 +103,35 @@ fn is_lossy_temporal_cast(from_type: &DataType, to_type: &DataType) -> bool {|| (is_date_type(to_type) && from_type.is_temporal())}+/// Returns true when casting a timestamp from `from_type` to `to_type` loses+/// timestamp precision.+///+/// This is used by comparison cast unwrapping to avoid rewrites such as+/// `CAST(ts_ns AS timestamp(ms)) = lit_ms` -> `ts_ns = lit_ns`. The original+/// predicate can match any nanosecond value in the same millisecond, while the+/// rewritten predicate only matches the exact millisecond boundary.+pub fn is_timestamp_precision_narrowing_cast(+ from_type: &DataType,+ to_type: &DataType,+) -> bool {+ let (DataType::Timestamp(from_unit, _), DataType::Timestamp(to_unit, _)) =+ (from_type, to_type)+ else {+ return false;+ };++ timestamp_unit_scale(from_unit) > timestamp_unit_scale(to_unit)+}++fn timestamp_unit_scale(unit: &TimeUnit) -> i128 {+ match unit {+ TimeUnit::Second => 1,+ TimeUnit::Millisecond => MILLISECONDS as i128,+ TimeUnit::Microsecond => MICROSECONDS as i128,+ TimeUnit::Nanosecond => NANOSECONDS as i128,+ }+}+/// Returns true if unwrap_cast_in_comparison supports this numeric typefn is_supported_numeric_type(data_type: &DataType) -> bool {matches!(@@ -784,6 +813,23 @@ mod tests {);}+ #[test]+ fn test_timestamp_precision_narrowing_cast() {+ let ts_ns = DataType::Timestamp(TimeUnit::Nanosecond, None);+ let ts_us = DataType::Timestamp(TimeUnit::Microsecond, None);+ let ts_ms = DataType::Timestamp(TimeUnit::Millisecond, None);+ let ts_s = DataType::Timestamp(TimeUnit::Second, None);++ assert!(is_timestamp_precision_narrowing_cast(&ts_ns, &ts_ms));+ assert!(is_timestamp_precision_narrowing_cast(&ts_us, &ts_s));+ assert!(!is_timestamp_precision_narrowing_cast(&ts_ms, &ts_ns));+ assert!(!is_timestamp_precision_narrowing_cast(&ts_ms, &ts_ms));+ assert!(!is_timestamp_precision_narrowing_cast(+ &DataType::Int64,+ &ts_ms+ ));+ }+#[test]fn test_try_cast_to_type_unsupported() {// int64 to listdiff --git a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rsindex a5b65d0d8e..c7f20a6b6f 100644--- a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs+++ b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs@@ -59,7 +59,9 @@ use datafusion_common::{Result, ScalarValue};use datafusion_common::{internal_err, tree_node::Transformed};use datafusion_expr::{BinaryExpr, lit};use datafusion_expr::{Cast, Expr, Operator, TryCast, simplify::SimplifyContext};-use datafusion_expr_common::casts::{is_supported_type, try_cast_literal_to_type};+use datafusion_expr_common::casts::{+ is_supported_type, is_timestamp_precision_narrowing_cast, try_cast_literal_to_type,+};pub(super) fn unwrap_cast_in_comparison_for_binary(info: &SimplifyContext,@@ -113,10 +115,14 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary(match (expr, literal) {(Expr::TryCast(TryCast {- expr: left_expr, ..+ expr: left_expr,+ field,+ ..})| Expr::Cast(Cast {- expr: left_expr, ..+ expr: left_expr,+ field,+ ..}),Expr::Literal(lit_val, _),) => {@@ -128,6 +134,10 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary(return false;};+ if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type()) {+ return false;+ }+if cast_literal_to_type_with_op(lit_val, &expr_type, op).is_some() {return true;}@@ -146,10 +156,14 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist(list: &[Expr],) -> bool {let (Expr::TryCast(TryCast {- expr: left_expr, ..+ expr: left_expr,+ field,+ ..})| Expr::Cast(Cast {- expr: left_expr, ..+ expr: left_expr,+ field,+ ..})) = exprelse {return false;@@ -163,6 +177,10 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist(return false;}+ if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type()) {+ return false;+ }+for right in list {let Ok(right_type) = info.get_data_type(right) else {return false;@@ -586,6 +604,25 @@ mod tests {assert_eq!(optimize_test(expr_lt, &schema), expected);}+ #[test]+ fn test_not_unwrap_cast_timestamp_precision_narrowing() {+ let schema = expr_test_schema();+ let expr_input = cast(col("ts_nano_none"), timestamp_millis_none_type())+ .eq(lit_timestamp_millis_none(1));++ assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);+ }++ #[test]+ fn test_unwrap_cast_timestamp_precision_widening() {+ let schema = expr_test_schema();+ let expr_input = cast(col("ts_millis_none"), timestamp_nano_none_type())+ .eq(lit_timestamp_nano_none(1_000_000));+ let expected = col("ts_millis_none").eq(lit_timestamp_millis_none(1));++ assert_eq!(optimize_test(expr_input, &schema), expected);+ }+fn optimize_test(expr: Expr, schema: &DFSchemaRef) -> Expr {let simplifier = ExprSimplifier::new(SimplifyContext::builder()@@ -607,6 +644,7 @@ mod tests {Field::new("c5", DataType::Float32, false),Field::new("c6", DataType::UInt32, false),Field::new("ts_nano_none", timestamp_nano_none_type(), false),+ Field::new("ts_millis_none", timestamp_millis_none_type(), false),Field::new("ts_nano_utf", timestamp_nano_utc_type(), false),Field::new("str1", DataType::Utf8, false),Field::new("largestr", DataType::LargeUtf8, false),@@ -643,6 +681,10 @@ mod tests {lit(ScalarValue::TimestampNanosecond(Some(ts), None))}+ fn lit_timestamp_millis_none(ts: i64) -> Expr {+ lit(ScalarValue::TimestampMillisecond(Some(ts), None))+ }+fn lit_timestamp_nano_utc(ts: i64) -> Expr {let utc = Some("+0:00".into());lit(ScalarValue::TimestampNanosecond(Some(ts), utc))@@ -652,6 +694,10 @@ mod tests {DataType::Timestamp(TimeUnit::Nanosecond, None)}+ fn timestamp_millis_none_type() -> DataType {+ DataType::Timestamp(TimeUnit::Millisecond, None)+ }+// this is the type that now() returnsfn timestamp_nano_utc_type() -> DataType {let utc = Some("+0:00".into());diff --git a/datafusion/optimizer/tests/optimizer_integration.rs b/datafusion/optimizer/tests/optimizer_integration.rsindex 6fad39dc33..1ecdf1e8a0 100644--- a/datafusion/optimizer/tests/optimizer_integration.rs+++ b/datafusion/optimizer/tests/optimizer_integration.rs@@ -795,7 +795,7 @@ fn extension_node_does_not_block_projection_pruning() -> Result<()> {Projection: t.a, CAST(t.ts AS Timestamp(ms, "UTC")) AS tsFilter: __common_expr_3 > TimestampMillisecond(1000, Some("UTC")) AND __common_expr_3 < TimestampMillisecond(2000, Some("UTC"))Projection: CAST(t.ts AS Timestamp(ms, "UTC")) AS __common_expr_3, t.a, t.ts- TableScan: t projection=[a, ts], partial_filters=[t.ts > TimestampNanosecond(1000000000, None), t.ts < TimestampNanosecond(2000000000, None), CAST(t.ts AS Timestamp(ms, "UTC")) > TimestampMillisecond(1000, Some("UTC")), CAST(t.ts AS Timestamp(ms, "UTC")) < TimestampMillisecond(2000, Some("UTC"))]+ TableScan: t projection=[a, ts], partial_filters=[CAST(t.ts AS Timestamp(ms, "UTC")) > TimestampMillisecond(1000, Some("UTC")), CAST(t.ts AS Timestamp(ms, "UTC")) < TimestampMillisecond(2000, Some("UTC"))]"#,);diff --git a/datafusion/physical-expr/src/simplifier/unwrap_cast.rs b/datafusion/physical-expr/src/simplifier/unwrap_cast.rsindex 4f4dfb2c20..5caee00962 100644--- a/datafusion/physical-expr/src/simplifier/unwrap_cast.rs+++ b/datafusion/physical-expr/src/simplifier/unwrap_cast.rs@@ -36,7 +36,9 @@ use std::sync::Arc;use arrow::datatypes::{DataType, Schema};use datafusion_common::{Result, ScalarValue, tree_node::Transformed};use datafusion_expr::Operator;-use datafusion_expr_common::casts::try_cast_literal_to_type;+use datafusion_expr_common::casts::{+ is_timestamp_precision_narrowing_cast, try_cast_literal_to_type,+};use crate::PhysicalExpr;use crate::expressions::{BinaryExpr, CastExpr, Literal, TryCastExpr, lit};@@ -60,13 +62,14 @@ fn try_unwrap_cast_binary(schema: &Schema,) -> Result<Option<Arc<dyn PhysicalExpr>>> {// Case 1: cast(left_expr) op literal- if let (Some((inner_expr, _cast_type)), Some(literal)) = (+ if let (Some((inner_expr, cast_type)), Some(literal)) = (extract_cast_info(binary.left()),binary.right().downcast_ref::<Literal>(),) && binary.op().supports_propagation()&& let Some(unwrapped) = try_unwrap_cast_comparison(Arc::clone(inner_expr),literal.value(),+ cast_type,*binary.op(),schema,)?@@ -75,7 +78,7 @@ fn try_unwrap_cast_binary(}// Case 2: literal op cast(right_expr)- if let (Some(literal), Some((inner_expr, _cast_type))) = (+ if let (Some(literal), Some((inner_expr, cast_type))) = (binary.left().downcast_ref::<Literal>(),extract_cast_info(binary.right()),) {@@ -85,6 +88,7 @@ fn try_unwrap_cast_binary(&& let Some(unwrapped) = try_unwrap_cast_comparison(Arc::clone(inner_expr),literal.value(),+ cast_type,swapped_op,schema,)?@@ -118,12 +122,17 @@ fn extract_cast_info(fn try_unwrap_cast_comparison(inner_expr: Arc<dyn PhysicalExpr>,literal_value: &ScalarValue,+ cast_type: &DataType,op: Operator,schema: &Schema,) -> Result<Option<Arc<dyn PhysicalExpr>>> {// Get the data type of the inner expressionlet inner_type = inner_expr.data_type(schema)?;+ if is_timestamp_precision_narrowing_cast(&inner_type, cast_type) {+ return Ok(None);+ }+// Try to cast the literal to the inner expression's typeif let Some(casted_literal) = try_cast_literal_to_type(literal_value, &inner_type) {let literal_expr = lit(casted_literal);@@ -138,7 +147,7 @@ fn try_unwrap_cast_comparison(mod tests {use super::*;use crate::expressions::col;- use arrow::datatypes::Field;+ use arrow::datatypes::{Field, TimeUnit};use datafusion_common::tree_node::TreeNode;/// Check if an expression is a cast expression@@ -548,6 +557,59 @@ mod tests {assert!(!result.transformed);}+ #[test]+ fn test_not_unwrap_timestamp_precision_narrowing() {+ let schema = Schema::new(vec![Field::new(+ "ts",+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ false,+ )]);++ let column_expr = col("ts", &schema).unwrap();+ let cast_expr = Arc::new(CastExpr::new(+ column_expr,+ DataType::Timestamp(TimeUnit::Millisecond, None),+ None,+ ));+ let literal_expr = lit(ScalarValue::TimestampMillisecond(Some(1), None));+ let binary_expr =+ Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal_expr));++ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();++ assert!(!result.transformed);+ }++ #[test]+ fn test_unwrap_timestamp_precision_widening() {+ let schema = Schema::new(vec![Field::new(+ "ts",+ DataType::Timestamp(TimeUnit::Millisecond, None),+ false,+ )]);++ let column_expr = col("ts", &schema).unwrap();+ let cast_expr = Arc::new(CastExpr::new(+ column_expr,+ DataType::Timestamp(TimeUnit::Nanosecond, None),+ None,+ ));+ let literal_expr = lit(ScalarValue::TimestampNanosecond(Some(1_000_000), None));+ let binary_expr =+ Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal_expr));++ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();++ assert!(result.transformed);+ let optimized_binary = result.data.downcast_ref::<BinaryExpr>().unwrap();+ assert!(!is_cast_expr(optimized_binary.left()));+ let right_literal = optimized_binary.right().downcast_ref::<Literal>().unwrap();+ assert_eq!(+ right_literal.value(),+ &ScalarValue::TimestampMillisecond(Some(1), None)+ );+ }+#[test]fn test_complex_nested_expression() {let schema = test_schema();