Skip to content

Apache DataFusion SQL Query Engine

refactor: Use cast preimages for cast predicate rewrites (#22906)

discord9fb7cf8e832be
diff --git a/datafusion/core/tests/expr_api/simplification.rs b/datafusion/core/tests/expr_api/simplification.rs
index 64ff1be99f..fd4cc872ad 100644
--- a/datafusion/core/tests/expr_api/simplification.rs
+++ b/datafusion/core/tests/expr_api/simplification.rs
@@ -20,25 +20,32 @@
use insta::assert_snapshot;
use arrow::array::types::IntervalDayTime;
-use arrow::array::{ArrayRef, Int32Array};
+use arrow::array::{
+ ArrayRef, BooleanArray, Int32Array, RecordBatch, TimestampNanosecondArray,
+};
use arrow::datatypes::{DataType, Field, Schema};
use chrono::{DateTime, TimeZone, Utc};
use datafusion::{error::Result, prelude::*};
use datafusion_common::ScalarValue;
use datafusion_common::cast::as_int32_array;
use datafusion_common::{DFSchemaRef, ToDFSchema};
-use datafusion_expr::expr::ScalarFunction;
+use datafusion_expr::execution_props::ExecutionProps;
+use datafusion_expr::expr::{ScalarFunction, TryCast};
use datafusion_expr::logical_plan::builder::table_scan_with_filters;
+use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
use datafusion_expr::simplify::SimplifyContext;
use datafusion_expr::{
- Cast, ColumnarValue, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Projection,
- ScalarUDF, Volatility, table_scan,
+ Cast, ColumnarValue, ExprSchemable, LogicalPlan, LogicalPlanBuilder, Operator,
+ Projection, ScalarUDF, Volatility, in_list, table_scan,
};
use datafusion_functions::math;
use datafusion_optimizer::optimizer::Optimizer;
use datafusion_optimizer::simplify_expressions::{ExprSimplifier, SimplifyExpressions};
use datafusion_optimizer::{OptimizerContext, OptimizerRule};
+use datafusion_physical_expr::PhysicalExprSimplifier;
+use datafusion_physical_expr::planner::create_physical_expr;
use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
/// A schema like:
///
@@ -152,6 +159,345 @@ fn to_timestamp_expr(arg: impl Into<String>) -> Expr {
to_timestamp(vec![lit(arg.into())])
}
+fn cast_or_try_cast(expr: Expr, data_type: DataType, try_cast: bool) -> Expr {
+ if try_cast {
+ Expr::TryCast(TryCast::new(Box::new(expr), data_type))
+ } else {
+ Expr::Cast(Cast::new(Box::new(expr), data_type))
+ }
+}
+
+fn simplify_logical_expr(expr: Expr, schema: DFSchemaRef) -> Expr {
+ ExprSimplifier::new(SimplifyContext::builder().with_schema(schema).build())
+ .simplify(expr)
+ .unwrap()
+}
+
+fn evaluate_physical_boolean_expr(
+ physical_expr: Arc<dyn datafusion_physical_expr::PhysicalExpr>,
+ batch: &RecordBatch,
+) -> Vec<Option<bool>> {
+ let value = physical_expr.evaluate(batch).unwrap();
+ let array = value.into_array(batch.num_rows()).unwrap();
+ array
+ .as_any()
+ .downcast_ref::<BooleanArray>()
+ .unwrap()
+ .iter()
+ .collect()
+}
+
+fn evaluate_boolean_expr(expr: &Expr, batch: &RecordBatch) -> Vec<Option<bool>> {
+ let schema = batch.schema().to_dfschema_ref().unwrap();
+ let physical_expr = create_physical_expr(
+ expr,
+ schema.as_ref(),
+ &ExecutionProps::new(),
+ &PhysicalPlanningContext::default(),
+ )
+ .unwrap();
+ evaluate_physical_boolean_expr(physical_expr, batch)
+}
+
+fn evaluate_physical_simplified_boolean_expr(
+ expr: &Expr,
+ batch: &RecordBatch,
+) -> Vec<Option<bool>> {
+ let schema = batch.schema().to_dfschema_ref().unwrap();
+ // Lower the unsimplified logical expression so this specifically tests the
+ // physical simplifier rather than a full SQL/planning pipeline.
+ let physical_expr = create_physical_expr(
+ expr,
+ schema.as_ref(),
+ &ExecutionProps::new(),
+ &PhysicalPlanningContext::default(),
+ )
+ .unwrap();
+ let simplified = PhysicalExprSimplifier::new(batch.schema().as_ref())
+ .simplify(physical_expr)
+ .unwrap();
+ evaluate_physical_boolean_expr(simplified, batch)
+}
+
+/// Raw timestamp values clustered near the epoch, expressed in `unit`. They are
+/// far below the one-hour `+01:00` shift, so the tz-aware target never matches
+/// any non-NULL row.
+fn timezone_cast_values(unit: &arrow::datatypes::TimeUnit) -> Vec<Option<i64>> {
+ match unit {
+ arrow::datatypes::TimeUnit::Nanosecond => vec![
+ Some(-1_000_000),
+ Some(-999_999),
+ Some(0),
+ Some(999_999),
+ Some(1_000_000),
+ None,
+ ],
+ _ => vec![
+ Some(-1000),
+ Some(-999),
+ Some(0),
+ Some(999),
+ Some(1000),
+ None,
+ ],
+ }
+}
+
+fn timezone_cast_array(
+ unit: &arrow::datatypes::TimeUnit,
+ values: Vec<Option<i64>>,
+) -> ArrayRef {
+ match unit {
+ arrow::datatypes::TimeUnit::Nanosecond => {
+ Arc::new(TimestampNanosecondArray::from(values))
+ }
+ arrow::datatypes::TimeUnit::Millisecond => {
+ Arc::new(arrow::array::TimestampMillisecondArray::from(values))
+ }
+ other => unreachable!("unsupported test time unit {other:?}"),
+ }
+}
+
+fn timezone_cast_literal(unit: &arrow::datatypes::TimeUnit, value: i64) -> ScalarValue {
+ match unit {
+ arrow::datatypes::TimeUnit::Nanosecond => {
+ ScalarValue::TimestampNanosecond(Some(value), Some("+01:00".into()))
+ }
+ arrow::datatypes::TimeUnit::Millisecond => {
+ ScalarValue::TimestampMillisecond(Some(value), Some("+01:00".into()))
+ }
+ other => unreachable!("unsupported test time unit {other:?}"),
+ }
+}
+
+#[test]
+fn timestamp_timezone_cast_preimage_preserves_results() {
+ use arrow::datatypes::TimeUnit;
+
+ // Same unit, widening and narrowing naive -> tz-aware casts: none of them may
+ // be preimage-rewritten, because Arrow shifts the naive value to UTC when the
+ // target carries a timezone.
+ for (source_unit, target_unit) in [
+ (TimeUnit::Nanosecond, TimeUnit::Nanosecond),
+ (TimeUnit::Millisecond, TimeUnit::Nanosecond),
+ (TimeUnit::Nanosecond, TimeUnit::Millisecond),
+ ] {
+ let source_type = DataType::Timestamp(source_unit, None);
+ let target_type = DataType::Timestamp(target_unit, Some("+01:00".into()));
+ let values = timezone_cast_values(&source_unit);
+ let batch = RecordBatch::try_from_iter(vec![(
+ "ts",
+ timezone_cast_array(&source_unit, values.clone()),
+ )])
+ .unwrap();
+ assert_eq!(batch.schema().field(0).data_type(), &source_type);
+ let expected: Vec<Option<bool>> =
+ values.iter().map(|value| value.map(|_| false)).collect();
+
+ for try_cast in [false, true] {
+ let make_expr = || {
+ cast_or_try_cast(col("ts"), target_type.clone(), try_cast)
+ .eq(lit(timezone_cast_literal(&target_unit, 0)))
+ };
+ let original = make_expr();
+ let logical = simplify_logical_expr(
+ make_expr(),
+ batch.schema().to_dfschema_ref().unwrap(),
+ );
+ let logical_rows = evaluate_boolean_expr(&logical, &batch);
+ let physical_rows =
+ evaluate_physical_simplified_boolean_expr(&make_expr(), &batch);
+
+ // Naive timestamps cannot use the timezone-aware target's preimage.
+ assert_eq!(evaluate_boolean_expr(&original, &batch), expected);
+ assert_eq!(logical_rows, expected);
+ assert_eq!(physical_rows, expected);
+ assert!(
+ format!("{logical}").contains("CAST(ts AS"),
+ "cast must be retained for {source_unit:?} -> {target_unit:?}, \
+ try_cast={try_cast}: {logical}"
+ );
+ }
+
+ // The IN-list rewrite uses the same exact-preimage helper, so a
+ // multi-item list must not fold either.
+ let in_list_expr = in_list(
+ cast_or_try_cast(col("ts"), target_type.clone(), false),
+ vec![
+ lit(timezone_cast_literal(&target_unit, 0)),
+ lit(timezone_cast_literal(&target_unit, 1)),
+ ],
+ false,
+ );
+ let logical = simplify_logical_expr(
+ in_list_expr.clone(),
+ batch.schema().to_dfschema_ref().unwrap(),
+ );
+ assert_eq!(evaluate_boolean_expr(&in_list_expr, &batch), expected);
+ assert_eq!(evaluate_boolean_expr(&logical, &batch), expected);
+ assert!(
+ format!("{logical}").contains("CAST(ts AS"),
+ "cast must be retained in the IN-list for {source_unit:?} -> \
+ {target_unit:?}: {logical}"
+ );
+ }
+}
+
+#[test]
+fn timestamp_narrowing_range_controls_still_rewrite() {
+ for timezone in [None, Some("+01:00".into())] {
+ let source_type =
+ DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, timezone.clone());
+ let target_type = DataType::Timestamp(
+ arrow::datatypes::TimeUnit::Millisecond,
+ timezone.clone(),
+ );
+ let batch = RecordBatch::try_from_iter(vec![(
+ "ts",
+ Arc::new(
+ TimestampNanosecondArray::from(vec![
+ Some(-1_000_000),
+ Some(-999_999),
+ Some(0),
+ Some(999_999),
+ Some(1_000_000),
+ None,
+ ])
+ .with_timezone_opt(timezone.clone()),
+ ) as ArrayRef,
+ )])
+ .unwrap();
+ assert_eq!(batch.schema().field(0).data_type(), &source_type);
+ let expected = vec![
+ Some(false),
+ Some(true),
+ Some(true),
+ Some(true),
+ Some(false),
+ None,
+ ];
+
+ for try_cast in [false, true] {
+ let make_expr = || {
+ cast_or_try_cast(col("ts"), target_type.clone(), try_cast).eq(lit(
+ ScalarValue::TimestampMillisecond(Some(0), timezone.clone()),
+ ))
+ };
+ let original = make_expr();
+ let logical = simplify_logical_expr(
+ make_expr(),
+ batch.schema().to_dfschema_ref().unwrap(),
+ );
+ let physical_rows =
+ evaluate_physical_simplified_boolean_expr(&make_expr(), &batch);
+ assert_ne!(logical, original);
+ assert_eq!(evaluate_boolean_expr(&original, &batch), expected);
+ assert_eq!(evaluate_boolean_expr(&logical, &batch), expected);
+ assert_eq!(physical_rows, expected);
+ }
+ }
+}
+
+fn alternating_timestamp_udf(counter: Arc<AtomicUsize>) -> Arc<ScalarUDF> {
+ Arc::new(create_udf(
+ "alternating_timestamp",
+ vec![],
+ DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, None),
+ Volatility::Volatile,
+ Arc::new(move |_args: &[ColumnarValue]| {
+ let value = if counter.fetch_add(1, Ordering::SeqCst).is_multiple_of(2) {
+ 1_000_000
+ } else {
+ -1_000_000
+ };
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ Some(value),
+ None,
+ )))
+ }),
+ ))
+}
+
+#[test]
+fn volatile_cast_preimage_does_not_duplicate_evaluation() {
+ let counter = Arc::new(AtomicUsize::new(0));
+ let batch = RecordBatch::try_from_iter(vec![(
+ "one_row",
+ Arc::new(Int32Array::from(vec![1])) as ArrayRef,
+ )])
+ .unwrap();
+ let target_type = DataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None);
+
+ for try_cast in [false, true] {
+ for op in [
+ Operator::Eq,
+ Operator::NotEq,
+ Operator::IsDistinctFrom,
+ Operator::IsNotDistinctFrom,
+ ] {
+ let make_expr = || {
+ binary_expr(
+ cast_or_try_cast(
+ Expr::ScalarFunction(ScalarFunction::new_udf(
+ alternating_timestamp_udf(Arc::clone(&counter)),
+ vec![],
+ )),
+ target_type.clone(),
+ try_cast,
+ ),
+ op,
+ lit(ScalarValue::TimestampMillisecond(Some(0), None)),
+ )
+ };
+ counter.store(0, Ordering::SeqCst);
+ let original = make_expr();
+ let expected = evaluate_boolean_expr(&original, &batch);
+ assert_eq!(counter.load(Ordering::SeqCst), 1);
+
+ counter.store(0, Ordering::SeqCst);
+ let logical = simplify_logical_expr(
+ make_expr(),
+ batch.schema().to_dfschema_ref().unwrap(),
+ );
+ assert_eq!(evaluate_boolean_expr(&logical, &batch), expected);
+ assert_eq!(counter.load(Ordering::SeqCst), 1, "logical {op:?}");
+
+ counter.store(0, Ordering::SeqCst);
+ let physical =
+ evaluate_physical_simplified_boolean_expr(&make_expr(), &batch);
+ assert_eq!(physical, expected);
+ assert_eq!(counter.load(Ordering::SeqCst), 1, "physical {op:?}");
+ }
+
+ // Ordered range preimages still have one input reference and remain enabled.
+ let make_expr = || {
+ cast_or_try_cast(
+ Expr::ScalarFunction(ScalarFunction::new_udf(
+ alternating_timestamp_udf(Arc::clone(&counter)),
+ vec![],
+ )),
+ target_type.clone(),
+ try_cast,
+ )
+ .lt(lit(ScalarValue::TimestampMillisecond(Some(0), None)))
+ };
+ counter.store(0, Ordering::SeqCst);
+ let original = make_expr();
+ let expected = evaluate_boolean_expr(&original, &batch);
+ assert_eq!(counter.load(Ordering::SeqCst), 1);
+ counter.store(0, Ordering::SeqCst);
+ let logical =
+ simplify_logical_expr(make_expr(), batch.schema().to_dfschema_ref().unwrap());
+ assert_ne!(logical, original);
+ assert_eq!(evaluate_boolean_expr(&logical, &batch), expected);
+ assert_eq!(counter.load(Ordering::SeqCst), 1);
+ counter.store(0, Ordering::SeqCst);
+ let physical = evaluate_physical_simplified_boolean_expr(&make_expr(), &batch);
+ assert_eq!(physical, expected);
+ assert_eq!(counter.load(Ordering::SeqCst), 1);
+ }
+}
+
#[test]
fn basic() {
let context = SimplifyContext::builder()
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index c15df47f62..62698c6186 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -27,7 +27,7 @@ use datafusion_common::{DataFusionError, ScalarValue};
use itertools::Itertools;
use crate::parquet::Unit::RowGroup;
-use crate::parquet::{ContextWithParquet, Scenario};
+use crate::parquet::{ContextWithParquet, Scenario, make_decimal_batch};
use datafusion_expr::{col, lit};
struct RowGroupPruningTest {
scenario: Scenario,
@@ -1003,6 +1003,8 @@ async fn prune_decimal_eq() {
.with_expected_rows(2)
.test_row_group_prune()
.await;
+ // Maximum-precision decimal widening accepts this aligned literal, so
+ // Bloom-filter pruning evaluates it just like the equivalent source cast.
RowGroupPruningTest::new()
.with_scenario(Scenario::DecimalLargePrecision)
.with_query("SELECT * FROM t where decimal_col = 4.00")
@@ -1015,7 +1017,90 @@ async fn prune_decimal_eq() {
.with_expected_rows(2)
.test_row_group_prune()
.await;
- // The data type of decimal_col is decimal(38,2)
+ // A source-typed decimal literal exposes a usable Bloom-filter guarantee.
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::DecimalLargePrecision)
+ .with_query("SELECT * FROM t where decimal_col = cast(4.00 as decimal(38,2))")
+ .with_expected_errors(Some(0))
+ .with_matched_by_stats(Some(2))
+ .with_pruned_by_stats(Some(1))
+ .with_pruned_files(Some(0))
+ .with_matched_by_bloom_filter(Some(2))
+ .with_pruned_by_bloom_filter(Some(0))
+ .with_expected_rows(2)
+ .test_row_group_prune()
+ .await;
+ // Source-precision regression: the same scale-up rewrite must admit
+ // sub-maximum decimal sources, not only decimal(38,x). Each pair below
+ // writes one row group per value list [1,2,3,4,6], [-5,1,3,4,6] and
+ // [20,30,30,40,60], stored unscaled at the source scale, so 4.00 is in the
+ // first two row groups only.
+ for (precision, scale) in [(30u8, 2i8), (28, 0), (36, 4)] {
+ let unit = 10_i128.pow(scale as u32);
+ let batches = [
+ vec![1, 2, 3, 4, 6],
+ vec![-5, 1, 3, 4, 6],
+ vec![20, 30, 30, 40, 60],
+ ]
+ .into_iter()
+ .map(|values| {
+ make_decimal_batch(
+ values.into_iter().map(|v| v * unit).collect(),
+ precision,
+ scale,
+ )
+ })
+ .collect::<Vec<_>>();
+
+ let mut ctx = ContextWithParquet::with_custom_data(
+ Scenario::DecimalLargePrecision,
+ RowGroup(5),
+ batches[0].schema(),
+ batches,
+ )
+ .await;
+ let output = ctx.query("SELECT * FROM t where decimal_col = 4.00").await;
+
+ println!(
+ "--- decimal source decimal({precision},{scale}) ---\n{}",
+ output.description()
+ );
+
+ let case = format!("decimal({precision},{scale})");
+ assert_eq!(
+ output.predicate_evaluation_errors(),
+ Some(0),
+ "predicate_evaluation_errors for {case}"
+ );
+ assert_eq!(
+ output.row_groups_matched_statistics(),
+ Some(2),
+ "row_groups_matched_statistics for {case}"
+ );
+ assert_eq!(
+ output.row_groups_pruned_statistics(),
+ Some(1),
+ "row_groups_pruned_statistics for {case}"
+ );
+ let bloom = output
+ .row_groups_bloom_filter()
+ .unwrap_or_else(|| panic!("no bloom filter metrics for {case}"));
+ assert_eq!(bloom.total_matched(), 2, "bloom filter matched for {case}");
+ assert_eq!(bloom.total_pruned(), 0, "bloom filter pruned for {case}");
+ assert_eq!(output.result_rows, 2, "result rows for {case}");
+
+ // The two returned rows are exactly 4.00 at the source scale.
+ let expected = ScalarValue::Decimal128(Some(4 * unit), precision, scale);
+ assert_eq!(
+ output
+ .pretty_results()
+ .matches(&format!("| {expected} "))
+ .count(),
+ 2,
+ "expected two {expected} rows for {case}, got:\n{}",
+ output.pretty_results()
+ );
+ }
}
#[tokio::test]
diff --git a/datafusion/expr-common/src/casts.rs b/datafusion/expr-common/src/casts.rs
index 3518c02772..0f643220cd 100644
--- a/datafusion/expr-common/src/casts.rs
+++ b/datafusion/expr-common/src/casts.rs
@@ -22,6 +22,10 @@
//! unwrap_cast module to be shared between logical and physical layers.
use std::cmp::Ordering;
+use std::sync::Arc;
+
+use crate::interval_arithmetic::Interval;
+use crate::operator::Operator;
use arrow::datatypes::{
DataType, MAX_DECIMAL32_FOR_EACH_PRECISION, MAX_DECIMAL64_FOR_EACH_PRECISION,
@@ -31,7 +35,36 @@ use arrow::datatypes::{
use arrow::temporal_conversions::{
MICROSECONDS, MILLISECONDS, MILLISECONDS_IN_DAY, NANOSECONDS,
};
-use datafusion_common::ScalarValue;
+use datafusion_common::{Result, ScalarValue};
+
+/// Source-domain preimage of `CAST(source_expr AS target_type) OP literal`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CastPredicatePreimage {
+ /// An exact same-operator source bound or literal, i.e. the rewrite is
+ /// `source_expr OP value` with the original comparison operator.
+ Exact(ScalarValue),
+ /// A half-open source-domain interval `[lower, upper)`; the caller maps the
+ /// comparison operator onto its bounds. Equal bounds mean no single source
+ /// value maps to the literal, an empty equality preimage, while ordered
+ /// comparisons still compare against that bound as in `< bound` or
+ /// `>= bound`.
+ Range(Interval),
+}
+
+impl CastPredicatePreimage {
+ /// Returns whether this preimage and comparison operator produce multiple
+ /// predicates that each evaluate the input expression.
+ pub fn duplicates_input(&self, op: Operator) -> bool {
+ matches!(self, Self::Range(_))
+ && matches!(
+ op,
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
+ )
+ }
+}
/// Convert a literal [`ScalarValue`] to `target_type`, preserving the exact value.
///
@@ -40,13 +73,16 @@ use datafusion_common::ScalarValue;
///
/// This is a restricted, value-preserving cast used to rewrite comparison
/// predicates of the form `CAST(col AS target_type) <op> literal` into
-/// `col <op> try_cast_literal_to_type(literal, col_type)`. That rewrite is
-/// only valid when the cast cannot change the comparison result.
+/// `col <op> try_cast_literal_to_type(literal, col_type)`. The rewrite requires
+/// an exact literal preimage and an accepted cast family; for integer or
+/// decimal sources with a decimal target of equal or greater scale, it may
+/// change cast overflow errors or NULLs to a boolean result for values outside
+/// the target range.
///
/// # Supported Casts
/// * numeric → numeric, including integers, decimals, `Date32`/`Date64` and
-/// `Timestamp`s, rejecting values outside the target's range or that would
-/// lose decimal digits
+/// `Timestamps`, rejecting literal conversions outside the target's range or
+/// that would lose decimal digits
/// * string → string between `Utf8`, `LargeUtf8` and `Utf8View`
/// * wrapping a value into, or unwrapping it out of, a `Dictionary` whose value
/// type matches the literal's type
@@ -76,6 +112,467 @@ pub fn try_cast_literal_to_type(
.or_else(|| try_cast_binary(lit_value, target_type))
}
+/// Computes a source-domain preimage for `CAST(source AS target_type) OP literal`.
+///
+/// This is the shared semantic core for logical and physical cast-predicate
+/// rewrites. It returns an exact same-operator source bound or literal as
+/// [`CastPredicatePreimage::Exact`] where moving the cast preserves comparison
+/// semantics subject to the documented family-level overflow exceptions, and a
+/// [`CastPredicatePreimage::Range`] for many-to-one casts with known preimages
+/// (timestamp precision narrowing) or for non-aligned timestamp widening
+/// literals.
+pub fn cast_predicate_preimage(
+ source_type: &DataType,
+ target_type: &DataType,
+ op: Operator,
+ lit_value: &ScalarValue,
+) -> Result<Option<CastPredicatePreimage>> {
+ // Timestamp casts have their own rules: a truncation bucket when the cast
+ // narrows, and the source values equal to the literal when it does not. A
+ // rejected timestamp cast keeps its `CAST` instead of falling back to
+ // another cast family.
+ if let Some((source, target)) = timestamp_cast_types(source_type, target_type) {
+ return Ok(timestamp_preimage(
+ source_type,
+ source,
+ target,
+ op,
+ lit_value,
+ ));
+ }
+
+ if let Some(value) =
+ exact_preimage_int_to_str_eq_like(source_type, target_type, op, lit_value)
+ {
+ return Ok(Some(CastPredicatePreimage::Exact(value)));
+ }
+
+ Ok(exact_preimage_cast(source_type, target_type, lit_value)
+ .map(CastPredicatePreimage::Exact))
+}
+
+/// Computes a singleton source-domain literal for exact cast-predicate rewrites.
+///
+/// This intentionally returns `None` for timestamp precision narrowing and for
+/// non-aligned timestamp widening: those casts need range preimages instead.
+pub fn exact_preimage_cast(
+ source_type: &DataType,
+ target_type: &DataType,
+ lit_value: &ScalarValue,
+) -> Option<ScalarValue> {
+ // Timestamp casts need unit arithmetic rather than a literal round trip; only
+ // their singleton preimages are valid here.
+ if let Some((source, target)) = timestamp_cast_types(source_type, target_type) {
+ return match timestamp_preimage(
+ source_type,
+ source,
+ target,
+ Operator::Eq,
+ lit_value,
+ )? {
+ CastPredicatePreimage::Exact(value) => Some(value),
+ CastPredicatePreimage::Range(_) => None,
+ };
+ }
+
+ // Apply a family-level gate: the source→target cast must normally preserve
+ // the full source domain. Integer or decimal sources to a decimal target
+ // are admitted by scale instead (see `is_exact_cast_safe`), accepting
+ // precision/width overflow as a boolean result for out-of-range rows.
+ if !is_exact_cast_safe(source_type, target_type) {
+ return None;
+ }
+
+ try_cast_literal_to_type(lit_value, source_type)
+}
+
+/// Returns true when casting a timestamp from `source_type` to `target_type`
+/// loses timestamp precision.
+pub fn is_timestamp_precision_narrowing_cast(
+ source_type: &DataType,
+ target_type: &DataType,
+) -> bool {
+ let (DataType::Timestamp(source_unit, _), DataType::Timestamp(target_unit, _)) =
+ (source_type, target_type)
+ else {
+ return false;
+ };
+
+ timestamp_unit_scale(source_unit) > timestamp_unit_scale(target_unit)
+}
+
+/// Returns the source and target types of a timestamp cast, each unwrapped from
+/// at most one `Dictionary` level.
+fn timestamp_cast_types<'a>(
+ source_type: &'a DataType,
+ target_type: &'a DataType,
+) -> Option<(&'a DataType, &'a DataType)> {
+ let unwrap = |data_type: &'a DataType| match data_type {
+ DataType::Dictionary(_, value_type) => value_type.as_ref(),
+ data_type => data_type,
+ };
+ let (source, target) = (unwrap(source_type), unwrap(target_type));
+ (matches!(source, DataType::Timestamp(_, _))
+ && matches!(target, DataType::Timestamp(_, _)))
+ .then_some((source, target))
+}
+
+/// Returns `true` when casting `source_type` to `target_type` is exact and
+/// order-preserving over the source values the cast succeeds on. Integer or
+/// decimal sources to a decimal target are admitted when the source scale is
+/// nonnegative and the target scale is at least the source scale: a successful
+/// cast only multiplies the unscaled value by an integral power of ten.
+/// Precision or width overflow is the explicit exception: `CAST` errors and
+/// `TRY_CAST` yields NULL for such source values, while the rewritten predicate
+/// compares the source value and returns TRUE/FALSE. This rule is type-based
+/// and does not distinguish implicit coercions from explicit casts.
+///
+/// This is a conservative gate used by [`exact_preimage_cast`] for the non
+/// timestamp cast families; timestamp casts are handled by
+/// [`timestamp_preimage`] instead.
+fn is_exact_cast_safe(source_type: &DataType, target_type: &DataType) -> bool {
+ // Unwrap at most one level of dictionary for family-level checking.
+ let (src, tgt) = match (source_type, target_type) {
+ (DataType::Dictionary(_, v_src), DataType::Dictionary(_, v_tgt)) => {
+ (v_src.as_ref(), v_tgt.as_ref())
+ }
+ (DataType::Dictionary(_, v_src), tgt) => (v_src.as_ref(), tgt),
+ (src, DataType::Dictionary(_, v_tgt)) => (src, v_tgt.as_ref()),
+ _ => (source_type, target_type),
+ };
+
+ // Integer or decimal sources to a decimal target are accepted when the
+ // source scale is nonnegative and the target only adds fractional digits.
+ // This precedes identity so negative-scale decimals are rejected by the
+ // scale check instead of reaching the scale arithmetic and panicking.
+ if is_decimal_type(tgt) && (is_decimal_type(src) || is_integer_type(src)) {
+ let s_src = if is_decimal_type(src) {
+ decimal_precision_scale(src).1
+ } else {
+ 0
+ };
+ return s_src >= 0 && decimal_precision_scale(tgt).1 >= s_src;
+ }
+
+ if src == tgt {
+ return true;
+ }
+
+ // Whitelist of family-level safe casts. Anything not listed here is
+ // considered potentially information-losing and blocked.
+ if is_integer_type(src) && is_integer_type(tgt) {
+ return is_safe_integer_cast(src, tgt);
+ }
+ if matches!(
+ (src, tgt),
+ (DataType::Date32, DataType::Int32)
+ | (DataType::Int32, DataType::Date32)
+ | (DataType::Date64, DataType::Int64)
+ | (DataType::Int64, DataType::Date64)
+ ) {
+ // Date32 ↔ Int32 and Date64 ↔ Int64 share the same internal
+ // representation.
+ return true;
+ }
+ if matches!((src, tgt), (DataType::Date32, DataType::Date64)) {
+ // Date32 → Date64 is injective: every source day maps to its midnight
+ // millisecond value. The literal conversion below enforces the exact
+ // day-boundary requirement in the reverse direction.
+ return true;
+ }
+ if is_supported_string_type(src) && is_supported_string_type(tgt) {
+ // String width casts among Utf8 / LargeUtf8 / Utf8View.
+ return true;
+ }
+ if matches!((src, tgt), (DataType::FixedSizeBinary(_), DataType::Binary)) {
+ // FixedSizeBinary(n) → Binary is always widening.
+ return true;
+ }
+ if matches!(
+ (src, tgt),
+ (DataType::FixedSizeBinary(n1), DataType::FixedSizeBinary(n2)) if n1 == n2
+ ) {
+ return true;
+ }
+ false
+}
+
+/// Returns `true` for integer types (signed and unsigned, excluding Date and
+/// Decimal).
+fn is_integer_type(dt: &DataType) -> bool {
+ matches!(
+ dt,
+ DataType::Int8
+ | DataType::Int16
+ | DataType::Int32
+ | DataType::Int64
+ | DataType::UInt8
+ | DataType::UInt16
+ | DataType::UInt32
+ | DataType::UInt64
+ )
+}
+
+/// Integer cast that is value-preserving over the full source domain.
+fn is_safe_integer_cast(src: &DataType, tgt: &DataType) -> bool {
+ use DataType::*;
+ match (src, tgt) {
+ // Signed → wider signed
+ (Int8, Int16 | Int32 | Int64) => true,
+ (Int16, Int32 | Int64) => true,
+ (Int32, Int64) => true,
+ // Unsigned → wider unsigned
+ (UInt8, UInt16 | UInt32 | UInt64) => true,
+ (UInt16, UInt32 | UInt64) => true,
+ (UInt32, UInt64) => true,
+ // Unsigned → wider signed (full source range fits in target)
+ (UInt8, Int16 | Int32 | Int64) => true,
+ (UInt16, Int32 | Int64) => true,
+ (UInt32, Int64) => true,
+ // Anything else is narrowing, partial, or crosses signedness
+ // without guaranteed domain containment.
+ _ => false,
+ }
+}
+
+fn is_decimal_type(dt: &DataType) -> bool {
+ matches!(
+ dt,
+ DataType::Decimal128(_, _)
+ | DataType::Decimal64(_, _)
+ | DataType::Decimal32(_, _)
+ )
+}
+
+fn decimal_precision_scale(dt: &DataType) -> (u8, i8) {
+ match dt {
+ DataType::Decimal128(p, s)
+ | DataType::Decimal64(p, s)
+ | DataType::Decimal32(p, s) => (*p, *s),
+ _ => unreachable!(),
+ }
+}
+
+/// Computes a singleton preimage for equality-like predicates over casts whose
+/// target value is a string representation of an integer source value.
+///
+/// For example, `CAST(int_col AS Utf8) = '123'` can be rewritten to
+/// `int_col = 123`, but `CAST(int_col AS Utf8) = '0123'` cannot be rewritten to
+/// `int_col = 123` because casting `123` back to a string yields `'123'`, not
+/// `'0123'`.
+fn exact_preimage_int_to_str_eq_like(
+ source_type: &DataType,
+ target_type: &DataType,
+ op: Operator,
+ lit_value: &ScalarValue,
+) -> Option<ScalarValue> {
+ if !matches!(
+ target_type,
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ ) {
+ return None;
+ }
+
+ match (op, lit_value) {
+ (
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom,
+ ScalarValue::Utf8(Some(_))
+ | ScalarValue::Utf8View(Some(_))
+ | ScalarValue::LargeUtf8(Some(_)),
+ ) => {
+ // Only try for integer types (TODO can we do this for other types
+ // like timestamps)?
+ use DataType::*;
+ if matches!(
+ source_type,
+ Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64
+ ) {
+ let casted = lit_value.cast_to(source_type).ok()?;
+ let round_tripped = casted.cast_to(&lit_value.data_type()).ok()?;
+ if lit_value != &round_tripped {
+ return None;
+ }
+ Some(casted)
+ } else {
+ None
+ }
+ }
+ _ => None,
+ }
+}
+
+/// Computes the source-domain preimage of `CAST(source AS target) OP literal`
+/// for a timestamp precision cast.
+///
+/// `source` and `target` are already unwrapped from one `Dictionary` level;
+/// `source_type` is the original source type, whose dictionary key is applied to
+/// every bound together with the source unit and timezone, so nothing from the
+/// target type leaks into the rewritten predicate. Rejected casts return `None`.
+///
+/// Retained policy: at extreme source values where the cast itself overflows,
+/// `CAST` can error and `TRY_CAST` can return `NULL` while the rewritten
+/// source-unit comparison returns a Boolean.
+fn timestamp_preimage(
+ source_type: &DataType,
+ source: &DataType,
+ target: &DataType,
+ op: Operator,
+ lit_value: &ScalarValue,
+) -> Option<CastPredicatePreimage> {
+ let (
+ DataType::Timestamp(source_unit, source_tz),
+ DataType::Timestamp(target_unit, target_tz),
+ ) = (source, target)
+ else {
+ return None;
+ };
+
+ // Arrow reads a naive value as local wall-clock time in the target timezone
+ // and shifts it to UTC (`adjust_timestamp_to_timezone`), which this unit
+ // arithmetic does not model, so the cast is kept.
+ if source_tz.is_none() && target_tz.is_some() {
+ return None;
+ }
+
+ // The literal must already be a timestamp in the target unit and timezone; it
+ // is never converted, and at most one dictionary wrapper is unwrapped.
+ let literal = match lit_value {
+ ScalarValue::Dictionary(_, inner) => inner.as_ref(),
+ literal => literal,
+ };
+ if literal.data_type() != *target {
+ return None;
+ }
+ let (ScalarValue::TimestampSecond(literal_value, _)
+ | ScalarValue::TimestampMillisecond(literal_value, _)
+ | ScalarValue::TimestampMicrosecond(literal_value, _)
+ | ScalarValue::TimestampNanosecond(literal_value, _)) = literal
+ else {
+ return None;
+ };
+
+ let bound = |value| {
+ let scalar = timestamp_scalar(source_unit, source_tz.clone(), value);
+ match source_type {
+ DataType::Dictionary(key_type, _) => {
+ ScalarValue::Dictionary(key_type.clone(), Box::new(scalar))
+ }
+ _ => scalar,
+ }
+ };
+
+ let source_scale = i128::from(timestamp_unit_scale(source_unit));
+ let target_scale = i128::from(timestamp_unit_scale(target_unit));
+
+ let Some(literal_value) = *literal_value else {
+ // A narrowing NULL has no bucket; widening and same-unit casts keep the
+ // NULL on the source side.
+ return (source_scale <= target_scale)
+ .then(|| CastPredicatePreimage::Exact(bound(None)));
+ };
+
+ // A single source-domain half-open interval `[lower, upper)`: narrowing casts
+ // many values onto the literal, so it is the truncation bucket; a widening or
+ // same-unit cast maps the literal from one source value when it is aligned and
+ // from none otherwise.
+ let (lower, upper) = if source_scale > target_scale {
+ trunc_toward_zero_bucket(literal_value, source_scale / target_scale)?
+ } else {
+ let ratio = target_scale / source_scale;
+ let literal_value = i128::from(literal_value);
+ let floor = literal_value.div_euclid(ratio);
+ let ceil = floor + i128::from(literal_value.rem_euclid(ratio) != 0);
+ (ceil, floor + 1)
+ };
+
+ if lower == upper {
+ // No source value maps to the literal, so only ordered comparisons can
+ // move the cast, and only when both sides share the source timezone.
+ if source_tz != target_tz
+ || !matches!(
+ op,
+ Operator::Lt | Operator::LtEq | Operator::Gt | Operator::GtEq
+ )
+ {
+ return None;
+ }
+ } else if upper == lower + 1 {
+ // Exactly one source value maps to the literal. Its exclusive upper bound
+ // can sit one past `i64::MAX`, so only `lower` is range-checked.
+ return Some(CastPredicatePreimage::Exact(bound(Some(
+ i64::try_from(lower).ok()?,
+ ))));
+ }
+
+ let (lower, upper) = (i64::try_from(lower).ok()?, i64::try_from(upper).ok()?);
+ Interval::try_new(bound(Some(lower)), bound(Some(upper)))
+ .ok()
+ .map(CastPredicatePreimage::Range)
+}
+
+/// Returns the half-open source-domain bucket `[lower, upper)` that truncates
+/// toward zero to `value` when divided by `bucket_width`.
+///
+/// Timestamp precision narrowing follows integer truncation toward zero rather
+/// than mathematical floor. For example, when `bucket_width = 1_000_000`, both
+/// `999_999` and `-999_999` truncate to `0`, while `-1_000_000` truncates to
+/// `-1`.
+///
+/// This makes the inverse bucket depend on the sign of `value`:
+///
+/// * `value > 0`: `[value * width, (value + 1) * width)`
+/// * `value == 0`: `[1 - width, width)`, spanning small negative and positive
+/// values that both truncate to zero
+/// * `value < 0`: `[(value - 1) * width + 1, value * width + 1)`
+///
+/// The arithmetic uses `checked_*` operations and returns `None` if an
+/// intermediate bound cannot be represented as `i128`.
+fn trunc_toward_zero_bucket(value: i64, bucket_width: i128) -> Option<(i128, i128)> {
+ let value = value as i128;
+ match value.cmp(&0) {
+ Ordering::Greater => {
+ let lower = value.checked_mul(bucket_width)?;
+ let upper = value.checked_add(1)?.checked_mul(bucket_width)?;
+ Some((lower, upper))
+ }
+ Ordering::Equal => Some((1_i128.checked_sub(bucket_width)?, bucket_width)),
+ Ordering::Less => {
+ let lower = value
+ .checked_sub(1)?
+ .checked_mul(bucket_width)?
+ .checked_add(1)?;
+ let upper = value.checked_mul(bucket_width)?.checked_add(1)?;
+ Some((lower, upper))
+ }
+ }
+}
+
+fn timestamp_unit_scale(unit: &TimeUnit) -> i64 {
+ match unit {
+ TimeUnit::Second => 1,
+ TimeUnit::Millisecond => MILLISECONDS,
+ TimeUnit::Microsecond => MICROSECONDS,
+ TimeUnit::Nanosecond => NANOSECONDS,
+ }
+}
+
+/// Builds a timestamp scalar in `unit` and `tz`.
+fn timestamp_scalar(
+ unit: &TimeUnit,
+ tz: Option<Arc<str>>,
+ value: Option<i64>,
+) -> ScalarValue {
+ match unit {
+ TimeUnit::Second => ScalarValue::TimestampSecond(value, tz),
+ TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(value, tz),
+ TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(value, tz),
+ TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(value, tz),
+ }
+}
+
/// Returns true if unwrap_cast_in_comparison supports this data type
pub fn is_supported_type(data_type: &DataType) -> bool {
is_supported_numeric_type(data_type)
@@ -124,48 +621,25 @@ fn is_lossy_temporal_cast(from_type: &DataType, to_type: &DataType) -> bool {
|| (is_date_type(to_type) && from_type.is_temporal())
}
-/// Returns true when casting a timestamp from `from_type` to `to_type` loses
-/// timestamp precision.
-///
-/// This is used by comparison cast unwrapping to avoid rewrites such as
-/// `CAST(ts_ns AS timestamp(ms)) = lit_ms` -> `ts_ns = lit_ns`. The original
-/// predicate can match any nanosecond value in the same millisecond, while the
-/// rewritten predicate only matches the exact millisecond boundary.
-pub fn is_timestamp_precision_narrowing_cast(
- from_type: &DataType,
- to_type: &DataType,
-) -> bool {
- let (DataType::Timestamp(from_unit, _), DataType::Timestamp(to_unit, _)) =
- (from_type, to_type)
- else {
- return false;
- };
-
- timestamp_unit_scale(from_unit) > timestamp_unit_scale(to_unit)
-}
-
/// Returns true when casting a date column from `from_type` to `to_type` narrows
/// `Date64` (milliseconds) to `Date32` (days).
///
-/// Like [`is_timestamp_precision_narrowing_cast`], this guards comparison cast
-/// unwrapping against a many-to-one column cast. `CAST(date64 AS Date32) = lit_day`
-/// matches any millisecond within that day, but the rewritten `date64 = lit_ms`
-/// matches only midnight. Arrow does not require `Date64` values to be whole days
-/// (see arrow-rs#5288), so the column may carry sub-day values the planner cannot
-/// see; the widening direction (`Date32 -> Date64`) is injective and stays allowed.
+/// `CAST(date64 AS Date32) = lit_day` matches any millisecond within that day,
+/// but the rewritten `date64 = lit_ms` matches only midnight. Arrow does not
+/// require `Date64` values to be whole days (see arrow-rs#5288), so the column
+/// may carry sub-day values the planner cannot see; the widening direction
+/// (`Date32 -> Date64`) is injective and stays allowed.
+///
+/// This pair is already rejected by the `is_exact_cast_safe` allowlist
+/// (including when wrapped in a `Dictionary`), so predicate rewrites do not need
+/// a separate `Date64 -> Date32` early return.
+#[deprecated(
+ since = "56.0.0",
+ note = "Date64 -> Date32 is rejected by the cast_predicate_preimage allowlist"
+)]
pub fn is_date_narrowing_cast(from_type: &DataType, to_type: &DataType) -> bool {
matches!((from_type, to_type), (DataType::Date64, DataType::Date32))
}
-
-fn timestamp_unit_scale(unit: &TimeUnit) -> i128 {
- match unit {
- TimeUnit::Second => 1,
- TimeUnit::Millisecond => MILLISECONDS as i128,
- TimeUnit::Microsecond => MICROSECONDS as i128,
- TimeUnit::Nanosecond => NANOSECONDS as i128,
- }
-}
-
/// Returns true if unwrap_cast_in_comparison supports this numeric type
fn is_supported_numeric_type(data_type: &DataType) -> bool {
matches!(
@@ -496,18 +970,12 @@ fn try_cast_dictionary(
fn cast_between_timestamp(from: &DataType, to: &DataType, value: i128) -> Option<i64> {
let value = value as i64;
let from_scale = match from {
- DataType::Timestamp(TimeUnit::Second, _) => 1,
- DataType::Timestamp(TimeUnit::Millisecond, _) => MILLISECONDS,
- DataType::Timestamp(TimeUnit::Microsecond, _) => MICROSECONDS,
- DataType::Timestamp(TimeUnit::Nanosecond, _) => NANOSECONDS,
+ DataType::Timestamp(unit, _) => timestamp_unit_scale(unit),
_ => return Some(value),
};
let to_scale = match to {
- DataType::Timestamp(TimeUnit::Second, _) => 1,
- DataType::Timestamp(TimeUnit::Millisecond, _) => MILLISECONDS,
- DataType::Timestamp(TimeUnit::Microsecond, _) => MICROSECONDS,
- DataType::Timestamp(TimeUnit::Nanosecond, _) => NANOSECONDS,
+ DataType::Timestamp(unit, _) => timestamp_unit_scale(unit),
_ => return Some(value),
};
@@ -1016,6 +1484,38 @@ mod tests {
}
#[test]
+ fn test_cast_predicate_preimage_duplicates_input() {
+ let exact = CastPredicatePreimage::Exact(ScalarValue::Int32(Some(1)));
+ let range = CastPredicatePreimage::Range(
+ Interval::try_new(ScalarValue::Int32(Some(0)), ScalarValue::Int32(Some(2)))
+ .unwrap(),
+ );
+ let cases = [
+ (Operator::Eq, true),
+ (Operator::NotEq, true),
+ (Operator::IsDistinctFrom, true),
+ (Operator::IsNotDistinctFrom, true),
+ (Operator::Lt, false),
+ (Operator::LtEq, false),
+ (Operator::Gt, false),
+ (Operator::GtEq, false),
+ ];
+
+ for (op, range_duplicates) in cases {
+ assert!(!exact.duplicates_input(op), "Exact with {op:?}");
+ assert_eq!(
+ range.duplicates_input(op),
+ range_duplicates,
+ "Range with {op:?}"
+ );
+ }
+ }
+
+ #[test]
+ #[expect(
+ deprecated,
+ reason = "kept to verify the deprecated helper's semantics"
+ )]
fn test_is_date_narrowing_cast() {
// Only Date64 -> Date32 narrows (ms -> days, many-to-one).
assert!(is_date_narrowing_cast(&DataType::Date64, &DataType::Date32));
@@ -1253,6 +1753,312 @@ mod tests {
assert_eq!(new_scalar, ScalarValue::TimestampMillisecond(None, None));
}
+ #[test]
+ fn test_cast_predicate_preimage_exact() {
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Int64,
+ Operator::Gt,
+ &ScalarValue::Int64(Some(10)),
+ ScalarValue::Int32(Some(10)),
+ );
+
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Utf8,
+ Operator::Eq,
+ &ScalarValue::Utf8(Some("123".to_string())),
+ ScalarValue::Int32(Some(123)),
+ );
+
+ assert_preimage_none(
+ &DataType::Int32,
+ &DataType::Utf8,
+ Operator::Eq,
+ &ScalarValue::Utf8(Some("0123".to_string())),
+ );
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timestamp_narrowing_range() {
+ // Truncation toward zero decides the bucket by sign: positive, zero spans
+ // both sides, and negative buckets shift by one.
+ for (source_unit, target_unit, lit, lower, upper) in [
+ (
+ TimeUnit::Nanosecond,
+ TimeUnit::Millisecond,
+ 1000,
+ 1_000_000_000,
+ 1_001_000_000,
+ ),
+ (
+ TimeUnit::Nanosecond,
+ TimeUnit::Millisecond,
+ 0,
+ -999_999,
+ 1_000_000,
+ ),
+ (
+ TimeUnit::Nanosecond,
+ TimeUnit::Millisecond,
+ -1,
+ -1_999_999,
+ -999_999,
+ ),
+ (
+ TimeUnit::Microsecond,
+ TimeUnit::Second,
+ 1,
+ 1_000_000,
+ 2_000_000,
+ ),
+ (
+ TimeUnit::Nanosecond,
+ TimeUnit::Second,
+ -2,
+ -2_999_999_999,
+ -1_999_999_999,
+ ),
+ (TimeUnit::Nanosecond, TimeUnit::Microsecond, 0, -999, 1_000),
+ (TimeUnit::Millisecond, TimeUnit::Second, 1, 1000, 2000),
+ (TimeUnit::Millisecond, TimeUnit::Second, -1, -1999, -999),
+ (TimeUnit::Microsecond, TimeUnit::Millisecond, 0, -999, 1000),
+ ] {
+ assert_timestamp_narrowing_range(source_unit, target_unit, lit, lower, upper);
+ }
+ }
+
+ #[test]
+ fn test_timestamp_narrowing_range_rejects_naive_to_timezone() {
+ let source = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ let target = DataType::Timestamp(TimeUnit::Millisecond, Some("+01:00".into()));
+ let literal = ScalarValue::TimestampMillisecond(Some(0), Some("+01:00".into()));
+ assert_preimage_none(&source, &target, Operator::Eq, &literal);
+ }
+
+ #[test]
+ fn test_naive_to_timezone_timestamp_rejected() {
+ // Same unit, widening and narrowing targets: Arrow adjusts naive
+ // values to UTC, which the precision arithmetic does not model. The
+ // literal is unit-aligned (0), so the rejection comes from the timezone
+ // adjustment and not from an unaligned widening bound.
+ for (source_unit, target_unit) in [
+ (TimeUnit::Nanosecond, TimeUnit::Nanosecond),
+ (TimeUnit::Millisecond, TimeUnit::Nanosecond),
+ (TimeUnit::Nanosecond, TimeUnit::Millisecond),
+ ] {
+ let source = DataType::Timestamp(source_unit, None);
+ let target = DataType::Timestamp(target_unit, Some("+07:00".into()));
+ let literal = timestamp_scalar(&target_unit, Some("+07:00".into()), Some(0));
+
+ assert!(
+ exact_preimage_cast(&source, &target, &literal).is_none(),
+ "{source:?} -> {target:?} must not have an exact preimage"
+ );
+ for op in [
+ Operator::Eq,
+ Operator::Gt,
+ Operator::LtEq,
+ Operator::IsDistinctFrom,
+ ] {
+ assert_preimage_none(&source, &target, op, &literal);
+ }
+ }
+
+ // Conservative for every timezone, including fixed zero offsets:
+ // `Some("UTC")` is rejected like `Some("+07:00")`.
+ let source = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ for tz in ["UTC", "+00:00"] {
+ let target = DataType::Timestamp(TimeUnit::Nanosecond, Some(tz.into()));
+ assert_preimage_none(
+ &source,
+ &target,
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(0), Some(tz.into())),
+ );
+ }
+
+ // Positive control: a timezone-aware source keeps the accepted same-unit
+ // and widening policies.
+ let aware_ms = DataType::Timestamp(TimeUnit::Millisecond, Some("+07:00".into()));
+ let aware_ns = DataType::Timestamp(TimeUnit::Nanosecond, Some("+07:00".into()));
+ assert_preimage_exact(
+ &aware_ns,
+ &aware_ns,
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(7), Some("+07:00".into())),
+ ScalarValue::TimestampNanosecond(Some(7), Some("+07:00".into())),
+ );
+ assert_preimage_exact(
+ &aware_ms,
+ &aware_ns,
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(7_000_000), Some("+07:00".into())),
+ ScalarValue::TimestampMillisecond(Some(7), Some("+07:00".into())),
+ );
+ }
+
+ #[test]
+ fn test_naive_to_timezone_dictionary_wrapped_rejected() {
+ let dict = |inner: DataType| {
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(inner))
+ };
+ let naive_ns = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ let aware_ns = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()));
+ let literal = ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into()));
+
+ // Timestamp preimages unwrap one level of dictionary, so a wrapped
+ // naive -> aware pair is rejected like the bare pair.
+ assert_preimage_none(&naive_ns, &aware_ns, Operator::Eq, &literal);
+ assert_preimage_none(
+ &dict(naive_ns.clone()),
+ &dict(aware_ns.clone()),
+ Operator::Eq,
+ &literal,
+ );
+ assert_preimage_none(&dict(naive_ns.clone()), &aware_ns, Operator::Eq, &literal);
+ assert_preimage_none(&naive_ns, &dict(aware_ns.clone()), Operator::Eq, &literal);
+
+ // Safe controls stay allowed: aware -> aware unit widening and identity.
+ assert_preimage_exact(
+ &DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
+ &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(1_000_000), Some("UTC".into())),
+ ScalarValue::TimestampMillisecond(Some(1), Some("UTC".into())),
+ );
+ assert_preimage_exact(
+ &aware_ns,
+ &aware_ns,
+ Operator::Eq,
+ &literal,
+ ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into())),
+ );
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timestamp_widening_ordered() {
+ let ts_ms = DataType::Timestamp(TimeUnit::Millisecond, None);
+ let ts_ns = DataType::Timestamp(TimeUnit::Nanosecond, None);
+
+ assert_preimage_exact(
+ &ts_ms,
+ &ts_ns,
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(123_000_000), None),
+ ScalarValue::TimestampMillisecond(Some(123), None),
+ );
+
+ for (source_unit, target_unit) in [
+ (TimeUnit::Second, TimeUnit::Millisecond),
+ (TimeUnit::Second, TimeUnit::Microsecond),
+ (TimeUnit::Second, TimeUnit::Nanosecond),
+ (TimeUnit::Millisecond, TimeUnit::Microsecond),
+ (TimeUnit::Millisecond, TimeUnit::Nanosecond),
+ (TimeUnit::Microsecond, TimeUnit::Nanosecond),
+ ] {
+ assert_timestamp_widening_ordered(source_unit, target_unit);
+ }
+
+ assert_preimage_none(
+ &ts_ms,
+ &ts_ns,
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(123_456_789), None),
+ );
+ }
+
+ #[test]
+ fn test_timestamp_widening_ordered_rejections_and_timezone() {
+ let source_type = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into()));
+ let target_type = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()));
+ let literal =
+ ScalarValue::TimestampNanosecond(Some(123_456_789), Some("UTC".into()));
+
+ // A non-aligned literal has no singleton source value, so ordered
+ // comparisons use the empty interval and equality-like ones keep the
+ // cast. `123_456_789 / 1_000_000` floors to 123, so the interval is
+ // `[124, 124)`.
+ let empty = Some(CastPredicatePreimage::Range(
+ Interval::try_new(
+ ScalarValue::TimestampMillisecond(Some(124), Some("UTC".into())),
+ ScalarValue::TimestampMillisecond(Some(124), Some("UTC".into())),
+ )
+ .unwrap(),
+ ));
+ for op in [Operator::GtEq, Operator::Gt, Operator::Lt, Operator::LtEq] {
+ assert_eq!(
+ cast_predicate_preimage(&source_type, &target_type, op, &literal)
+ .unwrap(),
+ empty
+ );
+ }
+ for op in [
+ Operator::Eq,
+ Operator::NotEq,
+ Operator::IsDistinctFrom,
+ Operator::IsNotDistinctFrom,
+ ] {
+ assert_preimage_none(&source_type, &target_type, op, &literal);
+ }
+
+ // A literal of the wrong unit or the wrong timezone is not converted.
+ for literal in [
+ ScalarValue::TimestampMicrosecond(Some(123_456), Some("UTC".into())),
+ ScalarValue::TimestampNanosecond(Some(123_456_789), None),
+ ScalarValue::Int64(Some(123_456_789)),
+ ] {
+ for op in [Operator::Eq, Operator::GtEq, Operator::LtEq] {
+ assert_preimage_none(&source_type, &target_type, op, &literal);
+ }
+ }
+
+ // A source/target timezone change has no source-domain bound.
+ assert_preimage_none(
+ &source_type,
+ &DataType::Timestamp(TimeUnit::Nanosecond, None),
+ Operator::GtEq,
+ &ScalarValue::TimestampNanosecond(Some(123_456_789), None),
+ );
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timestamp_null_literal() {
+ let ts_ns = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ let ts_ms = DataType::Timestamp(TimeUnit::Millisecond, None);
+ let null_ms = ScalarValue::TimestampMillisecond(None, None);
+
+ // Narrowing a typed NULL literal has no source-domain bucket.
+ for op in [
+ Operator::Eq,
+ Operator::NotEq,
+ Operator::IsDistinctFrom,
+ Operator::IsNotDistinctFrom,
+ Operator::Gt,
+ Operator::LtEq,
+ ] {
+ assert_preimage_none(&ts_ns, &ts_ms, op, &null_ms);
+ }
+
+ // Widening and same-unit casts keep the NULL result on the source side.
+ // The literal must be typed in the target unit, as SQL coercion produces.
+ for target_unit in [TimeUnit::Millisecond, TimeUnit::Nanosecond] {
+ let target_type = DataType::Timestamp(target_unit, None);
+ assert_eq!(
+ cast_predicate_preimage(
+ &ts_ms,
+ &target_type,
+ Operator::Eq,
+ &timestamp_scalar(&target_unit, None, None),
+ )
+ .unwrap(),
+ Some(CastPredicatePreimage::Exact(
+ ScalarValue::TimestampMillisecond(None, None)
+ ))
+ );
+ }
+ }
+
#[test]
fn test_try_cast_to_string_type() {
let scalars = vec![
@@ -1688,4 +2494,959 @@ mod tests {
ExpectedCast::NoValue,
);
}
+
+ #[test]
+ fn test_cast_predicate_preimage_extreme_literals() {
+ let ts_ns = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ let ts_ms = DataType::Timestamp(TimeUnit::Millisecond, None);
+
+ // These millisecond values expand beyond i64 range in nanoseconds, so
+ // the bucket has no representable endpoint and the preimage is None
+ // rather than a clamped or unbounded interval.
+ for value in [i64::MAX, i64::MIN] {
+ assert_preimage_none(
+ &ts_ns,
+ &ts_ms,
+ Operator::Eq,
+ &ScalarValue::TimestampMillisecond(Some(value), None),
+ );
+ }
+
+ // A widening literal at the i64 boundary produces a checked source
+ // endpoint: `i64::MAX` ns floors to 9223372036s with a 854775807ns
+ // remainder, so the empty-interval bound `floor + 1` is 9223372037s,
+ // still a representable second.
+ let source_type = DataType::Timestamp(TimeUnit::Second, None);
+ let target_type = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ let empty = Some(CastPredicatePreimage::Range(
+ Interval::try_new(
+ ScalarValue::TimestampSecond(Some(9_223_372_037), None),
+ ScalarValue::TimestampSecond(Some(9_223_372_037), None),
+ )
+ .unwrap(),
+ ));
+ assert_eq!(
+ cast_predicate_preimage(
+ &source_type,
+ &target_type,
+ Operator::Lt,
+ &ScalarValue::TimestampNanosecond(Some(i64::MAX), None),
+ )
+ .unwrap(),
+ empty
+ );
+
+ // A bucket whose upper endpoint is not representable in the source unit
+ // yields `None` rather than a clamped or unbounded interval.
+ assert_preimage_none(
+ &ts_ns,
+ &ts_ms,
+ Operator::Eq,
+ &ScalarValue::TimestampMillisecond(Some(9_223_372_036_854), None),
+ );
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timestamp_identity_anchors() {
+ // Same-unit casts anchor both ends of the domain, zero and NULL: the
+ // literal passes through untouched, including `i64::MAX`, whose exclusive
+ // upper bound would not fit in an `i64`.
+ for (unit, scalar) in [
+ (TimeUnit::Second, Some(i64::MIN)),
+ (TimeUnit::Second, None),
+ (TimeUnit::Second, Some(0)),
+ (TimeUnit::Second, Some(i64::MAX)),
+ (TimeUnit::Millisecond, Some(i64::MIN)),
+ (TimeUnit::Millisecond, Some(0)),
+ (TimeUnit::Millisecond, Some(i64::MAX)),
+ (TimeUnit::Microsecond, Some(i64::MIN)),
+ (TimeUnit::Microsecond, Some(0)),
+ (TimeUnit::Microsecond, Some(i64::MAX)),
+ (TimeUnit::Nanosecond, Some(i64::MIN)),
+ (TimeUnit::Nanosecond, Some(0)),
+ (TimeUnit::Nanosecond, Some(i64::MAX)),
+ ] {
+ let data_type = DataType::Timestamp(unit, None);
+ let literal = timestamp_scalar(&unit, None, scalar);
+ assert_preimage_exact(
+ &data_type,
+ &data_type,
+ Operator::Eq,
+ &literal,
+ literal.clone(),
+ );
+ assert_eq!(
+ exact_preimage_cast(&data_type, &data_type, &literal),
+ Some(literal)
+ );
+ }
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timestamp_rejects_nested_wrapper() {
+ let dict = |inner: ScalarValue| {
+ ScalarValue::Dictionary(Box::new(DataType::Int32), Box::new(inner))
+ };
+ let source_type = DataType::Timestamp(TimeUnit::Millisecond, None);
+ let target_type = DataType::Timestamp(TimeUnit::Nanosecond, None);
+ let literal = ScalarValue::TimestampNanosecond(Some(123_000_000), None);
+
+ // One dictionary level is unwrapped.
+ assert_preimage_exact(
+ &source_type,
+ &target_type,
+ Operator::Eq,
+ &dict(literal.clone()),
+ ScalarValue::TimestampMillisecond(Some(123), None),
+ );
+ // A second wrapper is rejected rather than recursively unwrapped.
+ assert_preimage_none(
+ &source_type,
+ &target_type,
+ Operator::Eq,
+ &dict(dict(literal)),
+ );
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timezone_preservation() {
+ let source_type =
+ DataType::Timestamp(TimeUnit::Nanosecond, Some("+05:30".into()));
+ let target_type = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into()));
+ let lit = ScalarValue::TimestampMillisecond(Some(1000), Some("UTC".into()));
+
+ let result =
+ cast_predicate_preimage(&source_type, &target_type, Operator::Eq, &lit)
+ .unwrap();
+ assert_preimage_data_type(result.as_ref(), &source_type);
+
+ match result {
+ Some(CastPredicatePreimage::Range(interval)) => {
+ let (lower, upper) = interval.into_bounds();
+ assert_eq!(
+ lower,
+ ScalarValue::TimestampNanosecond(
+ Some(1_000_000_000),
+ Some("+05:30".into())
+ ),
+ "lower bound should preserve source timezone +05:30"
+ );
+ assert_eq!(
+ upper,
+ ScalarValue::TimestampNanosecond(
+ Some(1_001_000_000),
+ Some("+05:30".into())
+ ),
+ "upper bound should preserve source timezone +05:30"
+ );
+ }
+ other => panic!("Expected CastPredicatePreimage::Range but got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_cast_predicate_preimage_timestamp_dictionary() {
+ let key_type = Box::new(DataType::Int32);
+ let dict = |key: &DataType, inner: DataType| {
+ DataType::Dictionary(Box::new(key.clone()), Box::new(inner))
+ };
+ let wrap = |value: ScalarValue| {
+ ScalarValue::Dictionary(key_type.clone(), Box::new(value))
+ };
+ let source_tz: Option<Arc<str>> = Some("+05:30".into());
+
+ // The target dictionary key (`Int16`) differs from the source key
+ // (`Int32`) so a leaked target key cannot go unnoticed.
+ let source_type = dict(
+ &DataType::Int32,
+ DataType::Timestamp(TimeUnit::Nanosecond, source_tz.clone()),
+ );
+ let target_type = dict(
+ &DataType::Int16,
+ DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
+ );
+ let lit = ScalarValue::TimestampMillisecond(Some(1000), Some("UTC".into()));
+
+ let result =
+ cast_predicate_preimage(&source_type, &target_type, Operator::Eq, &lit)
+ .unwrap();
+ // `ScalarValue` equality ignores timestamp timezones, so the data type
+ // (dictionary key + source unit + source timezone) is asserted too.
+ assert_preimage_data_type(result.as_ref(), &source_type);
+ assert_eq!(
+ result,
+ Some(CastPredicatePreimage::Range(
+ Interval::try_new(
+ wrap(ScalarValue::TimestampNanosecond(
+ Some(1_000_000_000),
+ source_tz.clone(),
+ )),
+ wrap(ScalarValue::TimestampNanosecond(
+ Some(1_001_000_000),
+ source_tz,
+ )),
+ )
+ .unwrap()
+ ))
+ );
+
+ // Exact preimages are dictionary-wrapped the same way.
+ let ms_source = dict(
+ &DataType::Int32,
+ DataType::Timestamp(TimeUnit::Millisecond, None),
+ );
+ let ns_target = dict(
+ &DataType::Int16,
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ );
+ let result = cast_predicate_preimage(
+ &ms_source,
+ &ns_target,
+ Operator::Eq,
+ &ScalarValue::TimestampNanosecond(Some(123_000_000), None),
+ )
+ .unwrap();
+ assert_preimage_data_type(result.as_ref(), &ms_source);
+ assert_eq!(
+ result,
+ Some(CastPredicatePreimage::Exact(wrap(
+ ScalarValue::TimestampMillisecond(Some(123), None)
+ )))
+ );
+
+ // A dictionary-wrapped literal is accepted for the same cast.
+ let result = cast_predicate_preimage(
+ &ms_source,
+ &ns_target,
+ Operator::Eq,
+ &wrap(ScalarValue::TimestampNanosecond(Some(123_000_000), None)),
+ )
+ .unwrap();
+ assert_preimage_data_type(result.as_ref(), &ms_source);
+ assert_eq!(
+ result,
+ Some(CastPredicatePreimage::Exact(wrap(
+ ScalarValue::TimestampMillisecond(Some(123), None)
+ )))
+ );
+
+ // `exact_preimage_cast` (the IN-list path) returns the same source-domain
+ // scalar, still dictionary-wrapped with the source key.
+ let scalar = exact_preimage_cast(
+ &ms_source,
+ &ns_target,
+ &ScalarValue::TimestampNanosecond(Some(123_000_000), None),
+ )
+ .unwrap();
+ assert_eq!(scalar.data_type(), ms_source);
+ assert_eq!(
+ scalar,
+ wrap(ScalarValue::TimestampMillisecond(Some(123), None))
+ );
+ }
+
+ // ── Gate safety tests ──────────────────────────────────────────────
+
+ #[test]
+ fn test_exact_predicate_gate_safe() {
+ // integer widening
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Int64,
+ Operator::Eq,
+ &ScalarValue::Int64(Some(10)),
+ ScalarValue::Int32(Some(10)),
+ );
+ // unsigned → wider signed
+ assert_preimage_exact(
+ &DataType::UInt32,
+ &DataType::Int64,
+ Operator::Eq,
+ &ScalarValue::Int64(Some(100)),
+ ScalarValue::UInt32(Some(100)),
+ );
+ // int → decimal scale-up
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Decimal128(12, 2),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(10000), 12, 2),
+ ScalarValue::Int32(Some(100)),
+ );
+ // uint → decimal at the same scale
+ assert_preimage_exact(
+ &DataType::UInt64,
+ &DataType::Decimal128(20, 0),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(123), 20, 0),
+ ScalarValue::UInt64(Some(123)),
+ );
+ // decimal scale-up at sub-maximum precision
+ assert_preimage_exact(
+ &DataType::Decimal128(10, 2),
+ &DataType::Decimal128(18, 4),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(1230000), 18, 4),
+ ScalarValue::Decimal128(Some(12300), 10, 2),
+ );
+ // Date32 ↔ Int32
+ assert_preimage_exact(
+ &DataType::Date32,
+ &DataType::Int32,
+ Operator::Eq,
+ &ScalarValue::Int32(Some(19000)),
+ ScalarValue::Date32(Some(19000)),
+ );
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Date32,
+ Operator::Eq,
+ &ScalarValue::Date32(Some(19000)),
+ ScalarValue::Int32(Some(19000)),
+ );
+ // Date32 → Date64 is injective; literals must still land on a day
+ // boundary to be exactly convertible back to the source type.
+ assert_preimage_exact(
+ &DataType::Date32,
+ &DataType::Date64,
+ Operator::Eq,
+ &ScalarValue::Date64(Some(1_641_600_000_000)),
+ ScalarValue::Date32(Some(19000)),
+ );
+ // FixedSizeBinary(n) → Binary
+ assert_preimage_exact(
+ &DataType::FixedSizeBinary(4),
+ &DataType::Binary,
+ Operator::Eq,
+ &ScalarValue::Binary(Some(vec![1, 2, 3, 4])),
+ ScalarValue::FixedSizeBinary(4, Some(vec![1, 2, 3, 4])),
+ );
+ }
+
+ #[test]
+ fn test_exact_predicate_gate_blocked() {
+ // numeric narrowing
+ assert_preimage_none(
+ &DataType::Int64,
+ &DataType::Int32,
+ Operator::Eq,
+ &ScalarValue::Int32(Some(10)),
+ );
+ // signed → unsigned (even small values)
+ assert_preimage_none(
+ &DataType::Int32,
+ &DataType::UInt32,
+ Operator::Eq,
+ &ScalarValue::UInt32(Some(10)),
+ );
+ // unsigned → narrower signed
+ assert_preimage_none(
+ &DataType::UInt64,
+ &DataType::Int64,
+ Operator::Eq,
+ &ScalarValue::Int64(Some(10)),
+ );
+ // decimal → int
+ assert_preimage_none(
+ &DataType::Decimal128(18, 2),
+ &DataType::Int64,
+ Operator::Eq,
+ &ScalarValue::Int64(Some(123)),
+ );
+ // decimal scale narrowing
+ assert_preimage_none(
+ &DataType::Decimal128(18, 2),
+ &DataType::Decimal128(18, 1),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(1230), 18, 1),
+ );
+ // negative decimal scale is not accepted for exact predicate rewrites
+ assert_preimage_none(
+ &DataType::Decimal128(10, -1),
+ &DataType::Decimal128(18, 0),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(120), 18, 0),
+ );
+ assert_preimage_none(
+ &DataType::Int32,
+ &DataType::Decimal128(12, -1),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(120), 12, -1),
+ );
+ // Binary → FixedSizeBinary (not a family-safe source cast)
+ assert_preimage_none(
+ &DataType::Binary,
+ &DataType::FixedSizeBinary(4),
+ Operator::Eq,
+ &ScalarValue::FixedSizeBinary(4, Some(vec![1, 2, 3, 4])),
+ );
+ }
+
+ #[test]
+ fn test_decimal_scale_up_preimages() {
+ let ops = [
+ Operator::Eq,
+ Operator::NotEq,
+ Operator::Lt,
+ Operator::LtEq,
+ Operator::Gt,
+ Operator::GtEq,
+ Operator::IsDistinctFrom,
+ Operator::IsNotDistinctFrom,
+ ];
+
+ // One same-variant representative per decimal variant. Positive,
+ // negative and zero literals use all eight comparison operators, and a
+ // NULL literal casts to a NULL preimage.
+ let cases = [
+ (
+ DataType::Decimal32(9, 1),
+ DataType::Decimal32(9, 2),
+ ScalarValue::Decimal32(Some(1200), 9, 2),
+ ScalarValue::Decimal32(Some(120), 9, 1),
+ ),
+ (
+ DataType::Decimal64(18, 1),
+ DataType::Decimal64(18, 2),
+ ScalarValue::Decimal64(Some(1200), 18, 2),
+ ScalarValue::Decimal64(Some(120), 18, 1),
+ ),
+ (
+ DataType::Decimal128(38, 1),
+ DataType::Decimal128(38, 2),
+ ScalarValue::Decimal128(Some(1200), 38, 2),
+ ScalarValue::Decimal128(Some(120), 38, 1),
+ ),
+ ];
+ for (source, target, literal, expected) in &cases {
+ for op in ops {
+ assert_preimage_exact(source, target, op, literal, expected.clone());
+ }
+ let (negative, negative_expected, zero, zero_expected) = match source {
+ DataType::Decimal32(_, _) => (
+ ScalarValue::Decimal32(Some(-1200), 9, 2),
+ ScalarValue::Decimal32(Some(-120), 9, 1),
+ ScalarValue::Decimal32(Some(0), 9, 2),
+ ScalarValue::Decimal32(Some(0), 9, 1),
+ ),
+ DataType::Decimal64(_, _) => (
+ ScalarValue::Decimal64(Some(-1200), 18, 2),
+ ScalarValue::Decimal64(Some(-120), 18, 1),
+ ScalarValue::Decimal64(Some(0), 18, 2),
+ ScalarValue::Decimal64(Some(0), 18, 1),
+ ),
+ _ => (
+ ScalarValue::Decimal128(Some(-1200), 38, 2),
+ ScalarValue::Decimal128(Some(-120), 38, 1),
+ ScalarValue::Decimal128(Some(0), 38, 2),
+ ScalarValue::Decimal128(Some(0), 38, 1),
+ ),
+ };
+ for (literal, expected) in
+ [(negative, negative_expected), (zero, zero_expected)]
+ {
+ for op in ops {
+ assert_preimage_exact(source, target, op, &literal, expected.clone());
+ }
+ }
+ let null = ScalarValue::try_from(target).unwrap();
+ assert_preimage_exact(
+ source,
+ target,
+ Operator::Eq,
+ &null,
+ ScalarValue::try_from(source).unwrap(),
+ );
+ }
+
+ // Additional shapes admitted by the scale-only rule: the motivating
+ // sub-maximum 30,2 / 28,0 / 36,4 sources, a precision-37 target,
+ // cross-width scale-up, cross-variant widening and signed / unsigned
+ // integer sources.
+ let extra_cases = [
+ (
+ DataType::Decimal128(30, 2),
+ DataType::Decimal128(38, 15),
+ ScalarValue::Decimal128(Some(4 * 10_i128.pow(15)), 38, 15),
+ ScalarValue::Decimal128(Some(400), 30, 2),
+ ),
+ (
+ DataType::Decimal128(28, 0),
+ DataType::Decimal128(38, 15),
+ ScalarValue::Decimal128(Some(4 * 10_i128.pow(15)), 38, 15),
+ ScalarValue::Decimal128(Some(4), 28, 0),
+ ),
+ (
+ DataType::Decimal128(36, 4),
+ DataType::Decimal128(38, 15),
+ ScalarValue::Decimal128(Some(4 * 10_i128.pow(15)), 38, 15),
+ ScalarValue::Decimal128(Some(40_000), 36, 4),
+ ),
+ (
+ DataType::Decimal128(37, 0),
+ DataType::Decimal128(38, 2),
+ ScalarValue::Decimal128(Some(1200), 38, 2),
+ ScalarValue::Decimal128(Some(12), 37, 0),
+ ),
+ (
+ DataType::Decimal128(38, 1),
+ DataType::Decimal128(37, 2),
+ ScalarValue::Decimal128(Some(1200), 37, 2),
+ ScalarValue::Decimal128(Some(120), 38, 1),
+ ),
+ (
+ DataType::Decimal64(18, 0),
+ DataType::Decimal128(38, 30),
+ ScalarValue::Decimal128(Some(10_i128.pow(30)), 38, 30),
+ ScalarValue::Decimal64(Some(1), 18, 0),
+ ),
+ (
+ DataType::Decimal128(38, 1),
+ DataType::Decimal64(18, 2),
+ ScalarValue::Decimal64(Some(1200), 18, 2),
+ ScalarValue::Decimal128(Some(120), 38, 1),
+ ),
+ (
+ DataType::Int32,
+ DataType::Decimal128(10, 2),
+ ScalarValue::Decimal128(Some(10000), 10, 2),
+ ScalarValue::Int32(Some(100)),
+ ),
+ (
+ DataType::UInt64,
+ DataType::Decimal128(20, 2),
+ ScalarValue::Decimal128(Some(12300), 20, 2),
+ ScalarValue::UInt64(Some(123)),
+ ),
+ ];
+ for (source, target, literal, expected) in &extra_cases {
+ for op in ops {
+ assert_preimage_exact(source, target, op, literal, expected.clone());
+ }
+ }
+
+ // The scale-only rule still rejects reduced or negative target scales,
+ // nonaligned literals, decimal → integer and Decimal256.
+ assert_preimage_none(
+ &DataType::Decimal128(38, 1),
+ &DataType::Decimal128(38, 0),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(12), 38, 0),
+ );
+ assert_preimage_none(
+ &DataType::Decimal128(18, 0),
+ &DataType::Decimal128(10, -1),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(12), 10, -1),
+ );
+ // Negative source scales are rejected by the gate instead of reaching
+ // the scale arithmetic: both plain and one-level dictionary identity
+ // return None.
+ let negative_scale = DataType::Decimal128(10, -1);
+ assert_preimage_none(
+ &negative_scale,
+ &negative_scale,
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(12), 10, -1),
+ );
+ let dict_negative_scale =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(negative_scale));
+ assert_preimage_none(
+ &dict_negative_scale,
+ &dict_negative_scale,
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(12), 10, -1),
+ );
+ for literal in [
+ ScalarValue::Decimal128(Some(1201), 38, 2),
+ ScalarValue::Decimal128(Some(-1201), 38, 2),
+ ] {
+ assert_preimage_none(
+ &DataType::Decimal128(38, 1),
+ &DataType::Decimal128(38, 2),
+ Operator::Eq,
+ &literal,
+ );
+ }
+ assert_preimage_none(
+ &DataType::Decimal128(18, 2),
+ &DataType::Int64,
+ Operator::Eq,
+ &ScalarValue::Int64(Some(123)),
+ );
+ assert_preimage_none(
+ &DataType::Decimal256(76, 1),
+ &DataType::Decimal256(76, 2),
+ Operator::Eq,
+ &ScalarValue::Decimal256(Some(1200.into()), 76, 2),
+ );
+
+ // The literal still has to convert exactly into the source type and
+ // into its range: a negative literal is not a UInt64 preimage, and a
+ // cross-width decimal literal with sub-unit digits has no decimal128
+ // preimage at the source scale.
+ assert_preimage_none(
+ &DataType::UInt64,
+ &DataType::Decimal128(20, 2),
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(-12300), 20, 2),
+ );
+ assert_preimage_none(
+ &DataType::Decimal128(38, 1),
+ &DataType::Decimal64(18, 2),
+ Operator::Eq,
+ &ScalarValue::Decimal64(Some(1201), 18, 2),
+ );
+
+ // Arrow's strict cast errors and TRY_CAST returns NULL when this valid
+ // Decimal128 source exceeds the target's integer capacity.
+ let target_type = DataType::Decimal128(38, 2);
+ let source = ScalarValue::Decimal128(Some(i128::MAX / 10), 38, 1);
+ let array = source.to_array_of_size(1).unwrap();
+ assert!(
+ cast_with_options(
+ &array,
+ &target_type,
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ )
+ .is_err()
+ );
+ assert_eq!(
+ cast_with_options(&array, &target_type, &CastOptions::default())
+ .unwrap()
+ .null_count(),
+ 1,
+ );
+
+ // Newly admitted integer source: i32::MAX is a valid Int32 value but
+ // overflows decimal(10, 2). The rewritten predicate compares the
+ // integer preimage instead, so an out-of-range row is simply FALSE for
+ // `=` rather than an error or NULL.
+ let int_target = DataType::Decimal128(10, 2);
+ let int_literal = ScalarValue::Decimal128(Some(10000), 10, 2);
+ let int_preimage =
+ exact_preimage_cast(&DataType::Int32, &int_target, &int_literal)
+ .expect("integer scale-up preimage");
+ assert_eq!(int_preimage, ScalarValue::Int32(Some(100)));
+ let int_max = ScalarValue::Int32(Some(i32::MAX));
+ let int_max_array = int_max.to_array_of_size(1).unwrap();
+ assert!(
+ cast_with_options(
+ &int_max_array,
+ &int_target,
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ )
+ .is_err()
+ );
+ assert_eq!(
+ cast_with_options(&int_max_array, &int_target, &CastOptions::default())
+ .unwrap()
+ .null_count(),
+ 1,
+ );
+ assert_eq!(int_max.partial_cmp(&int_preimage), Some(Ordering::Greater));
+
+ // Newly admitted sub-maximum decimal source: a valid decimal(30, 2)
+ // value outside decimal(38, 15)'s integer capacity errors under a
+ // strict cast and is NULL under TRY_CAST, while the rewritten equality
+ // predicate is a boolean.
+ let wide_source = DataType::Decimal128(30, 2);
+ let wide_target = DataType::Decimal128(38, 15);
+ let wide_literal = ScalarValue::Decimal128(Some(4 * 10_i128.pow(15)), 38, 15);
+ let wide_preimage =
+ exact_preimage_cast(&wide_source, &wide_target, &wide_literal)
+ .expect("sub-maximum scale-up preimage");
+ assert_eq!(wide_preimage, ScalarValue::Decimal128(Some(400), 30, 2));
+ let out_of_range = ScalarValue::Decimal128(Some(10_i128.pow(25)), 30, 2);
+ let out_of_range_array = out_of_range.to_array_of_size(1).unwrap();
+ assert!(
+ cast_with_options(
+ &out_of_range_array,
+ &wide_target,
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ )
+ .is_err()
+ );
+ assert_eq!(
+ cast_with_options(&out_of_range_array, &wide_target, &CastOptions::default())
+ .unwrap()
+ .null_count(),
+ 1,
+ );
+ assert_eq!(
+ out_of_range.partial_cmp(&wide_preimage),
+ Some(Ordering::Greater)
+ );
+
+ // Newly admitted cross-width decimal source: decimal128 to a narrower
+ // decimal64, where a large but valid source value overflows.
+ let cross_source = ScalarValue::Decimal128(Some(i128::MAX / 10), 38, 1);
+ let cross_target = DataType::Decimal64(18, 2);
+ let cross_preimage = exact_preimage_cast(
+ &cross_source.data_type(),
+ &cross_target,
+ &ScalarValue::Decimal64(Some(1200), 18, 2),
+ )
+ .expect("cross-width scale-up preimage");
+ assert_eq!(cross_preimage, ScalarValue::Decimal128(Some(120), 38, 1));
+ let cross_array = cross_source.to_array_of_size(1).unwrap();
+ assert!(
+ cast_with_options(
+ &cross_array,
+ &cross_target,
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ )
+ .is_err()
+ );
+ assert_eq!(
+ cast_with_options(&cross_array, &cross_target, &CastOptions::default())
+ .unwrap()
+ .null_count(),
+ 1,
+ );
+ assert_eq!(
+ cross_source.partial_cmp(&cross_preimage),
+ Some(Ordering::Greater)
+ );
+
+ // This larger-scale Decimal128 value overflows the physical i128
+ // representation during the cast, not only the target precision.
+ let extreme = ScalarValue::Decimal128(Some(10_i128.pow(37)), 38, 2);
+ let extreme_array = extreme.to_array_of_size(1).unwrap();
+ let extreme_target = DataType::Decimal128(38, 15);
+ assert_preimage_exact(
+ &DataType::Decimal128(38, 2),
+ &extreme_target,
+ Operator::Eq,
+ &ScalarValue::Decimal128(Some(4 * 10_i128.pow(15)), 38, 15),
+ ScalarValue::Decimal128(Some(400), 38, 2),
+ );
+ assert!(
+ cast_with_options(
+ &extreme_array,
+ &extreme_target,
+ &CastOptions {
+ safe: false,
+ ..Default::default()
+ },
+ )
+ .is_err()
+ );
+ assert_eq!(
+ cast_with_options(&extreme_array, &extreme_target, &CastOptions::default())
+ .unwrap()
+ .null_count(),
+ 1,
+ );
+ }
+
+ // ── Integer → string distinctness tests ─────────────────────────────
+
+ #[test]
+ fn test_cast_to_string_is_distinct_from_round_trip() {
+ // IS DISTINCT FROM with round-trippable string
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Utf8,
+ Operator::IsDistinctFrom,
+ &ScalarValue::Utf8(Some("123".to_string())),
+ ScalarValue::Int32(Some(123)),
+ );
+ // IS NOT DISTINCT FROM with round-trippable string
+ assert_preimage_exact(
+ &DataType::Int32,
+ &DataType::Utf8,
+ Operator::IsNotDistinctFrom,
+ &ScalarValue::Utf8(Some("123".to_string())),
+ ScalarValue::Int32(Some(123)),
+ );
+ // IS NOT DISTINCT FROM with non-round-trippable string (leading zero)
+ assert_preimage_none(
+ &DataType::Int32,
+ &DataType::Utf8,
+ Operator::IsNotDistinctFrom,
+ &ScalarValue::Utf8(Some("0123".to_string())),
+ );
+ // IS DISTINCT FROM with non-round-trippable string
+ assert_preimage_none(
+ &DataType::Int32,
+ &DataType::Utf8,
+ Operator::IsDistinctFrom,
+ &ScalarValue::Utf8(Some("0123".to_string())),
+ );
+ }
+
+ // ── Helpers ─────────────────────────────────────────────────────────
+
+ fn assert_preimage_exact(
+ source_type: &DataType,
+ target_type: &DataType,
+ op: Operator,
+ lit_value: &ScalarValue,
+ expected_source: ScalarValue,
+ ) {
+ let result =
+ cast_predicate_preimage(source_type, target_type, op, lit_value).unwrap();
+ // The preimage must be a source-domain value: same unit, timezone and
+ // dictionary key as the source, never the target.
+ assert_preimage_data_type(result.as_ref(), source_type);
+ assert_eq!(
+ result,
+ Some(CastPredicatePreimage::Exact(expected_source)),
+ "expected Exact preimage for {source_type:?} → {target_type:?} {op:?} {lit_value:?}"
+ );
+ }
+
+ fn assert_preimage_none(
+ source_type: &DataType,
+ target_type: &DataType,
+ op: Operator,
+ lit_value: &ScalarValue,
+ ) {
+ let result =
+ cast_predicate_preimage(source_type, target_type, op, lit_value).unwrap();
+ assert_eq!(
+ result, None,
+ "expected None preimage for {source_type:?} → {target_type:?} {op:?} {lit_value:?}"
+ );
+ }
+
+ /// Asserts that every endpoint of a preimage has the expected data type.
+ ///
+ /// [`ScalarValue`] equality ignores a timestamp's timezone metadata, and a
+ /// dictionary wrapper is compared only by key type plus inner value, so
+ /// value-based assertions alone cannot catch a leaked target timezone, unit
+ /// or dictionary key. This check must accompany them.
+ fn assert_preimage_data_type(
+ preimage: Option<&CastPredicatePreimage>,
+ expected: &DataType,
+ ) {
+ let (lower, upper) = match preimage {
+ Some(CastPredicatePreimage::Exact(value)) => (value, value),
+ Some(CastPredicatePreimage::Range(interval)) => {
+ (interval.lower(), interval.upper())
+ }
+ None => return,
+ };
+ assert_eq!(&lower.data_type(), expected);
+ assert_eq!(&upper.data_type(), expected);
+ }
+
+ /// Asserts the widening preimage shapes for a `source → target` unit pair
+ /// with `target` finer than `source`.
+ fn assert_timestamp_widening_ordered(source_unit: TimeUnit, target_unit: TimeUnit) {
+ let source_type = DataType::Timestamp(source_unit, None);
+ let target_type = DataType::Timestamp(target_unit, None);
+ let quotient = i128::from(timestamp_unit_scale(&target_unit))
+ / i128::from(timestamp_unit_scale(&source_unit));
+ assert!(quotient > 1, "expected a widening unit pair");
+
+ // Aligned literals have a singleton preimage for every operator.
+ let aligned = timestamp_scalar(
+ &target_unit,
+ None,
+ Some(i64::try_from(123 * quotient).unwrap()),
+ );
+ for op in [
+ Operator::Eq,
+ Operator::NotEq,
+ Operator::IsDistinctFrom,
+ Operator::IsNotDistinctFrom,
+ Operator::GtEq,
+ Operator::Gt,
+ Operator::Lt,
+ Operator::LtEq,
+ ] {
+ assert_preimage_exact(
+ &source_type,
+ &target_type,
+ op,
+ &aligned,
+ timestamp_scalar(&source_unit, None, Some(123)),
+ );
+ }
+
+ // A non-aligned literal has no singleton preimage: ordered comparisons
+ // get the empty interval `[floor + 1, floor + 1)` and equality-like ones
+ // keep the cast. The bounds are checked real source-unit values, so the
+ // extreme literal is covered without any clamping.
+ for target_value in [
+ quotient + 1,
+ 1 - quotient,
+ i128::from(i64::MAX),
+ i128::from(i64::MIN),
+ ] {
+ let literal = timestamp_scalar(&target_unit, None, Some(target_value as i64));
+ let bound = i64::try_from(target_value.div_euclid(quotient) + 1).unwrap();
+ let empty = Some(CastPredicatePreimage::Range(
+ Interval::try_new(
+ timestamp_scalar(&source_unit, None, Some(bound)),
+ timestamp_scalar(&source_unit, None, Some(bound)),
+ )
+ .unwrap(),
+ ));
+
+ for op in [Operator::GtEq, Operator::Gt, Operator::Lt, Operator::LtEq] {
+ let result =
+ cast_predicate_preimage(&source_type, &target_type, op, &literal)
+ .unwrap();
+ assert_preimage_data_type(result.as_ref(), &source_type);
+ assert_eq!(
+ result, empty,
+ "unexpected preimage for {source_unit:?} → {target_unit:?} {op:?} {target_value}"
+ );
+ }
+ for op in [
+ Operator::Eq,
+ Operator::NotEq,
+ Operator::IsDistinctFrom,
+ Operator::IsNotDistinctFrom,
+ ] {
+ assert_preimage_none(&source_type, &target_type, op, &literal);
+ }
+ }
+ }
+
+ /// Asserts the truncation bucket `[lower, upper)` of a narrowing cast.
+ fn assert_timestamp_narrowing_range(
+ source_unit: TimeUnit,
+ target_unit: TimeUnit,
+ lit: i64,
+ lower: i64,
+ upper: i64,
+ ) {
+ let source_type = DataType::Timestamp(source_unit, None);
+ let target_type = DataType::Timestamp(target_unit, None);
+ let expected = Some(CastPredicatePreimage::Range(
+ Interval::try_new(
+ timestamp_scalar(&source_unit, None, Some(lower)),
+ timestamp_scalar(&source_unit, None, Some(upper)),
+ )
+ .unwrap(),
+ ));
+ let result = cast_predicate_preimage(
+ &source_type,
+ &target_type,
+ Operator::Eq,
+ &timestamp_scalar(&target_unit, None, Some(lit)),
+ )
+ .unwrap();
+ assert_preimage_data_type(result.as_ref(), &source_type);
+ assert_eq!(
+ result, expected,
+ "unexpected narrowing bucket for {source_unit:?} → {target_unit:?} {lit}"
+ );
+ }
}
diff --git a/datafusion/optimizer/src/simplify_expressions/cast_preimage.rs b/datafusion/optimizer/src/simplify_expressions/cast_preimage.rs
new file mode 100644
index 0000000000..1c00c35bf0
--- /dev/null
+++ b/datafusion/optimizer/src/simplify_expressions/cast_preimage.rs
@@ -0,0 +1,644 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Preimage rewrites for cast comparisons.
+//!
+//! This module computes source-domain predicates for expressions such as
+//! `CAST(expr AS target_type) OP literal`. For casts that are many-to-one, a
+//! same-operator unwrap is not equivalent; the correct rewrite is a preimage
+//! range over the input expression.
+
+use arrow::datatypes::DataType;
+use datafusion_common::{Result, internal_err, tree_node::Transformed};
+use datafusion_expr::expr::InList;
+use datafusion_expr::{
+ BinaryExpr, Cast, Expr, Operator, TryCast, lit, simplify::SimplifyContext,
+};
+use datafusion_expr_common::casts::{
+ CastPredicatePreimage, cast_predicate_preimage, exact_preimage_cast,
+};
+
+use super::udf_preimage::rewrite_with_preimage;
+
+pub(super) fn rewrite_cast_predicate_for_binary(
+ info: &SimplifyContext,
+ cast_expr: Expr,
+ literal: Expr,
+ op: Operator,
+) -> Result<Transformed<Expr>> {
+ let Some((expr, target_type)) = cast_input_and_type(cast_expr) else {
+ return internal_err!("Expect cast expr");
+ };
+ let Expr::Literal(lit_value, _) = literal else {
+ return internal_err!("Expect literal expr");
+ };
+
+ let source_type = info.get_data_type(&expr)?;
+ match cast_predicate_preimage(&source_type, &target_type, op, &lit_value)? {
+ Some(CastPredicatePreimage::Range(interval)) => {
+ rewrite_with_preimage(interval, op, *expr)
+ }
+ Some(CastPredicatePreimage::Exact(value)) => {
+ Ok(Transformed::yes(Expr::BinaryExpr(BinaryExpr {
+ left: expr,
+ op,
+ right: Box::new(lit(value)),
+ })))
+ }
+ None => internal_err!(
+ "Can't compute cast predicate preimage for source type {} target type {} literal {:?}",
+ source_type,
+ target_type,
+ lit_value
+ ),
+ }
+}
+
+pub(super) fn supports_cast_predicate_for_binary(
+ info: &SimplifyContext,
+ expr: &Expr,
+ op: Operator,
+ literal: &Expr,
+) -> bool {
+ if !matches!(
+ op,
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::Lt
+ | Operator::LtEq
+ | Operator::Gt
+ | Operator::GtEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
+ ) {
+ return false;
+ }
+
+ let Some((inner_expr, target_type)) = cast_input_and_type_ref(expr) else {
+ return false;
+ };
+ let Expr::Literal(lit_value, _) = literal else {
+ return false;
+ };
+ let Ok(source_type) = info.get_data_type(inner_expr) else {
+ return false;
+ };
+
+ let Ok(preimage) = cast_predicate_preimage(&source_type, target_type, op, lit_value)
+ else {
+ return false;
+ };
+ // Volatile expressions cannot be duplicated.
+ preimage.is_some_and(|p| !(p.duplicates_input(op) && inner_expr.is_volatile()))
+}
+
+pub(super) fn supports_cast_predicate_for_inlist(
+ info: &SimplifyContext,
+ expr: &Expr,
+ list: &[Expr],
+) -> bool {
+ let Some((inner_expr, target_type)) = cast_input_and_type_ref(expr) else {
+ return false;
+ };
+ let Ok(source_type) = info.get_data_type(inner_expr) else {
+ return false;
+ };
+
+ // IN-list rewrites only support singleton exact cast preimages. They do
+ // not use range preimages (for example timestamp precision narrowing) or
+ // binary-operator-only special cases such as integer-to-string equality.
+ list.iter().all(|right| match right {
+ Expr::Literal(lit_val, _) => {
+ exact_preimage_cast(&source_type, target_type, lit_val).is_some()
+ }
+ _ => false,
+ })
+}
+
+pub(super) fn rewrite_cast_predicate_for_inlist(
+ info: &SimplifyContext,
+ expr: Expr,
+ list: Vec<Expr>,
+ negated: bool,
+) -> Result<Transformed<Expr>> {
+ let Some((inner_expr, target_type)) = cast_input_and_type(expr) else {
+ return internal_err!("Expect cast expr");
+ };
+ let source_type = info.get_data_type(&inner_expr)?;
+
+ let list = list
+ .into_iter()
+ .map(|right| match right {
+ Expr::Literal(lit_value, _) => {
+ let Some(value) =
+ exact_preimage_cast(&source_type, &target_type, &lit_value)
+ else {
+ return internal_err!(
+ "Can't cast the list expr {:?} to type {}",
+ lit_value,
+ &source_type
+ );
+ };
+ Ok(lit(value))
+ }
+ other_expr => internal_err!(
+ "Only support literal expr to optimize, but the expr is {:?}",
+ &other_expr
+ ),
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(Transformed::yes(Expr::InList(InList {
+ expr: inner_expr,
+ list,
+ negated,
+ })))
+}
+
+fn cast_input_and_type(cast_expr: Expr) -> Option<(Box<Expr>, DataType)> {
+ match cast_expr {
+ Expr::TryCast(TryCast { expr, field }) | Expr::Cast(Cast { expr, field }) => {
+ Some((expr, field.data_type().clone()))
+ }
+ _ => None,
+ }
+}
+
+fn cast_input_and_type_ref(cast_expr: &Expr) -> Option<(&Expr, &DataType)> {
+ match cast_expr {
+ Expr::TryCast(TryCast { expr, field }) | Expr::Cast(Cast { expr, field }) => {
+ Some((expr.as_ref(), field.data_type()))
+ }
+ _ => None,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::collections::HashMap;
+ use std::sync::Arc;
+
+ use crate::simplify_expressions::ExprSimplifier;
+ use arrow::datatypes::{Field, TimeUnit};
+ use datafusion_common::{DFSchema, DFSchemaRef, ScalarValue};
+ use datafusion_expr::simplify::SimplifyContext;
+ use datafusion_expr::{binary_expr, cast, col, in_list, try_cast};
+
+ #[test]
+ fn test_exact_preimage_cast_unwrap() {
+ let schema = expr_test_schema();
+
+ let expr = cast(col("c1"), DataType::Int64).gt(lit(10_i64));
+ let expected = col("c1").gt(lit(10_i32));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ let expr = lit(10_i64).lt(cast(col("c1"), DataType::Int64));
+ let expected = col("c1").gt(lit(10_i32));
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+
+ #[test]
+ fn test_cast_predicate_string_integer_round_trip() {
+ let schema = expr_test_schema();
+
+ let expr = cast(col("c1"), DataType::Utf8).eq(lit("123"));
+ let expected = col("c1").eq(lit(123_i32));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ let expr = cast(col("c1"), DataType::Utf8).eq(lit("0123"));
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+ }
+
+ #[test]
+ fn test_cast_predicate_inlist_exact_unwrap() {
+ let schema = expr_test_schema();
+
+ let expr = in_list(
+ cast(col("c1"), DataType::Int64),
+ vec![lit(0_i64), lit(1_i64), lit(2_i64), lit(3_i64), lit(4_i64)],
+ false,
+ );
+ let expected = in_list(
+ col("c1"),
+ vec![lit(0_i32), lit(1_i32), lit(2_i32), lit(3_i32), lit(4_i32)],
+ false,
+ );
+
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+
+ #[test]
+ fn test_no_unwrap_date64_to_date32_narrowing() {
+ let schema = Arc::new(
+ DFSchema::from_unqualified_fields(
+ vec![Field::new("d64", DataType::Date64, false)].into(),
+ HashMap::new(),
+ )
+ .unwrap(),
+ );
+ let expr =
+ cast(col("d64"), DataType::Date32).eq(lit(ScalarValue::Date32(Some(20089))));
+
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_precision_narrowing_eq() {
+ let schema = expr_test_schema();
+
+ let expr =
+ cast(col("ts_nano"), timestamp_millis_type()).eq(lit_timestamp_millis(1000));
+ let expected = col("ts_nano")
+ .gt_eq(lit_timestamp_nano(1_000_000_000))
+ .and(col("ts_nano").lt(lit_timestamp_nano(1_001_000_000)));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ let expr =
+ cast(col("ts_nano"), timestamp_millis_type()).eq(lit_timestamp_millis(0));
+ let expected = col("ts_nano")
+ .gt_eq(lit_timestamp_nano(-999_999))
+ .and(col("ts_nano").lt(lit_timestamp_nano(1_000_000)));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ let expr =
+ cast(col("ts_nano"), timestamp_millis_type()).eq(lit_timestamp_millis(-1));
+ let expected = col("ts_nano")
+ .gt_eq(lit_timestamp_nano(-1_999_999))
+ .and(col("ts_nano").lt(lit_timestamp_nano(-999_999)));
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_precision_narrowing_inequality() {
+ let schema = expr_test_schema();
+
+ for (op, lit_ms, expected) in [
+ (
+ Operator::Gt,
+ 1000,
+ col("ts_nano").gt_eq(lit_timestamp_nano(1_001_000_000)),
+ ),
+ (
+ Operator::LtEq,
+ -1,
+ col("ts_nano").lt(lit_timestamp_nano(-999_999)),
+ ),
+ (
+ Operator::Lt,
+ 0,
+ col("ts_nano").lt(lit_timestamp_nano(-999_999)),
+ ),
+ (
+ Operator::LtEq,
+ 0,
+ col("ts_nano").lt(lit_timestamp_nano(1_000_000)),
+ ),
+ (
+ Operator::Gt,
+ 0,
+ col("ts_nano").gt_eq(lit_timestamp_nano(1_000_000)),
+ ),
+ (
+ Operator::GtEq,
+ 0,
+ col("ts_nano").gt_eq(lit_timestamp_nano(-999_999)),
+ ),
+ (
+ Operator::NotEq,
+ 0,
+ col("ts_nano")
+ .lt(lit_timestamp_nano(-999_999))
+ .or(col("ts_nano").gt_eq(lit_timestamp_nano(1_000_000))),
+ ),
+ ] {
+ let expr = binary_expr(
+ cast(col("ts_nano"), timestamp_millis_type()),
+ op,
+ lit_timestamp_millis(lit_ms),
+ );
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_precision_narrowing_distinctness() {
+ let schema = expr_test_schema();
+
+ for (op, expected) in [
+ (
+ Operator::IsNotDistinctFrom,
+ col("ts_nano")
+ .gt_eq(lit_timestamp_nano(-999_999))
+ .and(col("ts_nano").lt(lit_timestamp_nano(1_000_000))),
+ ),
+ (
+ Operator::IsDistinctFrom,
+ col("ts_nano")
+ .lt(lit_timestamp_nano(-999_999))
+ .or(col("ts_nano").gt_eq(lit_timestamp_nano(1_000_000))),
+ ),
+ ] {
+ let expr = binary_expr(
+ cast(col("ts_nano"), timestamp_millis_type()),
+ op,
+ lit_timestamp_millis(0),
+ );
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_widening_ordered() {
+ let schema = expr_test_schema();
+
+ let expr = cast(col("ts_milli"), timestamp_nano_type())
+ .eq(lit_timestamp_nano(123_000_000));
+ let expected = col("ts_milli").eq(lit_timestamp_millis(123));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ // A non-aligned widening literal maps from no source value, so only
+ // ordered comparisons are rewritten, and the bound lands on the first
+ // source value above the literal: `Gt`/`GtEq` become `GtEq` and
+ // `Lt`/`LtEq` become `Lt` against `floor + 1`.
+ for (literal, ceil) in [(123_456_789, 124), (-123_456_789, -123)] {
+ for (op, expected_op) in [
+ (Operator::GtEq, Operator::GtEq),
+ (Operator::Gt, Operator::GtEq),
+ (Operator::Lt, Operator::Lt),
+ (Operator::LtEq, Operator::Lt),
+ ] {
+ let expr = binary_expr(
+ cast(col("ts_milli"), timestamp_nano_type()),
+ op,
+ lit_timestamp_nano(literal),
+ );
+ let expected =
+ binary_expr(col("ts_milli"), expected_op, lit_timestamp_millis(ceil));
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+ }
+
+ let expr = cast(col("ts_milli"), timestamp_nano_type())
+ .gt_eq(lit_timestamp_nano(123_000_000));
+ let expected = col("ts_milli").gt_eq(lit_timestamp_millis(123));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ let expr = try_cast(col("ts_milli"), timestamp_nano_type())
+ .gt_eq(lit_timestamp_nano(123_456_789));
+ let expected = col("ts_milli").gt_eq(lit_timestamp_millis(124));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ let expr = cast(col("ts_milli"), timestamp_nano_type())
+ .eq(lit_timestamp_nano(123_456_789));
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+
+ let expr = in_list(
+ cast(col("ts_milli"), timestamp_nano_type()),
+ vec![
+ lit_timestamp_nano(123_456_789),
+ lit_timestamp_nano(987_654_321),
+ ],
+ false,
+ );
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_widening_literal_left_ordered() {
+ let schema = expr_test_schema();
+
+ for (op, expected_op) in [
+ (Operator::Lt, Operator::GtEq),
+ (Operator::LtEq, Operator::GtEq),
+ (Operator::Gt, Operator::Lt),
+ (Operator::GtEq, Operator::Lt),
+ ] {
+ let expr = binary_expr(
+ lit_timestamp_nano(123_456_789),
+ op,
+ cast(col("ts_milli"), timestamp_nano_type()),
+ );
+ let expected =
+ binary_expr(col("ts_milli"), expected_op, lit_timestamp_millis(124));
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+
+ let expr = binary_expr(
+ lit_timestamp_nano(-123_456_789),
+ Operator::GtEq,
+ try_cast(col("ts_milli"), timestamp_nano_type()),
+ );
+ let expected = col("ts_milli").lt(lit_timestamp_millis(-123));
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_dictionary_and_naive_aware() {
+ let dictionary = |data_type: DataType| {
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(data_type))
+ };
+ let dict_scalar = |value: ScalarValue| {
+ ScalarValue::Dictionary(Box::new(DataType::Int32), Box::new(value))
+ };
+ let source_tz: Option<Arc<str>> = Some("+05:30".into());
+ let schema = Arc::new(
+ DFSchema::from_unqualified_fields(
+ vec![
+ Field::new(
+ "ts_dict",
+ dictionary(DataType::Timestamp(
+ TimeUnit::Nanosecond,
+ source_tz.clone(),
+ )),
+ false,
+ ),
+ Field::new(
+ "ts",
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ false,
+ ),
+ ]
+ .into(),
+ HashMap::new(),
+ )
+ .unwrap(),
+ );
+
+ // Narrowing a dictionary-wrapped source keeps the source dictionary key,
+ // unit and timezone on both bounds. The literal is already in the target
+ // unit and timezone, as SQL coercion produces.
+ let expr = cast(
+ col("ts_dict"),
+ DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
+ )
+ .eq(lit(ScalarValue::TimestampMillisecond(
+ Some(1000),
+ Some("UTC".into()),
+ )));
+ let expected =
+ col("ts_dict")
+ .gt_eq(lit(dict_scalar(ScalarValue::TimestampNanosecond(
+ Some(1_000_000_000),
+ source_tz.clone(),
+ ))))
+ .and(col("ts_dict").lt(lit(dict_scalar(
+ ScalarValue::TimestampNanosecond(Some(1_001_000_000), source_tz),
+ ))));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ // Naive -> timezone-aware casts stay in place for exact, narrowing and
+ // IN-list shapes: Arrow reads the naive value as local wall-clock time in
+ // the target timezone and shifts it to UTC, which the unit arithmetic does
+ // not model.
+ let aware_type = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into()));
+ let aware_literal = ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into()));
+ let expr = cast(col("ts"), aware_type.clone()).eq(lit(aware_literal.clone()));
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+
+ let expr = cast(col("ts"), aware_type.clone()).gt(lit(aware_literal.clone()));
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+
+ let expr = in_list(
+ cast(col("ts"), aware_type),
+ vec![
+ lit(aware_literal.clone()),
+ lit(ScalarValue::TimestampNanosecond(
+ Some(1_000_000_000),
+ Some("UTC".into()),
+ )),
+ ],
+ false,
+ );
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+ }
+
+ #[test]
+ fn test_cast_preimage_timestamp_literal_left_range() {
+ let schema = expr_test_schema();
+
+ for (op, expected) in [
+ (
+ Operator::Lt,
+ col("ts_nano").gt_eq(lit_timestamp_nano(1_001_000_000)),
+ ),
+ (
+ Operator::LtEq,
+ col("ts_nano").gt_eq(lit_timestamp_nano(1_000_000_000)),
+ ),
+ (
+ Operator::Gt,
+ col("ts_nano").lt(lit_timestamp_nano(1_000_000_000)),
+ ),
+ (
+ Operator::Eq,
+ col("ts_nano")
+ .gt_eq(lit_timestamp_nano(1_000_000_000))
+ .and(col("ts_nano").lt(lit_timestamp_nano(1_001_000_000))),
+ ),
+ ] {
+ let expr = binary_expr(
+ lit_timestamp_millis(1000),
+ op,
+ cast(col("ts_nano"), timestamp_millis_type()),
+ );
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+ }
+
+ #[test]
+ fn test_cast_predicate_unsafe_narrowing_kept() {
+ let schema = expr_test_schema();
+
+ // CAST(c2 AS Int32) = 5 — should NOT be unwrapped because
+ // Int64→Int32 is narrowing (blocked by the family gate).
+ let expr = cast(col("c2"), DataType::Int32).eq(lit(5_i32));
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+
+ // CAST(c1 AS Int64) = 5 — should still be unwrapped
+ // (Int32→Int64 is widening, safe).
+ let expr = cast(col("c1"), DataType::Int64).eq(lit(5_i64));
+ let expected = col("c1").eq(lit(5_i32));
+ assert_eq!(optimize_test(expr, &schema), expected);
+ }
+
+ #[test]
+ fn test_cast_predicate_int_to_string_distinctness() {
+ let schema = expr_test_schema();
+
+ // CAST(c1 AS Utf8) IS NOT DISTINCT FROM '123'
+ let expr = binary_expr(
+ cast(col("c1"), DataType::Utf8),
+ Operator::IsNotDistinctFrom,
+ lit("123"),
+ );
+ let expected = binary_expr(col("c1"), Operator::IsNotDistinctFrom, lit(123_i32));
+ assert_eq!(optimize_test(expr, &schema), expected);
+
+ // CAST(c1 AS Utf8) IS DISTINCT FROM '0123'
+ // Round-trip fails (0123 → 123 → "123" ≠ "0123"), so rewrite
+ // should NOT happen.
+ let expr = binary_expr(
+ cast(col("c1"), DataType::Utf8),
+ Operator::IsDistinctFrom,
+ lit("0123"),
+ );
+ assert_eq!(optimize_test(expr.clone(), &schema), expr);
+ }
+
+ fn optimize_test(expr: Expr, schema: &DFSchemaRef) -> Expr {
+ let simplifier = ExprSimplifier::new(
+ SimplifyContext::builder()
+ .with_schema(Arc::clone(schema))
+ .build(),
+ );
+
+ simplifier.simplify(expr).unwrap()
+ }
+
+ fn expr_test_schema() -> DFSchemaRef {
+ Arc::new(
+ DFSchema::from_unqualified_fields(
+ vec![
+ Field::new("c1", DataType::Int32, false),
+ Field::new("c2", DataType::Int64, false),
+ Field::new("ts_milli", timestamp_millis_type(), false),
+ Field::new("ts_nano", timestamp_nano_type(), false),
+ ]
+ .into(),
+ HashMap::new(),
+ )
+ .unwrap(),
+ )
+ }
+
+ fn lit_timestamp_nano(ts: i64) -> Expr {
+ lit(ScalarValue::TimestampNanosecond(Some(ts), None))
+ }
+
+ fn lit_timestamp_millis(ts: i64) -> Expr {
+ lit(ScalarValue::TimestampMillisecond(Some(ts), None))
+ }
+
+ fn timestamp_nano_type() -> DataType {
+ DataType::Timestamp(TimeUnit::Nanosecond, None)
+ }
+
+ fn timestamp_millis_type() -> DataType {
+ DataType::Timestamp(TimeUnit::Millisecond, None)
+ }
+}
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 63929263d4..dfecaa92d3 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -59,18 +59,16 @@ use datafusion_physical_expr::{create_physical_expr, execution_props::ExecutionP
use super::inlist_simplifier::ShortenInListSimplifier;
use super::utils::*;
use crate::simplify_expressions::SimplifyContext;
-use crate::simplify_expressions::regex::simplify_regex_expr;
-use crate::simplify_expressions::unwrap_cast::{
- is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary,
- is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist,
- unwrap_cast_in_comparison_for_binary,
+use crate::simplify_expressions::cast_preimage::{
+ rewrite_cast_predicate_for_binary, rewrite_cast_predicate_for_inlist,
+ supports_cast_predicate_for_binary, supports_cast_predicate_for_inlist,
};
+use crate::simplify_expressions::regex::simplify_regex_expr;
use crate::{
analyzer::type_coercion::TypeCoercionRewriter,
simplify_expressions::udf_preimage::rewrite_with_preimage,
};
use datafusion_expr::expr_rewriter::rewrite_with_guarantees_map;
-use datafusion_expr_common::casts::try_cast_literal_to_type;
use indexmap::IndexSet;
use regex::{Error as RegexError, Regex, RegexBuilder};
@@ -1998,28 +1996,27 @@ impl TreeNodeRewriter for Simplifier<'_> {
}
// =======================================
- // unwrap_cast_in_comparison
+ // cast_predicate_in_comparison
// =======================================
//
// For case:
// try_cast/cast(expr as data_type) op literal
Expr::BinaryExpr(BinaryExpr { left, op, right })
- if is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary(
- info, &left, op, &right,
- ) && op.supports_propagation() =>
+ if supports_cast_predicate_for_binary(info, &left, op, &right)
+ && op.supports_propagation() =>
{
- unwrap_cast_in_comparison_for_binary(info, *left, *right, op)?
+ rewrite_cast_predicate_for_binary(info, *left, *right, op)?
}
// literal op try_cast/cast(expr as data_type)
// -->
// try_cast/cast(expr as data_type) op_swap literal
Expr::BinaryExpr(BinaryExpr { left, op, right })
- if is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary(
- info, &right, op, &left,
- ) && op.supports_propagation()
- && op.swap().is_some() =>
+ if op.supports_propagation()
+ && op.swap().is_some_and(|swapped| {
+ supports_cast_predicate_for_binary(info, &right, swapped, &left)
+ }) =>
{
- unwrap_cast_in_comparison_for_binary(
+ rewrite_cast_predicate_for_binary(
info,
*right,
*left,
@@ -2029,52 +2026,11 @@ impl TreeNodeRewriter for Simplifier<'_> {
// For case:
// try_cast/cast(expr as left_type) in (expr1,expr2,expr3)
Expr::InList(InList {
- expr: mut left,
+ expr,
list,
negated,
- }) if is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist(
- info, &left, &list,
- ) =>
- {
- let (Expr::TryCast(TryCast {
- expr: left_expr, ..
- })
- | Expr::Cast(Cast {
- expr: left_expr, ..
- })) = left.as_mut()
- else {
- return internal_err!("Expect cast expr, but got {:?}", left)?;
- };
-
- let expr_type = info.get_data_type(left_expr)?;
- let right_exprs = list
- .into_iter()
- .map(|right| {
- match right {
- Expr::Literal(right_lit_value, _) => {
- // if the right_lit_value can be casted to the type of internal_left_expr
- // we need to unwrap the cast for cast/try_cast expr, and add cast to the literal
- let Some(value) = try_cast_literal_to_type(&right_lit_value, &expr_type) else {
- internal_err!(
- "Can't cast the list expr {:?} to type {}",
- right_lit_value, &expr_type
- )?
- };
- Ok(lit(value))
- }
- other_expr => internal_err!(
- "Only support literal expr to optimize, but the expr is {:?}",
- &other_expr
- ),
- }
- })
- .collect::<Result<Vec<_>>>()?;
-
- Transformed::yes(Expr::InList(InList {
- expr: std::mem::take(left_expr),
- list: right_exprs,
- negated,
- }))
+ }) if supports_cast_predicate_for_inlist(info, &expr, &list) => {
+ rewrite_cast_predicate_for_inlist(info, *expr, list, negated)?
}
// =======================================
diff --git a/datafusion/optimizer/src/simplify_expressions/mod.rs b/datafusion/optimizer/src/simplify_expressions/mod.rs
index e0b53b79d4..3812f292e1 100644
--- a/datafusion/optimizer/src/simplify_expressions/mod.rs
+++ b/datafusion/optimizer/src/simplify_expressions/mod.rs
@@ -18,6 +18,7 @@
//! [`SimplifyExpressions`] simplifies expressions in the logical plan,
//! [`ExprSimplifier`] simplifies individual `Expr`s.
+mod cast_preimage;
pub mod expr_simplifier;
mod inlist_simplifier;
mod linear_aggregates;
@@ -27,7 +28,6 @@ pub mod simplify_exprs;
pub mod simplify_literal;
mod simplify_predicates;
mod udf_preimage;
-mod unwrap_cast;
mod utils;
// backwards compatibility
diff --git a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs
deleted file mode 100644
index d0165b568a..0000000000
--- a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs
+++ /dev/null
@@ -1,712 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Unwrap casts in binary comparisons
-//!
-//! The functions in this module attempt to remove casts from
-//! comparisons to literals ([`ScalarValue`]s) by applying the casts
-//! to the literals if possible. It is inspired by the optimizer rule
-//! `UnwrapCastInBinaryComparison` of Spark.
-//!
-//! Removing casts often improves performance because:
-//! 1. The cast is done once (to the literal) rather than to every value
-//! 2. Can enable other optimizations such as predicate pushdown that
-//! don't support casting
-//!
-//! The rule is applied to expressions of the following forms:
-//!
-//! 1. `cast(left_expr as data_type) comparison_op literal_expr`
-//! 2. `literal_expr comparison_op cast(left_expr as data_type)`
-//! 3. `cast(literal_expr) IN (expr1, expr2, ...)`
-//! 4. `literal_expr IN (cast(expr1) , cast(expr2), ...)`
-//!
-//! If the expression matches one of the forms above, the rule will
-//! ensure the value of `literal` is in range(min, max) of the
-//! expr's data_type, and if the scalar is within range, the literal
-//! will be casted to the data type of expr on the other side, and the
-//! cast will be removed from the other side.
-//!
-//! # Example
-//!
-//! If the DataType of c1 is INT32. Given the filter
-//!
-//! ```text
-//! cast(c1 as INT64) > INT64(10)`
-//! ```
-//!
-//! This rule will remove the cast and rewrite the expression to:
-//!
-//! ```text
-//! c1 > INT32(10)
-//! ```
-
-use arrow::datatypes::DataType;
-use datafusion_common::{Result, ScalarValue};
-use datafusion_common::{internal_err, tree_node::Transformed};
-use datafusion_expr::{BinaryExpr, lit};
-use datafusion_expr::{Cast, Expr, Operator, TryCast, simplify::SimplifyContext};
-use datafusion_expr_common::casts::{
- is_date_narrowing_cast, is_supported_type, is_timestamp_precision_narrowing_cast,
- try_cast_literal_to_type,
-};
-
-pub(super) fn unwrap_cast_in_comparison_for_binary(
- info: &SimplifyContext,
- cast_expr: Expr,
- literal: Expr,
- op: Operator,
-) -> Result<Transformed<Expr>> {
- match (cast_expr, literal) {
- (
- Expr::TryCast(TryCast { expr, .. }) | Expr::Cast(Cast { expr, .. }),
- Expr::Literal(lit_value, _),
- ) => {
- let Ok(expr_type) = info.get_data_type(&expr) else {
- return internal_err!("Can't get the data type of the expr {:?}", &expr);
- };
-
- if let Some(value) = cast_literal_to_type_with_op(&lit_value, &expr_type, op)
- {
- return Ok(Transformed::yes(Expr::BinaryExpr(BinaryExpr {
- left: expr,
- op,
- right: Box::new(lit(value)),
- })));
- }
-
- // if the lit_value can be casted to the type of internal_left_expr
- // we need to unwrap the cast for cast/try_cast expr, and add cast to the literal
- let Some(value) = try_cast_literal_to_type(&lit_value, &expr_type) else {
- return internal_err!(
- "Can't cast the literal expr {:?} to type {}",
- &lit_value,
- &expr_type
- );
- };
- Ok(Transformed::yes(Expr::BinaryExpr(BinaryExpr {
- left: expr,
- op,
- right: Box::new(lit(value)),
- })))
- }
- _ => internal_err!("Expect cast expr and literal"),
- }
-}
-
-pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary(
- info: &SimplifyContext,
- expr: &Expr,
- op: Operator,
- literal: &Expr,
-) -> bool {
- match (expr, literal) {
- (
- Expr::TryCast(TryCast {
- expr: left_expr,
- field,
- })
- | Expr::Cast(Cast {
- expr: left_expr,
- field,
- }),
- Expr::Literal(lit_val, _),
- ) => {
- let Ok(expr_type) = info.get_data_type(left_expr) else {
- return false;
- };
-
- let Ok(lit_type) = info.get_data_type(literal) else {
- return false;
- };
-
- if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type())
- || is_date_narrowing_cast(&expr_type, field.data_type())
- {
- return false;
- }
-
- if cast_literal_to_type_with_op(lit_val, &expr_type, op).is_some() {
- return true;
- }
-
- try_cast_literal_to_type(lit_val, &expr_type).is_some()
- && is_supported_type(&expr_type)
- && is_supported_type(&lit_type)
- }
- _ => false,
- }
-}
-
-pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist(
- info: &SimplifyContext,
- expr: &Expr,
- list: &[Expr],
-) -> bool {
- let (Expr::TryCast(TryCast {
- expr: left_expr,
- field,
- })
- | Expr::Cast(Cast {
- expr: left_expr,
- field,
- })) = expr
- else {
- return false;
- };
-
- let Ok(expr_type) = info.get_data_type(left_expr) else {
- return false;
- };
-
- if !is_supported_type(&expr_type) {
- return false;
- }
-
- if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type())
- || is_date_narrowing_cast(&expr_type, field.data_type())
- {
- return false;
- }
-
- for right in list {
- let Ok(right_type) = info.get_data_type(right) else {
- return false;
- };
-
- if !is_supported_type(&right_type) {
- return false;
- }
-
- match right {
- Expr::Literal(lit_val, _)
- if try_cast_literal_to_type(lit_val, &expr_type).is_some() => {}
- _ => return false,
- }
- }
-
- true
-}
-
-///// Tries to move a cast from an expression (such as column) to the literal other side of a comparison operator./
-///
-/// Specifically, rewrites
-/// ```sql
-/// cast(col) <op> <literal>
-/// ```
-///
-/// To
-///
-/// ```sql
-/// col <op> cast(<literal>)
-/// col <op> <casted_literal>
-/// ```
-fn cast_literal_to_type_with_op(
- lit_value: &ScalarValue,
- target_type: &DataType,
- op: Operator,
-) -> Option<ScalarValue> {
- match (op, lit_value) {
- (
- Operator::Eq | Operator::NotEq,
- ScalarValue::Utf8(Some(_))
- | ScalarValue::Utf8View(Some(_))
- | ScalarValue::LargeUtf8(Some(_)),
- ) => {
- // Only try for integer types (TODO can we do this for other types
- // like timestamps)?
- use DataType::*;
- if matches!(
- target_type,
- Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64
- ) {
- let casted = lit_value.cast_to(target_type).ok()?;
- let round_tripped = casted.cast_to(&lit_value.data_type()).ok()?;
- if lit_value != &round_tripped {
- return None;
- }
- Some(casted)
- } else {
- None
- }
- }
- _ => None,
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::collections::HashMap;
- use std::sync::Arc;
-
- use crate::simplify_expressions::ExprSimplifier;
- use arrow::datatypes::{Field, TimeUnit};
- use datafusion_common::{DFSchema, DFSchemaRef};
- use datafusion_expr::simplify::SimplifyContext;
- use datafusion_expr::{cast, col, in_list, try_cast};
-
- #[test]
- fn test_not_unwrap_cast_comparison() {
- let schema = expr_test_schema();
- // cast(INT32(c1), INT64) > INT64(c2)
- let c1_gt_c2 = cast(col("c1"), DataType::Int64).gt(col("c2"));
- assert_eq!(optimize_test(c1_gt_c2.clone(), &schema), c1_gt_c2);
-
- // INT32(c1) < INT32(16), the type is same
- let expr_lt = col("c1").lt(lit(16i32));
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // the 99999999999 is not within the range of MAX(int32) and MIN(int32), we don't cast the lit(99999999999) to int32 type
- let expr_lt = cast(col("c1"), DataType::Int64).lt(lit(99999999999i64));
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // cast(c1, UTF8) < '123', only eq/not_eq should be optimized
- let expr_lt = cast(col("c1"), DataType::Utf8).lt(lit("123"));
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // cast(c1, UTF8) = '0123', cast(cast('0123', Int32), UTF8) != '0123', so '0123' should not
- // be casted
- let expr_lt = cast(col("c1"), DataType::Utf8).lt(lit("0123"));
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // cast(c1, UTF8) = 'not a number', should not be able to cast to column type
- let expr_input = cast(col("c1"), DataType::Utf8).eq(lit("not a number"));
- assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
-
- // cast(c1, UTF8) = '99999999999', where '99999999999' does not fit into int32, so it will
- // not be optimized to integer comparison
- let expr_input = cast(col("c1"), DataType::Utf8).eq(lit("99999999999"));
- assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
- }
-
- #[test]
- fn test_unwrap_cast_comparison() {
- let schema = expr_test_schema();
- // cast(c1, INT64) < INT64(16) -> INT32(c1) < cast(INT32(16))
- // the 16 is within the range of MAX(int32) and MIN(int32), we can cast the 16 to int32(16)
- let expr_lt = cast(col("c1"), DataType::Int64).lt(lit(16i64));
- let expected = col("c1").lt(lit(16i32));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- let expr_lt = try_cast(col("c1"), DataType::Int64).lt(lit(16i64));
- let expected = col("c1").lt(lit(16i32));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
-
- // cast(c2, INT32) = INT32(16) => INT64(c2) = INT64(16)
- let c2_eq_lit = cast(col("c2"), DataType::Int32).eq(lit(16i32));
- let expected = col("c2").eq(lit(16i64));
- assert_eq!(optimize_test(c2_eq_lit, &schema), expected);
-
- // cast(c1, INT64) < INT64(NULL) => NULL
- let c1_lt_lit_null = cast(col("c1"), DataType::Int64).lt(null_i64());
- let expected = null_bool();
- assert_eq!(optimize_test(c1_lt_lit_null, &schema), expected);
-
- // cast(INT8(NULL), INT32) < INT32(12) => INT8(NULL) < INT8(12) => BOOL(NULL)
- let lit_lt_lit = cast(null_i8(), DataType::Int32).lt(lit(12i32));
- let expected = null_bool();
- assert_eq!(optimize_test(lit_lt_lit, &schema), expected);
-
- // cast(c1, UTF8) = '123' => c1 = 123
- let expr_input = cast(col("c1"), DataType::Utf8).eq(lit("123"));
- let expected = col("c1").eq(lit(123i32));
- assert_eq!(optimize_test(expr_input, &schema), expected);
-
- // cast(c1, UTF8) != '123' => c1 != 123
- let expr_input = cast(col("c1"), DataType::Utf8).not_eq(lit("123"));
- let expected = col("c1").not_eq(lit(123i32));
- assert_eq!(optimize_test(expr_input, &schema), expected);
-
- // cast(c1, UTF8) = NULL => NULL
- let expr_input = cast(col("c1"), DataType::Utf8).eq(lit(ScalarValue::Utf8(None)));
- let expected = null_bool();
- assert_eq!(optimize_test(expr_input, &schema), expected);
- }
-
- #[test]
- fn test_unwrap_cast_comparison_unsigned() {
- // "cast(c6, UINT64) = 0u64 => c6 = 0u32
- let schema = expr_test_schema();
- let expr_input = cast(col("c6"), DataType::UInt64).eq(lit(0u64));
- let expected = col("c6").eq(lit(0u32));
- assert_eq!(optimize_test(expr_input, &schema), expected);
-
- // cast(c6, UTF8) = "123" => c6 = 123
- let expr_input = cast(col("c6"), DataType::Utf8).eq(lit("123"));
- let expected = col("c6").eq(lit(123u32));
- assert_eq!(optimize_test(expr_input, &schema), expected);
-
- // cast(c6, UTF8) != "123" => c6 != 123
- let expr_input = cast(col("c6"), DataType::Utf8).not_eq(lit("123"));
- let expected = col("c6").not_eq(lit(123u32));
- assert_eq!(optimize_test(expr_input, &schema), expected);
- }
-
- #[test]
- fn test_unwrap_cast_comparison_string() {
- let schema = expr_test_schema();
- let dict = ScalarValue::Dictionary(
- Box::new(DataType::Int32),
- Box::new(ScalarValue::from("value")),
- );
-
- // cast(str1 as Dictionary<Int32, Utf8>) = arrow_cast('value', 'Dictionary<Int32, Utf8>') => str1 = Utf8('value1')
- let expr_input = cast(col("str1"), dict.data_type()).eq(lit(dict.clone()));
- let expected = col("str1").eq(lit("value"));
- assert_eq!(optimize_test(expr_input, &schema), expected);
-
- // cast(tag as Utf8) = Utf8('value') => tag = arrow_cast('value', 'Dictionary<Int32, Utf8>')
- let expr_input = cast(col("tag"), DataType::Utf8).eq(lit("value"));
- let expected = col("tag").eq(lit(dict.clone()));
- assert_eq!(optimize_test(expr_input, &schema), expected);
-
- // Verify reversed argument order
- // arrow_cast('value', 'Dictionary<Int32, Utf8>') = cast(str1 as Dictionary<Int32, Utf8>) => Utf8('value1') = str1
- let expr_input = lit(dict.clone()).eq(cast(col("str1"), dict.data_type()));
- let expected = col("str1").eq(lit("value"));
- assert_eq!(optimize_test(expr_input, &schema), expected);
- }
-
- #[test]
- fn test_unwrap_cast_comparison_large_string() {
- let schema = expr_test_schema();
- // cast(largestr as Dictionary<Int32, LargeUtf8>) = arrow_cast('value', 'Dictionary<Int32, LargeUtf8>') => str1 = LargeUtf8('value1')
- let dict = ScalarValue::Dictionary(
- Box::new(DataType::Int32),
- Box::new(ScalarValue::LargeUtf8(Some("value".to_owned()))),
- );
- let expr_input = cast(col("largestr"), dict.data_type()).eq(lit(dict));
- let expected =
- col("largestr").eq(lit(ScalarValue::LargeUtf8(Some("value".to_owned()))));
- assert_eq!(optimize_test(expr_input, &schema), expected);
- }
-
- #[test]
- fn test_not_unwrap_cast_with_decimal_comparison() {
- let schema = expr_test_schema();
- // integer to decimal: value is out of the bounds of the decimal
- // cast(c3, INT64) = INT64(100000000000000000)
- let expr_eq = cast(col("c3"), DataType::Int64).eq(lit(100000000000000000i64));
- assert_eq!(optimize_test(expr_eq.clone(), &schema), expr_eq);
-
- // cast(c4, INT64) = INT64(1000) will overflow the i128
- let expr_eq = cast(col("c4"), DataType::Int64).eq(lit(1000i64));
- assert_eq!(optimize_test(expr_eq.clone(), &schema), expr_eq);
-
- // decimal to decimal: value will lose the scale when convert to the target data type
- // c3 = DECIMAL(12340,20,4)
- let expr_eq =
- cast(col("c3"), DataType::Decimal128(20, 4)).eq(lit_decimal(12340, 20, 4));
- assert_eq!(optimize_test(expr_eq.clone(), &schema), expr_eq);
-
- // decimal to integer
- // c1 = DECIMAL(123, 10, 1): value will lose the scale when convert to the target data type
- let expr_eq =
- cast(col("c1"), DataType::Decimal128(10, 1)).eq(lit_decimal(123, 10, 1));
- assert_eq!(optimize_test(expr_eq.clone(), &schema), expr_eq);
-
- // c1 = DECIMAL(1230, 10, 2): value will lose the scale when convert to the target data type
- let expr_eq =
- cast(col("c1"), DataType::Decimal128(10, 2)).eq(lit_decimal(1230, 10, 2));
- assert_eq!(optimize_test(expr_eq.clone(), &schema), expr_eq);
- }
-
- #[test]
- fn test_unwrap_cast_with_decimal_lit_comparison() {
- let schema = expr_test_schema();
- // integer to decimal
- // c3 < INT64(16) -> c3 < (CAST(INT64(16) AS DECIMAL(18,2));
- let expr_lt = try_cast(col("c3"), DataType::Int64).lt(lit(16i64));
- let expected = col("c3").lt(lit_decimal(1600, 18, 2));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
-
- // c3 < INT64(NULL)
- let c1_lt_lit_null = cast(col("c3"), DataType::Int64).lt(null_i64());
- let expected = null_bool();
- assert_eq!(optimize_test(c1_lt_lit_null, &schema), expected);
-
- // decimal to decimal
- // c3 < Decimal(123,10,0) -> c3 < CAST(DECIMAL(123,10,0) AS DECIMAL(18,2)) -> c3 < DECIMAL(12300,18,2)
- let expr_lt =
- cast(col("c3"), DataType::Decimal128(10, 0)).lt(lit_decimal(123, 10, 0));
- let expected = col("c3").lt(lit_decimal(12300, 18, 2));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
-
- // c3 < Decimal(1230,10,3) -> c3 < CAST(DECIMAL(1230,10,3) AS DECIMAL(18,2)) -> c3 < DECIMAL(123,18,2)
- let expr_lt =
- cast(col("c3"), DataType::Decimal128(10, 3)).lt(lit_decimal(1230, 10, 3));
- let expected = col("c3").lt(lit_decimal(123, 18, 2));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
-
- // decimal to integer
- // c1 < Decimal(12300, 10, 2) -> c1 < CAST(DECIMAL(12300,10,2) AS INT32) -> c1 < INT32(123)
- let expr_lt =
- cast(col("c1"), DataType::Decimal128(10, 2)).lt(lit_decimal(12300, 10, 2));
- let expected = col("c1").lt(lit(123i32));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- }
-
- #[test]
- fn test_not_unwrap_list_cast_lit_comparison() {
- let schema = expr_test_schema();
- // internal left type is not supported
- // FLOAT32(C5) in ...
- let expr_lt =
- cast(col("c5"), DataType::Int64).in_list(vec![lit(12i64), lit(12i64)], false);
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // cast(INT32(C1), Float32) in (FLOAT32(1.23), Float32(12), Float32(12))
- let expr_lt = cast(col("c1"), DataType::Float32)
- .in_list(vec![lit(12.0f32), lit(12.0f32), lit(1.23f32)], false);
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // INT32(C1) in (INT64(99999999999), INT64(12))
- let expr_lt = cast(col("c1"), DataType::Int64)
- .in_list(vec![lit(12i32), lit(99999999999i64)], false);
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
-
- // DECIMAL(C3) in (INT64(12), INT32(12), DECIMAL(128,12,3))
- let expr_lt = cast(col("c3"), DataType::Decimal128(12, 3)).in_list(
- vec![
- lit_decimal(12, 12, 3),
- lit_decimal(12, 12, 3),
- lit_decimal(128, 12, 3),
- ],
- false,
- );
- assert_eq!(optimize_test(expr_lt.clone(), &schema), expr_lt);
- }
-
- #[test]
- fn test_unwrap_list_cast_comparison() {
- let schema = expr_test_schema();
- // INT32(C1) IN (INT32(12),INT64(23),INT64(34),INT64(56),INT64(78)) ->
- // INT32(C1) IN (INT32(12),INT32(23),INT32(34),INT32(56),INT32(78))
- let expr_lt = cast(col("c1"), DataType::Int64).in_list(
- vec![lit(12i64), lit(23i64), lit(34i64), lit(56i64), lit(78i64)],
- false,
- );
- let expected = col("c1").in_list(
- vec![lit(12i32), lit(23i32), lit(34i32), lit(56i32), lit(78i32)],
- false,
- );
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- // INT32(C2) IN (INT64(NULL),INT64(24),INT64(34),INT64(56),INT64(78)) ->
- // INT32(C2) IN (INT32(NULL),INT32(24),INT32(34),INT32(56),INT32(78))
- let expr_lt = cast(col("c2"), DataType::Int32).in_list(
- vec![null_i32(), lit(24i32), lit(34i64), lit(56i64), lit(78i64)],
- false,
- );
- let expected = col("c2").in_list(
- vec![null_i64(), lit(24i64), lit(34i64), lit(56i64), lit(78i64)],
- false,
- );
-
- assert_eq!(optimize_test(expr_lt, &schema), expected);
-
- // decimal test case
- // c3 is decimal(18,2)
- let expr_lt = cast(col("c3"), DataType::Decimal128(19, 3)).in_list(
- vec![
- lit_decimal(12000, 19, 3),
- lit_decimal(24000, 19, 3),
- lit_decimal(1280, 19, 3),
- lit_decimal(1240, 19, 3),
- ],
- false,
- );
- let expected = col("c3").in_list(
- vec![
- lit_decimal(1200, 18, 2),
- lit_decimal(2400, 18, 2),
- lit_decimal(128, 18, 2),
- lit_decimal(124, 18, 2),
- ],
- false,
- );
- assert_eq!(optimize_test(expr_lt, &schema), expected);
-
- // cast(INT32(12), INT64) IN (.....) =>
- // INT64(12) IN (INT64(12),INT64(13),INT64(14),INT64(15),INT64(16))
- // => true
- let expr_lt = cast(lit(12i32), DataType::Int64).in_list(
- vec![lit(12i64), lit(13i64), lit(14i64), lit(15i64), lit(16i64)],
- false,
- );
- let expected = lit(true);
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- }
-
- #[test]
- fn aliased() {
- let schema = expr_test_schema();
- // c1 < INT64(16) -> c1 < cast(INT32(16))
- // the 16 is within the range of MAX(int32) and MIN(int32), we can cast the 16 to int32(16)
- let expr_lt = cast(col("c1"), DataType::Int64).lt(lit(16i64)).alias("x");
- let expected = col("c1").lt(lit(16i32)).alias("x");
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- }
-
- #[test]
- fn nested() {
- let schema = expr_test_schema();
- // c1 < INT64(16) OR c1 > INT64(32) -> c1 < INT32(16) OR c1 > INT32(32)
- // the 16 and 32 are within the range of MAX(int32) and MIN(int32), we can cast them to int32
- let expr_lt = cast(col("c1"), DataType::Int64).lt(lit(16i64)).or(cast(
- col("c1"),
- DataType::Int64,
- )
- .gt(lit(32i64)));
- let expected = col("c1").lt(lit(16i32)).or(col("c1").gt(lit(32i32)));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- }
-
- #[test]
- fn test_not_support_data_type() {
- // "c6 > 0" will be cast to `cast(c6 as float) > 0
- // but the type of c6 is uint32
- // the rewriter will not throw error and just return the original expr
- let schema = expr_test_schema();
- let expr_input = cast(col("c6"), DataType::Float64).eq(lit(0f64));
- assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
-
- // inlist for unsupported data type
- let expr_input = in_list(
- cast(col("c6"), DataType::Float64),
- // need more literals to avoid rewriting to binary expr
- vec![lit(0f64), lit(1f64), lit(2f64), lit(3f64), lit(4f64)],
- false,
- );
- assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
- }
-
- #[test]
- /// Basic integration test for unwrapping casts with different timezones
- fn test_unwrap_cast_with_timestamp_nanos() {
- let schema = expr_test_schema();
- // cast(ts_nano as Timestamp(Nanosecond, UTC)) < 1666612093000000000::Timestamp(Nanosecond, Utc))
- let expr_lt = try_cast(col("ts_nano_none"), timestamp_nano_utc_type())
- .lt(lit_timestamp_nano_utc(1666612093000000000));
- let expected =
- col("ts_nano_none").lt(lit_timestamp_nano_none(1666612093000000000));
- assert_eq!(optimize_test(expr_lt, &schema), expected);
- }
-
- #[test]
- fn test_not_unwrap_cast_timestamp_precision_narrowing() {
- let schema = expr_test_schema();
- let expr_input = cast(col("ts_nano_none"), timestamp_millis_none_type())
- .eq(lit_timestamp_millis_none(1));
-
- assert_eq!(optimize_test(expr_input.clone(), &schema), expr_input);
- }
-
- #[test]
- fn test_unwrap_cast_timestamp_precision_widening() {
- let schema = expr_test_schema();
- let expr_input = cast(col("ts_millis_none"), timestamp_nano_none_type())
- .eq(lit_timestamp_nano_none(1_000_000));
- let expected = col("ts_millis_none").eq(lit_timestamp_millis_none(1));
-
- assert_eq!(optimize_test(expr_input, &schema), expected);
- }
-
- fn optimize_test(expr: Expr, schema: &DFSchemaRef) -> Expr {
- let simplifier = ExprSimplifier::new(
- SimplifyContext::builder()
- .with_schema(Arc::clone(schema))
- .build(),
- );
-
- simplifier.simplify(expr).unwrap()
- }
-
- fn expr_test_schema() -> DFSchemaRef {
- Arc::new(
- DFSchema::from_unqualified_fields(
- vec![
- Field::new("c1", DataType::Int32, false),
- Field::new("c2", DataType::Int64, false),
- Field::new("c3", DataType::Decimal128(18, 2), false),
- Field::new("c4", DataType::Decimal128(38, 37), false),
- Field::new("c5", DataType::Float32, false),
- Field::new("c6", DataType::UInt32, false),
- Field::new("ts_nano_none", timestamp_nano_none_type(), false),
- Field::new("ts_millis_none", timestamp_millis_none_type(), false),
- Field::new("ts_nano_utf", timestamp_nano_utc_type(), false),
- Field::new("str1", DataType::Utf8, false),
- Field::new("largestr", DataType::LargeUtf8, false),
- Field::new("tag", dictionary_tag_type(), false),
- ]
- .into(),
- HashMap::new(),
- )
- .unwrap(),
- )
- }
-
- fn null_bool() -> Expr {
- lit(ScalarValue::Boolean(None))
- }
-
- fn null_i8() -> Expr {
- lit(ScalarValue::Int8(None))
- }
-
- fn null_i32() -> Expr {
- lit(ScalarValue::Int32(None))
- }
-
- fn null_i64() -> Expr {
- lit(ScalarValue::Int64(None))
- }
-
- fn lit_decimal(value: i128, precision: u8, scale: i8) -> Expr {
- lit(ScalarValue::Decimal128(Some(value), precision, scale))
- }
-
- fn lit_timestamp_nano_none(ts: i64) -> Expr {
- lit(ScalarValue::TimestampNanosecond(Some(ts), None))
- }
-
- fn lit_timestamp_millis_none(ts: i64) -> Expr {
- lit(ScalarValue::TimestampMillisecond(Some(ts), None))
- }
-
- fn lit_timestamp_nano_utc(ts: i64) -> Expr {
- let utc = Some("+0:00".into());
- lit(ScalarValue::TimestampNanosecond(Some(ts), utc))
- }
-
- fn timestamp_nano_none_type() -> DataType {
- DataType::Timestamp(TimeUnit::Nanosecond, None)
- }
-
- fn timestamp_millis_none_type() -> DataType {
- DataType::Timestamp(TimeUnit::Millisecond, None)
- }
-
- // this is the type that now() returns
- fn timestamp_nano_utc_type() -> DataType {
- let utc = Some("+0:00".into());
- DataType::Timestamp(TimeUnit::Nanosecond, utc)
- }
-
- // a dictionary type for storing string tags
- fn dictionary_tag_type() -> DataType {
- DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8))
- }
-}
diff --git a/datafusion/physical-expr/src/simplifier/mod.rs b/datafusion/physical-expr/src/simplifier/mod.rs
index af87ce7f61..dd935effc9 100644
--- a/datafusion/physical-expr/src/simplifier/mod.rs
+++ b/datafusion/physical-expr/src/simplifier/mod.rs
@@ -154,6 +154,8 @@ mod tests {
let simplifier = PhysicalExprSimplifier::new(&schema);
// Create: cast(c2 as INT32) != INT32(99)
+ // c2 is Int64 → Int64→Int32 is narrowing → blocked by family gate.
+ // The cast should stay.
let column_expr = col("c2", &schema).unwrap();
let cast_expr = Arc::new(CastExpr::new(column_expr, DataType::Int32, None));
let literal_expr = lit(ScalarValue::Int32(Some(99)));
@@ -163,16 +165,14 @@ mod tests {
// Apply full simplification (uses TreeNodeRewriter)
let optimized = simplifier.simplify(binary_expr).unwrap();
- let optimized_binary = as_binary(&optimized);
-
- // Should be optimized to: c2 != INT64(99) (c2 is INT64, literal cast to match)
- let left_expr = optimized_binary.left();
+ // With the family gate, Int64→Int32 is blocked and the cast remains.
+ let result_binary = as_binary(&optimized);
+ let result_left = result_binary.left();
assert!(
- left_expr.downcast_ref::<CastExpr>().is_none()
- && left_expr.downcast_ref::<TryCastExpr>().is_none()
+ result_left.downcast_ref::<CastExpr>().is_some()
+ || result_left.downcast_ref::<TryCastExpr>().is_some(),
+ "CAST(c2 AS Int32) should remain as Int64→Int32 is narrowing"
);
- let right_literal = as_literal(optimized_binary.right());
- assert_eq!(right_literal.value(), &ScalarValue::Int64(Some(99)));
}
#[test]
@@ -198,7 +198,7 @@ mod tests {
let or_binary = as_binary(&optimized);
- // Verify left side: c1 > INT32(5)
+ // Verify left side: c1 > INT32(5) (Int32→Int64 widening, unwrapped)
let left_binary = as_binary(or_binary.left());
let left_left_expr = left_binary.left();
assert!(
@@ -208,15 +208,15 @@ mod tests {
let left_literal = as_literal(left_binary.right());
assert_eq!(left_literal.value(), &ScalarValue::Int32(Some(5)));
- // Verify right side: c2 <= INT64(10)
+ // Verify right side: CAST(c2 AS Int32) <= 10 — Int64→Int32 is
+ // narrowing, blocked by the family gate, so the cast stays.
let right_binary = as_binary(or_binary.right());
let right_left_expr = right_binary.left();
assert!(
- right_left_expr.downcast_ref::<CastExpr>().is_none()
- && right_left_expr.downcast_ref::<TryCastExpr>().is_none()
+ right_left_expr.downcast_ref::<CastExpr>().is_some()
+ || right_left_expr.downcast_ref::<TryCastExpr>().is_some(),
+ "CAST(c2 AS Int32) should remain as Int64→Int32 is narrowing"
);
- let right_literal = as_literal(right_binary.right());
- assert_eq!(right_literal.value(), &ScalarValue::Int64(Some(10)));
}
#[test]
diff --git a/datafusion/physical-expr/src/simplifier/unwrap_cast.rs b/datafusion/physical-expr/src/simplifier/unwrap_cast.rs
index 3e67fc8291..d3c0884864 100644
--- a/datafusion/physical-expr/src/simplifier/unwrap_cast.rs
+++ b/datafusion/physical-expr/src/simplifier/unwrap_cast.rs
@@ -34,15 +34,15 @@
use std::sync::Arc;
use arrow::datatypes::{DataType, Schema};
-use datafusion_common::{Result, ScalarValue, tree_node::Transformed};
+use datafusion_common::{Result, ScalarValue, internal_err, tree_node::Transformed};
use datafusion_expr::Operator;
-use datafusion_expr_common::casts::{
- is_date_narrowing_cast, is_timestamp_precision_narrowing_cast,
- try_cast_literal_to_type,
-};
+use datafusion_expr_common::casts::{CastPredicatePreimage, cast_predicate_preimage};
use crate::PhysicalExpr;
-use crate::expressions::{BinaryExpr, CastExpr, Literal, TryCastExpr, lit};
+use crate::expressions::{
+ BinaryExpr, CastExpr, Literal, TryCastExpr, is_not_null, is_null, lit,
+};
+use datafusion_physical_expr_common::physical_expr::is_volatile;
/// Attempts to unwrap casts in comparison expressions.
pub(crate) fn unwrap_cast_in_comparison(
@@ -69,8 +69,8 @@ fn try_unwrap_cast_binary(
) && binary.op().supports_propagation()
&& let Some(unwrapped) = try_unwrap_cast_comparison(
Arc::clone(inner_expr),
- literal.value(),
cast_type,
+ literal.value(),
*binary.op(),
schema,
)?
@@ -88,8 +88,8 @@ fn try_unwrap_cast_binary(
&& binary.op().supports_propagation()
&& let Some(unwrapped) = try_unwrap_cast_comparison(
Arc::clone(inner_expr),
- literal.value(),
cast_type,
+ literal.value(),
swapped_op,
schema,
)?
@@ -122,28 +122,92 @@ fn extract_cast_info(
/// Try to unwrap a cast in comparison by moving the cast to the literal
fn try_unwrap_cast_comparison(
inner_expr: Arc<dyn PhysicalExpr>,
- literal_value: &ScalarValue,
cast_type: &DataType,
+ literal_value: &ScalarValue,
op: Operator,
schema: &Schema,
) -> Result<Option<Arc<dyn PhysicalExpr>>> {
// Get the data type of the inner expression
let inner_type = inner_expr.data_type(schema)?;
- if is_timestamp_precision_narrowing_cast(&inner_type, cast_type)
- || is_date_narrowing_cast(&inner_type, cast_type)
- {
- return Ok(None);
+ match cast_predicate_preimage(&inner_type, cast_type, op, literal_value)? {
+ Some(CastPredicatePreimage::Exact(casted_literal)) => {
+ let literal_expr = lit(casted_literal);
+ let binary_expr = BinaryExpr::new(inner_expr, op, literal_expr);
+ Ok(Some(Arc::new(binary_expr)))
+ }
+ Some(CastPredicatePreimage::Range(interval)) => {
+ // Equality-like range predicates duplicate their input; do not duplicate
+ // volatile expressions.
+ if matches!(
+ op,
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
+ ) && is_volatile(&inner_expr)
+ {
+ return Ok(None);
+ }
+ rewrite_with_preimage(interval, op, inner_expr).map(Some)
+ }
+ None => Ok(None),
}
+}
- // Try to cast the literal to the inner expression's type
- if let Some(casted_literal) = try_cast_literal_to_type(literal_value, &inner_type) {
- let literal_expr = lit(casted_literal);
- let binary_expr = BinaryExpr::new(inner_expr, op, literal_expr);
- return Ok(Some(Arc::new(binary_expr)));
- }
+fn rewrite_with_preimage(
+ interval: datafusion_expr_common::interval_arithmetic::Interval,
+ op: Operator,
+ expr: Arc<dyn PhysicalExpr>,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let (lower, upper) = interval.into_bounds();
+ let (lower, upper) = (lit(lower), lit(upper));
+
+ let rewritten_expr = match op {
+ Operator::Lt => binary(Arc::clone(&expr), Operator::Lt, lower),
+ Operator::GtEq => binary(Arc::clone(&expr), Operator::GtEq, lower),
+ Operator::Gt => binary(Arc::clone(&expr), Operator::GtEq, upper),
+ Operator::LtEq => binary(Arc::clone(&expr), Operator::Lt, upper),
+ Operator::Eq => binary(
+ binary(Arc::clone(&expr), Operator::GtEq, lower),
+ Operator::And,
+ binary(expr, Operator::Lt, upper),
+ ),
+ Operator::NotEq => binary(
+ binary(Arc::clone(&expr), Operator::Lt, lower),
+ Operator::Or,
+ binary(expr, Operator::GtEq, upper),
+ ),
+ Operator::IsNotDistinctFrom => binary(
+ binary(
+ is_not_null(Arc::clone(&expr))?,
+ Operator::And,
+ binary(Arc::clone(&expr), Operator::GtEq, lower),
+ ),
+ Operator::And,
+ binary(expr, Operator::Lt, upper),
+ ),
+ Operator::IsDistinctFrom => binary(
+ binary(
+ binary(Arc::clone(&expr), Operator::Lt, lower),
+ Operator::Or,
+ binary(Arc::clone(&expr), Operator::GtEq, upper),
+ ),
+ Operator::Or,
+ is_null(expr)?,
+ ),
+ _ => return internal_err!("Expect comparison operators, got {op}"),
+ };
+
+ Ok(rewritten_expr)
+}
- Ok(None)
+fn binary(
+ left: Arc<dyn PhysicalExpr>,
+ op: Operator,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ Arc::new(BinaryExpr::new(left, op, right))
}
#[cfg(test)]
@@ -180,6 +244,47 @@ mod tests {
])
}
+ fn timestamp_schema(unit: TimeUnit) -> Schema {
+ Schema::new(vec![Field::new(
+ "ts",
+ DataType::Timestamp(unit, None),
+ false,
+ )])
+ }
+
+ fn timestamp_cast_comparison(
+ schema: &Schema,
+ target_unit: TimeUnit,
+ op: Operator,
+ literal: ScalarValue,
+ ) -> Arc<dyn PhysicalExpr> {
+ let column_expr = col("ts", schema).unwrap();
+ let cast_expr = Arc::new(CastExpr::new(
+ column_expr,
+ DataType::Timestamp(target_unit, None),
+ None,
+ ));
+ Arc::new(BinaryExpr::new(cast_expr, op, lit(literal)))
+ }
+
+ fn assert_timestamp_widening_rewrite(
+ schema: &Schema,
+ binary_expr: Arc<dyn PhysicalExpr>,
+ expected_op: Operator,
+ expected_value: i64,
+ ) {
+ let result = unwrap_cast_in_comparison(binary_expr, schema).unwrap();
+ assert!(result.transformed);
+ let optimized_binary = result.data.downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*optimized_binary.op(), expected_op);
+ assert!(!is_cast_expr(optimized_binary.left()));
+ let right_literal = optimized_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ right_literal.value(),
+ &ScalarValue::TimestampMillisecond(Some(expected_value), None)
+ );
+ }
+
#[test]
fn test_unwrap_cast_in_binary_comparison() {
let schema = test_schema();
@@ -251,6 +356,173 @@ mod tests {
assert!(!result.transformed);
}
+ /// Unit-specific timestamp literal, e.g. `0` in milliseconds or nanoseconds.
+ fn timestamp_literal(
+ unit: &TimeUnit,
+ value: Option<i64>,
+ tz: Option<Arc<str>>,
+ ) -> ScalarValue {
+ match unit {
+ TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(value, tz),
+ TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(value, tz),
+ other => unreachable!("unsupported test time unit {other:?}"),
+ }
+ }
+
+ #[test]
+ fn test_no_unwrap_naive_to_timezone_timestamp() {
+ let tz: Option<Arc<str>> = Some("+07:00".into());
+
+ // cast(naive_ts AS Timestamp(unit, "+07:00")) must NOT unwrap for CAST or
+ // TRY_CAST: Arrow reads the naive value as local wall-clock time in
+ // "+07:00" and shifts it to UTC, which the unit arithmetic does not
+ // model. Same unit, widening and narrowing targets are all rejected.
+ for (source_unit, target_unit) in [
+ (TimeUnit::Nanosecond, TimeUnit::Nanosecond),
+ (TimeUnit::Millisecond, TimeUnit::Nanosecond),
+ (TimeUnit::Nanosecond, TimeUnit::Millisecond),
+ ] {
+ let schema = timestamp_schema(source_unit);
+ let column_expr = col("ts", &schema).unwrap();
+ let target_type = DataType::Timestamp(target_unit, tz.clone());
+ let literal = lit(timestamp_literal(&target_unit, Some(0), tz.clone()));
+
+ for try_cast in [false, true] {
+ let cast_expr: Arc<dyn PhysicalExpr> = if try_cast {
+ Arc::new(TryCastExpr::new(
+ Arc::clone(&column_expr),
+ target_type.clone(),
+ ))
+ } else {
+ Arc::new(CastExpr::new(
+ Arc::clone(&column_expr),
+ target_type.clone(),
+ None,
+ ))
+ };
+ let binary_expr = Arc::new(BinaryExpr::new(
+ cast_expr,
+ Operator::Eq,
+ Arc::clone(&literal),
+ ));
+
+ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
+ assert!(
+ !result.transformed,
+ "unexpected rewrite: {source_unit:?} -> {target_unit:?}, try_cast={try_cast}"
+ );
+ }
+ }
+
+ // Safe control: the same widening to a *naive* target still unwraps.
+ let ms_schema = timestamp_schema(TimeUnit::Millisecond);
+ let column_expr = col("ts", &ms_schema).unwrap();
+ let cast_expr = Arc::new(CastExpr::new(
+ column_expr,
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ None,
+ ));
+ let literal_expr = lit(ScalarValue::TimestampNanosecond(Some(123_000_000), None));
+ let binary_expr =
+ Arc::new(BinaryExpr::new(cast_expr, Operator::GtEq, literal_expr));
+
+ let result = unwrap_cast_in_comparison(binary_expr, &ms_schema).unwrap();
+ assert!(result.transformed);
+ }
+
+ #[test]
+ fn test_timestamp_precision_narrowing_typed_null_literal_kept() {
+ let schema = timestamp_schema(TimeUnit::Nanosecond);
+
+ // A typed NULL literal has no truncation bucket, so the CAST is kept
+ // rather than folded to an unbounded interval.
+ for op in [Operator::Eq, Operator::Gt, Operator::IsDistinctFrom] {
+ let binary_expr = Arc::new(BinaryExpr::new(
+ Arc::new(CastExpr::new(
+ col("ts", &schema).unwrap(),
+ DataType::Timestamp(TimeUnit::Millisecond, None),
+ None,
+ )),
+ op,
+ lit(ScalarValue::TimestampMillisecond(None, None)),
+ ));
+ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
+ assert!(!result.transformed, "unexpected rewrite for {op:?}");
+ }
+ }
+
+ #[test]
+ fn test_timestamp_dictionary_source_keeps_source_type() {
+ let key_type = Box::new(DataType::Int32);
+ let source_type = DataType::Dictionary(
+ key_type.clone(),
+ Box::new(DataType::Timestamp(
+ TimeUnit::Nanosecond,
+ Some("+05:30".into()),
+ )),
+ );
+ let schema = Schema::new(vec![Field::new("ts", source_type.clone(), false)]);
+
+ // Narrowing a dictionary-typed column must keep the source dictionary
+ // key, unit and timezone on both bucket bounds.
+ let binary_expr = Arc::new(BinaryExpr::new(
+ Arc::new(CastExpr::new(
+ col("ts", &schema).unwrap(),
+ DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
+ None,
+ )),
+ Operator::Eq,
+ lit(ScalarValue::TimestampMillisecond(
+ Some(-1),
+ Some("UTC".into()),
+ )),
+ ));
+
+ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
+ assert!(result.transformed);
+ let and_binary = result.data.downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*and_binary.op(), Operator::And);
+
+ for (binary, expected) in [
+ (and_binary.left(), -1_999_999),
+ (and_binary.right(), -999_999),
+ ] {
+ let binary = binary.downcast_ref::<BinaryExpr>().unwrap();
+ let literal = binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(literal.value().data_type(), source_type);
+ assert_eq!(
+ literal.value(),
+ &ScalarValue::Dictionary(
+ key_type.clone(),
+ Box::new(ScalarValue::TimestampNanosecond(
+ Some(expected),
+ Some("+05:30".into())
+ ))
+ )
+ );
+ }
+ }
+
+ #[test]
+ fn test_rewrite_with_preimage_rejects_non_comparison_operator() {
+ // Range preimages only map onto comparison operators; anything else is
+ // an internal error rather than a panic.
+ let schema = test_schema();
+ let expr = col("c1", &schema).unwrap();
+ let interval = datafusion_expr_common::interval_arithmetic::Interval::try_new(
+ ScalarValue::Int32(Some(0)),
+ ScalarValue::Int32(Some(10)),
+ )
+ .unwrap();
+
+ let err = rewrite_with_preimage(interval, Operator::Plus, expr).unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Expect comparison operators, got +"),
+ "unexpected error: {err}"
+ );
+ }
+
#[test]
fn test_no_unwrap_when_types_unsupported() {
let schema = Schema::new(vec![Field::new("f1", DataType::Float32, false)]);
@@ -578,55 +850,260 @@ mod tests {
}
#[test]
- fn test_not_unwrap_timestamp_precision_narrowing() {
- let schema = Schema::new(vec![Field::new(
- "ts",
- DataType::Timestamp(TimeUnit::Nanosecond, None),
- false,
- )]);
+ fn test_timestamp_precision_narrowing_range_preimage_gt() {
+ let schema = timestamp_schema(TimeUnit::Nanosecond);
+ let binary_expr = timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Millisecond,
+ Operator::Gt,
+ ScalarValue::TimestampMillisecond(Some(1000), None),
+ );
- let column_expr = col("ts", &schema).unwrap();
- let cast_expr = Arc::new(CastExpr::new(
- column_expr,
- DataType::Timestamp(TimeUnit::Millisecond, None),
- None,
- ));
- let literal_expr = lit(ScalarValue::TimestampMillisecond(Some(1), None));
- let binary_expr =
- Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal_expr));
+ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
+
+ assert!(result.transformed);
+ let optimized_binary = result.data.downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*optimized_binary.op(), Operator::GtEq);
+ assert!(!is_cast_expr(optimized_binary.left()));
+ let right_literal = optimized_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ right_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(1_001_000_000), None)
+ );
+ }
+
+ #[test]
+ fn test_timestamp_precision_narrowing_range_preimage_eq() {
+ let schema = timestamp_schema(TimeUnit::Nanosecond);
+ let binary_expr = timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Millisecond,
+ Operator::Eq,
+ ScalarValue::TimestampMillisecond(Some(-1), None),
+ );
let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
+ assert!(result.transformed);
+ let and_binary = result.data.downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*and_binary.op(), Operator::And);
+
+ let lower_binary = and_binary.left().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*lower_binary.op(), Operator::GtEq);
+ let lower_literal = lower_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ lower_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(-1_999_999), None)
+ );
+
+ let upper_binary = and_binary.right().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*upper_binary.op(), Operator::Lt);
+ let upper_literal = upper_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ upper_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(-999_999), None)
+ );
+ }
+
+ #[test]
+ fn test_timestamp_widening_ordered() {
+ let schema = timestamp_schema(TimeUnit::Millisecond);
+ assert_timestamp_widening_rewrite(
+ &schema,
+ timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Nanosecond,
+ Operator::GtEq,
+ ScalarValue::TimestampNanosecond(Some(123_000_000), None),
+ ),
+ Operator::GtEq,
+ 123,
+ );
+
+ // A non-aligned widening literal maps from no source value, so the
+ // rewrite uses the empty interval `[floor + 1, floor + 1)`.
+ for (literal, ceil) in [(123_456_789, 124), (-123_456_789, -123)] {
+ for (op, expected_op) in [
+ (Operator::GtEq, Operator::GtEq),
+ (Operator::Gt, Operator::GtEq),
+ (Operator::Lt, Operator::Lt),
+ (Operator::LtEq, Operator::Lt),
+ ] {
+ assert_timestamp_widening_rewrite(
+ &schema,
+ timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Nanosecond,
+ op,
+ ScalarValue::TimestampNanosecond(Some(literal), None),
+ ),
+ expected_op,
+ ceil,
+ );
+ }
+ }
+
+ let try_cast_expr = Arc::new(TryCastExpr::new(
+ col("ts", &schema).unwrap(),
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ ));
+ assert_timestamp_widening_rewrite(
+ &schema,
+ Arc::new(BinaryExpr::new(
+ try_cast_expr,
+ Operator::GtEq,
+ lit(ScalarValue::TimestampNanosecond(Some(123_456_789), None)),
+ )),
+ Operator::GtEq,
+ 124,
+ );
+
+ let result = unwrap_cast_in_comparison(
+ timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Nanosecond,
+ Operator::Eq,
+ ScalarValue::TimestampNanosecond(Some(123_456_789), None),
+ ),
+ &schema,
+ )
+ .unwrap();
assert!(!result.transformed);
}
#[test]
- fn test_unwrap_timestamp_precision_widening() {
- let schema = Schema::new(vec![Field::new(
- "ts",
- DataType::Timestamp(TimeUnit::Millisecond, None),
- false,
- )]);
+ fn test_timestamp_widening_literal_left_ordered() {
+ let schema = timestamp_schema(TimeUnit::Millisecond);
+ for (op, expected_op) in [
+ (Operator::Lt, Operator::GtEq),
+ (Operator::LtEq, Operator::GtEq),
+ (Operator::Gt, Operator::Lt),
+ (Operator::GtEq, Operator::Lt),
+ ] {
+ let cast_expr = Arc::new(CastExpr::new(
+ col("ts", &schema).unwrap(),
+ DataType::Timestamp(TimeUnit::Nanosecond, None),
+ None,
+ ));
+ let binary_expr = Arc::new(BinaryExpr::new(
+ lit(ScalarValue::TimestampNanosecond(Some(123_456_789), None)),
+ op,
+ cast_expr,
+ ));
+ assert_timestamp_widening_rewrite(&schema, binary_expr, expected_op, 124);
+ }
- let column_expr = col("ts", &schema).unwrap();
- let cast_expr = Arc::new(CastExpr::new(
- column_expr,
+ let try_cast_expr = Arc::new(TryCastExpr::new(
+ col("ts", &schema).unwrap(),
DataType::Timestamp(TimeUnit::Nanosecond, None),
- None,
));
- let literal_expr = lit(ScalarValue::TimestampNanosecond(Some(1_000_000), None));
- let binary_expr =
- Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal_expr));
+ assert_timestamp_widening_rewrite(
+ &schema,
+ Arc::new(BinaryExpr::new(
+ lit(ScalarValue::TimestampNanosecond(Some(-123_456_789), None)),
+ Operator::GtEq,
+ try_cast_expr,
+ )),
+ Operator::Lt,
+ -123,
+ );
+ }
+
+ #[test]
+ fn test_timestamp_precision_narrowing_range_preimage_is_distinct_from() {
+ let schema = timestamp_schema(TimeUnit::Nanosecond);
+ let binary_expr = timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Millisecond,
+ Operator::IsDistinctFrom,
+ ScalarValue::TimestampMillisecond(Some(1000), None),
+ );
let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
assert!(result.transformed);
- let optimized_binary = result.data.downcast_ref::<BinaryExpr>().unwrap();
- assert!(!is_cast_expr(optimized_binary.left()));
- let right_literal = optimized_binary.right().downcast_ref::<Literal>().unwrap();
+
+ // Expected: OR( OR(expr < lower, expr >= upper), IS NULL(expr) )
+ let outer_or = result.data.downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*outer_or.op(), Operator::Or);
+
+ // Right side of outer OR → IS NULL
+ assert!(
+ outer_or
+ .right()
+ .downcast_ref::<crate::expressions::IsNullExpr>()
+ .is_some()
+ );
+
+ // Left side of outer OR → OR(expr < lower, expr >= upper)
+ let inner_or = outer_or.left().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*inner_or.op(), Operator::Or);
+
+ // Left-left: expr < lower
+ let lt_binary = inner_or.left().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*lt_binary.op(), Operator::Lt);
+ let lt_literal = lt_binary.right().downcast_ref::<Literal>().unwrap();
assert_eq!(
- right_literal.value(),
- &ScalarValue::TimestampMillisecond(Some(1), None)
+ lt_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(1_000_000_000), None)
+ );
+
+ // Left-right: expr >= upper
+ let gte_binary = inner_or.right().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*gte_binary.op(), Operator::GtEq);
+ let gte_literal = gte_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ gte_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(1_001_000_000), None)
+ );
+ }
+
+ #[test]
+ fn test_timestamp_precision_narrowing_range_preimage_is_not_distinct_from() {
+ let schema = timestamp_schema(TimeUnit::Nanosecond);
+ let binary_expr = timestamp_cast_comparison(
+ &schema,
+ TimeUnit::Millisecond,
+ Operator::IsNotDistinctFrom,
+ ScalarValue::TimestampMillisecond(Some(1000), None),
+ );
+
+ let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap();
+
+ assert!(result.transformed);
+
+ // Expected: AND( AND(IS NOT NULL(expr), expr >= lower), expr < upper )
+ let outer_and = result.data.downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*outer_and.op(), Operator::And);
+
+ // Right side of outer AND → expr < upper
+ let upper_binary = outer_and.right().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*upper_binary.op(), Operator::Lt);
+ let upper_literal = upper_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ upper_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(1_001_000_000), None)
+ );
+
+ // Left side of outer AND → AND(IS NOT NULL(expr), expr >= lower)
+ let inner_and = outer_and.left().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*inner_and.op(), Operator::And);
+
+ // Left-left: IS NOT NULL
+ assert!(
+ inner_and
+ .left()
+ .downcast_ref::<crate::expressions::IsNotNullExpr>()
+ .is_some()
+ );
+
+ // Left-right: expr >= lower
+ let gte_binary = inner_and.right().downcast_ref::<BinaryExpr>().unwrap();
+ assert_eq!(*gte_binary.op(), Operator::GtEq);
+ let gte_literal = gte_binary.right().downcast_ref::<Literal>().unwrap();
+ assert_eq!(
+ gte_literal.value(),
+ &ScalarValue::TimestampNanosecond(Some(1_000_000_000), None)
);
}
@@ -636,6 +1113,8 @@ mod tests {
// Create a more complex expression with nested casts
// (cast(c1 as INT64) > INT64(10)) AND (cast(c2 as INT32) = INT32(20))
+ // c2 is Int64 → the Int64→Int32 cast is narrowing and blocked
+ // by the family gate, so only the left side should be unwrapped.
let c1_expr = col("c1", &schema).unwrap();
let c1_cast = Arc::new(CastExpr::new(c1_expr, DataType::Int64, None));
let c1_literal = lit(10i64);
@@ -654,23 +1133,22 @@ mod tests {
.transform_down(|node| unwrap_cast_in_comparison(node, &schema))
.unwrap();
- // Should be transformed
+ // Should be transformed (at least the left side)
assert!(result.transformed);
- // Verify both sides of the AND were optimized
+ // Verify both sides of the AND
let optimized = result.data;
let and_binary = optimized.downcast_ref::<BinaryExpr>().unwrap();
- // Left side should be: c1 > INT32(10)
+ // Left side should be: c1 > INT32(10) (Int32→Int64 widening, unwrapped)
let left_binary = and_binary.left().downcast_ref::<BinaryExpr>().unwrap();
assert!(!is_cast_expr(left_binary.left()));
let left_literal = left_binary.right().downcast_ref::<Literal>().unwrap();
assert_eq!(left_literal.value(), &ScalarValue::Int32(Some(10)));
- // Right side should be: c2 = INT64(20) (c2 is already INT64, literal cast to match)
+ // Right side CAST(c2 as INT32) = 20 should stay as-is
+ // (Int64→Int32 is narrowing, blocked by the family gate).
let right_binary = and_binary.right().downcast_ref::<BinaryExpr>().unwrap();
- assert!(!is_cast_expr(right_binary.left()));
- let right_literal = right_binary.right().downcast_ref::<Literal>().unwrap();
- assert_eq!(right_literal.value(), &ScalarValue::Int64(Some(20)));
+ assert!(is_cast_expr(right_binary.left()));
}
}
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index d608d857ec..c58fbddcee 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -5422,6 +5422,12 @@ query P
SELECT column1 FROM t_europe WHERE column1 = '2024-01-31T16:00:01' AT TIME ZONE 'America/Los_Angeles';
----
+# Los Angeles 15:00 is the same instant as Brussels midnight.
+query P
+SELECT column1 FROM t_europe WHERE column1 = '2024-01-31T15:00:01' AT TIME ZONE 'America/Los_Angeles';
+----
+2024-02-01T00:00:01+01:00
+
query P
SELECT column1 FROM t_europe WHERE column1 BETWEEN '2020-01-01T00:00:00' AT TIME ZONE 'Australia/Brisbane' AND '2024-02-01T00:00:00' AT TIME ZONE 'America/Los_Angeles';
----
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
index 90b553551d..dfdc23b19e 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt
@@ -604,23 +604,21 @@ SELECT column1 FROM naive_col WHERE column1 = '2024-07-01T12:00:00Z'::timestampt
----
2024-07-01T12:00:00
-# KNOWN WRONG. The same comparison under a non-UTC session zone. The literal is
-# now tz-aware (18:00Z, i.e. 12:00 Denver), so coercion casts the tz-naive
-# column to America/Denver, which reads its 12:00 as 12:00 Denver and should
-# match the July row. unwrap_cast then moves the cast off the column and onto
-# the literal, and that cast renders the literal as its UTC wall clock (18:00),
-# so no row matches. PostgreSQL under TimeZone='America/Denver' returns 1.
-# See https://github.com/apache/datafusion/issues/25095
+# tz-naive column compared to a tz-aware literal under a non-UTC session zone.
+# The tz-aware literal is 18:00Z, i.e. 12:00 Denver, so the column's 12:00 read
+# in America/Denver matches it. The cast stays on the column (it cannot move to
+# the literal: Arrow adjusts naive values to UTC when the target is tz-aware).
+# PostgreSQL under TimeZone='America/Denver' returns 1.
statement ok
SET datafusion.execution.time_zone = 'America/Denver'
query I
SELECT count(*) FROM naive_col WHERE column1 = '2024-07-01T18:00:00Z'::timestamptz
----
-0
+1
-# The plan shows the rewrite: the filter compares the naive column against a
-# naive 18:00 (1719856800 s), not against 12:00.
+# The plan keeps the cast on the naive column, comparing against the tz-aware
+# 18:00 (1719856800 s) literal.
statement ok
set datafusion.explain.logical_plan_only = true
@@ -628,7 +626,7 @@ query TT
EXPLAIN SELECT column1 FROM naive_col WHERE column1 = '2024-07-01T18:00:00Z'::timestamptz
----
logical_plan
-01)Filter: naive_col.column1 = TimestampNanosecond(1719856800000000000, None)
+01)Filter: CAST(naive_col.column1 AS Timestamp(ns, "America/Denver")) = TimestampNanosecond(1719856800000000000, Some("America/Denver"))
02)--TableScan: naive_col projection=[column1]
statement ok
diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt
index c70ff2b955..0ebe97e58f 100644
--- a/datafusion/sqllogictest/test_files/simplify_expr.slt
+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt
@@ -248,6 +248,205 @@ physical_plan
statement ok
drop table dates;
+# Decimal casts from integer or decimal sources unwrap for aligned literals
+# when the target only adds scale, while an unaligned IN member keeps the cast.
+# NULL inputs retain SQL comparison behavior.
+statement ok
+create table decimal_prices(
+ price decimal(38,2),
+ price30 decimal(30,2),
+ price28 decimal(28,0),
+ price36 decimal(36,4)
+) as values
+ (4.00, 4.00, 4, 4.0000),
+ (-1.00, -1.00, -1, -1.0000),
+ (0.00, 0.00, 0, 0.0000),
+ (5.00, 5.00, 5, 5.0000),
+ (NULL, NULL, NULL, NULL);
+
+query TT
+explain select price from decimal_prices where price = 4.00;
+----
+logical_plan
+01)Filter: decimal_prices.price = Decimal128(4.00,38,2)
+02)--TableScan: decimal_prices projection=[price]
+physical_plan
+01)FilterExec: price@0 = 4.00
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query BBBBBBBB
+select
+ cast(price as decimal(38,15)) = 4.000000000000000,
+ cast(price as decimal(38,15)) <> 4.000000000000000,
+ cast(price as decimal(38,15)) < 4.000000000000000,
+ cast(price as decimal(38,15)) <= 4.000000000000000,
+ cast(price as decimal(38,15)) > 4.000000000000000,
+ cast(price as decimal(38,15)) >= 4.000000000000000,
+ cast(price as decimal(38,15)) is distinct from 4.000000000000000,
+ cast(price as decimal(38,15)) is not distinct from 4.000000000000000
+from decimal_prices order by price nulls last;
+----
+false true true true false false true false
+false true true true false false true false
+true false false true false true false true
+false true false false true true true false
+NULL NULL NULL NULL NULL NULL true false
+
+query R
+select price from decimal_prices where price in (
+ cast(4.000000000000000 as decimal(38,15)),
+ cast(4.000000000000001 as decimal(38,15)),
+ NULL
+);
+----
+4.00
+
+query TT
+explain select price from decimal_prices where cast(price as decimal(38,15)) in (
+ 4.000000000000000,
+ 4.000000000000001,
+ NULL
+);
+----
+logical_plan
+01)Filter: CAST(decimal_prices.price AS Decimal128(38, 15)) IN ([Decimal128(4.000000000000000,38,15), Decimal128(4.000000000000001,38,15), Decimal128(NULL,38,15)])
+02)--TableScan: decimal_prices projection=[price]
+physical_plan
+01)FilterExec: CAST(price@0 AS Decimal128(38, 15)) IN (SET) ([4.000000000000000, 4.000000000000001, NULL])
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+explain select price from decimal_prices where cast(price as decimal(38,15)) in (
+ 4.000000000000000,
+ 5.000000000000000,
+ NULL
+);
+----
+logical_plan
+01)Filter: decimal_prices.price = Decimal128(4.00,38,2) OR decimal_prices.price = Decimal128(5.00,38,2)
+02)--TableScan: decimal_prices projection=[price]
+physical_plan
+01)FilterExec: price@0 = 4.00 OR price@0 = 5.00
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query BB
+select
+ price in (4.000000000000000, 5.000000000000000, NULL),
+ price not in (4.000000000000000, 5.000000000000000, NULL)
+from decimal_prices order by price nulls last;
+----
+NULL NULL
+NULL NULL
+true false
+true false
+NULL NULL
+
+# Scale-up regression for the review's three source shapes. The natural
+# comparisons coerce to a scale-up cast that unwraps to a bare-column
+# comparison at the source type, and the queries return the actual 4.00 value,
+# not only a matching plan snapshot.
+query R
+select price30 from decimal_prices where price30 = 4.00;
+----
+4.00
+
+query TT
+explain select price30 from decimal_prices where price30 = 4.00;
+----
+logical_plan
+01)Filter: decimal_prices.price30 = Decimal128(4.00,30,2)
+02)--TableScan: decimal_prices projection=[price30]
+physical_plan
+01)FilterExec: price30@0 = 4.00
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query R
+select price28 from decimal_prices where price28 = 4.00;
+----
+4
+
+query TT
+explain select price28 from decimal_prices where price28 = 4.00;
+----
+logical_plan
+01)Filter: decimal_prices.price28 = Decimal128(4,28,0)
+02)--TableScan: decimal_prices projection=[price28]
+physical_plan
+01)FilterExec: price28@0 = 4
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query R
+select price36 from decimal_prices where price36 = 4.00;
+----
+4.0000
+
+query TT
+explain select price36 from decimal_prices where price36 = 4.00;
+----
+logical_plan
+01)Filter: decimal_prices.price36 = Decimal128(4.0000,36,4)
+02)--TableScan: decimal_prices projection=[price36]
+physical_plan
+01)FilterExec: price36@0 = 4.0000
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop table decimal_prices;
+
+# Integer -> decimal scale-up is admitted by the same type-based rule: the
+# rewrite compares the integer column directly, so a source value that Arrow's
+# strict cast rejects (and TRY_CAST renders NULL) simply fails the predicate
+# instead of raising an error.
+statement ok
+create table int_prices(v int) as values (4), (2147483647), (-2000000000), (NULL);
+
+statement error
+select cast(v as decimal(10,2)) from int_prices;
+
+query IR
+select v, try_cast(v as decimal(10,2)) as d from int_prices order by v nulls last;
+----
+-2000000000 NULL
+4 4.00
+2147483647 NULL
+NULL NULL
+
+# Newly admitted overflow policy: the strict comparison compares the integer
+# column directly, so the rows whose cast would overflow are TRUE/FALSE instead
+# of a cast error. The TRY_CAST comparison is rewritten the same way and so no
+# longer NULL for those rows; only the NULL source row stays NULL. A plain
+# TRY_CAST projection above still yields NULL, since it is not a comparison.
+query BBB
+select
+ cast(v as decimal(10,2)) > cast(4 as decimal(10,2)) as strict_gt,
+ try_cast(v as decimal(10,2)) > cast(4 as decimal(10,2)) as try_gt,
+ try_cast(v as decimal(10,2)) is distinct
+ from cast(null as decimal(10,2)) as try_distinct
+from int_prices order by v nulls last;
+----
+false false true
+false false true
+true true true
+NULL NULL false
+
+query I
+select v from int_prices where cast(v as decimal(10,2)) = 4.00;
+----
+4
+
+query TT
+explain select v from int_prices where cast(v as decimal(10,2)) = 4.00;
+----
+logical_plan
+01)Filter: int_prices.v = Int32(4)
+02)--TableScan: int_prices projection=[v]
+physical_plan
+01)FilterExec: v@0 = 4
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop table int_prices;
+
# ------------------------------------------------------------------------
# Unwrapping Date32 <-> Date64 casts in comparison predicates.
#
@@ -518,3 +717,253 @@ SELECT
FROM (VALUES (2, NULL::INT)) AS t(i, n);
----
NULL NULL
+
+# Cast predicate preimage rewrites for timestamp precision casts
+statement ok
+CREATE TABLE cast_preimage_ts AS
+SELECT
+ arrow_cast(0, 'Timestamp(ns)') AS ts_nano,
+ arrow_cast(0, 'Timestamp(ms)') AS ts_milli;
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_nano AS TIMESTAMP(3)) > arrow_cast(1000, 'Timestamp(ms)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_nano >= TimestampNanosecond(1001000000, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_nano@0 >= 1001000000
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_nano AS TIMESTAMP(3)) IS DISTINCT FROM arrow_cast(0, 'Timestamp(ms)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_nano < TimestampNanosecond(-999999, None) OR cast_preimage_ts.ts_nano >= TimestampNanosecond(1000000, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_nano@0 < -999999 OR ts_nano@0 >= 1000000
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_milli AS TIMESTAMP(9)) >= arrow_cast(123000000, 'Timestamp(ns)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_milli >= TimestampMillisecond(123, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_milli@1 >= 123
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_milli AS TIMESTAMP(9)) >= arrow_cast(123456789, 'Timestamp(ns)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_milli >= TimestampMillisecond(124, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_milli@1 >= 124
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# A non-aligned widening literal has no singleton source preimage. Ordered
+# comparisons use the empty preimage interval `[floor + 1, floor + 1)`, so both
+# strict and inclusive bounds move to the first source value at or above the
+# literal: `> 123456789ns` and `>= 123456789ns` become `>= 124ms`, while
+# `< 123456789ns` and `<= 123456789ns` become `< 124ms`. The matched rows are
+# unchanged; only the boundary predicate is expressed in source units.
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_milli AS TIMESTAMP(9)) > arrow_cast(123456789, 'Timestamp(ns)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_milli >= TimestampMillisecond(124, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_milli@1 >= 124
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_milli AS TIMESTAMP(9)) < arrow_cast(123456789, 'Timestamp(ns)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_milli < TimestampMillisecond(124, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_milli@1 < 124
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_milli AS TIMESTAMP(9)) <= arrow_cast(123456789, 'Timestamp(ns)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_milli < TimestampMillisecond(124, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_milli@1 < 124
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Equality-like operators keep the CAST: a non-aligned literal has no
+# equivalent source-domain singleton, so there is nothing to fold.
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_milli AS TIMESTAMP(9)) = arrow_cast(123456789, 'Timestamp(ns)');
+----
+logical_plan
+01)Filter: CAST(cast_preimage_ts.ts_milli AS Timestamp(ns)) = TimestampNanosecond(123456789, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: CAST(ts_milli@1 AS Timestamp(ns)) = 123456789
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query TT
+EXPLAIN SELECT * FROM cast_preimage_ts
+WHERE CAST(ts_nano AS TIMESTAMP(3)) = arrow_cast(-1, 'Timestamp(ms)');
+----
+logical_plan
+01)Filter: cast_preimage_ts.ts_nano >= TimestampNanosecond(-1999999, None) AND cast_preimage_ts.ts_nano < TimestampNanosecond(-999999, None)
+02)--TableScan: cast_preimage_ts projection=[ts_nano, ts_milli]
+physical_plan
+01)FilterExec: ts_nano@0 >= -1999999 AND ts_nano@0 < -999999
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+# Signed boundary rows, selected by id so the returned rows are explicit.
+statement ok
+CREATE TABLE cast_preimage_ms AS
+SELECT id, arrow_cast(v, 'Timestamp(ms)') AS ts_milli
+FROM (VALUES (1, 123), (2, 124), (3, -124), (4, -123)) AS t(id, v);
+
+statement ok
+CREATE TABLE cast_preimage_nano AS
+SELECT id, arrow_cast(v, 'Timestamp(ns)') AS ts_nano
+FROM (VALUES (1, -1999999), (2, -1999998), (3, -1000000), (4, -1000001),
+ (5, -999999), (6, -999998)) AS t(id, v);
+
+# Negative non-aligned widening: `> -123456789ns` becomes `>= -123ms`, so the
+# negative neighbour -124ms is excluded while -123ms stays (ids 4, 1, 2).
+query I
+SELECT id FROM cast_preimage_ms
+WHERE CAST(ts_milli AS TIMESTAMP(9)) > arrow_cast(-123456789, 'Timestamp(ns)')
+ORDER BY ts_milli;
+----
+4
+1
+2
+
+# Non-aligned equality keeps the CAST and matches nothing: no millisecond value
+# equals a sub-millisecond literal.
+query I
+SELECT count(*) FROM cast_preimage_ms
+WHERE CAST(ts_milli AS TIMESTAMP(9)) = arrow_cast(-123456789, 'Timestamp(ns)');
+----
+0
+
+# The narrowing bucket for -1ms is `[-1999999ns, -999999ns)`: ids 1-4 truncate
+# to -1, while ids 5 and 6 truncate to 0 and are excluded.
+query I
+SELECT id FROM cast_preimage_nano
+WHERE CAST(ts_nano AS TIMESTAMP(3)) = arrow_cast(-1, 'Timestamp(ms)')
+ORDER BY ts_nano;
+----
+1
+2
+4
+3
+
+# A typed NULL literal in a narrowing cast has no truncation bucket, so the CAST
+# is not rewritten and the query still matches no rows (three-valued logic).
+query I
+SELECT count(*) FROM cast_preimage_ts
+WHERE CAST(ts_nano AS TIMESTAMP(3)) = arrow_cast(NULL, 'Timestamp(ms)');
+----
+0
+
+statement ok
+DROP TABLE cast_preimage_nano;
+
+statement ok
+DROP TABLE cast_preimage_ms;
+
+statement ok
+DROP TABLE cast_preimage_ts;
+
+# Naive -> timezone-aware timestamp casts are not preimage-rewritable.
+#
+# Arrow reads a timezone-naive timestamp as local wall-clock time and shifts it
+# to UTC when the target is timezone-aware (`adjust_timestamp_to_timezone`), so
+# `CAST(ts AS Timestamp(unit, Some(tz))) OP literal` is not a pure unit change.
+# The timestamp preimage works on raw source units and does not model that
+# shift, so the CAST must stay on the column for same-unit, widening and
+# narrowing targets alike; a rejected timestamp cast never falls back to a
+# generic cast-family rewrite.
+#
+# `2024-01-01T00:00:00.5` is 1704067200500000000 ns of naive wall-clock time;
+# read in `+07:00` and shifted to UTC it is 1704042000500000000 ns
+# (1704042000500 ms). The rewritten (buggy) predicate would compare the raw
+# source units against those shifted values and drop the row.
+statement ok
+create table cast_preimage_tz(ts timestamp, l timestamp) as values
+ ('2024-01-01T00:00:00.5'::timestamp, '2024-01-01T00:00:00.5'::timestamp);
+
+statement ok
+create table cast_preimage_tz_ms as
+select arrow_cast('2024-01-01T00:00:00'::timestamp, 'Timestamp(Millisecond, None)') as ts_milli;
+
+# Column-vs-column control: there is no literal to fold, so the CAST is kept and
+# the comparison is evaluated as written.
+query I
+select count(*) from cast_preimage_tz
+where arrow_cast(ts, 'Timestamp(Millisecond, Some("+07:00"))')
+ = arrow_cast(l, 'Timestamp(Millisecond, Some("+07:00"))');
+----
+1
+
+# Narrowing (ns -> ms) with a literal: the naive -> aware rejection applies to
+# the bucket path too, so it stays at 1 (without the guard it would return 0).
+query I
+select count(*) from cast_preimage_tz
+where arrow_cast(ts, 'Timestamp(Millisecond, Some("+07:00"))')
+ = arrow_cast('2024-01-01T00:00:00.5'::timestamp, 'Timestamp(Millisecond, Some("+07:00"))');
+----
+1
+
+# Same unit, only the timezone changes: this pair must be rejected. Without the
+# guard the literal folds to a naive nanosecond literal and the comparison drops
+# the row (0).
+query I
+select count(*) from cast_preimage_tz
+where arrow_cast(ts, 'Timestamp(Nanosecond, Some("+07:00"))')
+ = arrow_cast('2024-01-01T00:00:00.5'::timestamp, 'Timestamp(Nanosecond, Some("+07:00"))');
+----
+1
+
+# Widening (ms -> ns) with an aligned literal goes through the same timestamp
+# preimage and must be rejected there too.
+query I
+select count(*) from cast_preimage_tz_ms
+where arrow_cast(ts_milli, 'Timestamp(Nanosecond, Some("+07:00"))')
+ = arrow_cast('2024-01-01T00:00:00'::timestamp, 'Timestamp(Nanosecond, Some("+07:00"))');
+----
+1
+
+# The IN-list rewrite uses `exact_preimage_cast`, which routes timestamp casts
+# through the same helper, and must be rejected as well. Two literal items keep
+# this on the IN-list path rather than folding to a single equality.
+query I
+select count(*) from cast_preimage_tz
+where arrow_cast(ts, 'Timestamp(Nanosecond, Some("+07:00"))')
+ in (arrow_cast('2024-01-01T00:00:00.5'::timestamp, 'Timestamp(Nanosecond, Some("+07:00"))'),
+ arrow_cast('2023-06-01T00:00:00'::timestamp, 'Timestamp(Nanosecond, Some("+07:00"))'));
+----
+1
+
+statement ok
+drop table cast_preimage_tz;
+
+statement ok
+drop table cast_preimage_tz_ms;