Skip to content

Apache DataFusion SQL Query Engine

fix: avoid query failures from inferred numeric cast bounds (#25528)

Huaijin827ca29e981c
diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs
index 48bf8a391d..8df35d363a 100644
--- a/datafusion/expr-common/src/interval_arithmetic.rs
+++ b/datafusion/expr-common/src/interval_arithmetic.rs
@@ -417,11 +417,23 @@ impl Interval {
}
/// Casts this interval to `data_type` using `cast_options`.
+ /// Numeric endpoints that fail to cast become unbounded,
+ /// regardless of `cast_options.safe`. Other cast errors are propagated.
pub fn cast_to(
&self,
data_type: &DataType,
cast_options: &CastOptions,
) -> Result<Self> {
+ // Estimated endpoints may overflow even when all runtime values fit.
+ let bound_options = CastOptions {
+ safe: true,
+ ..cast_options.clone()
+ };
+ let cast_options = if self.data_type().is_numeric() && data_type.is_numeric() {
+ &bound_options
+ } else {
+ cast_options
+ };
Self::try_new(
cast_scalar_value(&self.lower, data_type, cast_options)?,
cast_scalar_value(&self.upper, data_type, cast_options)?,
@@ -2268,16 +2280,240 @@ impl NullableInterval {
mod tests {
use crate::{
interval_arithmetic::{
- Interval, handle_overflow, next_value, prev_value, satisfy_greater,
+ Interval, cast_scalar_value, handle_overflow, next_value, prev_value,
+ satisfy_greater,
},
operator::Operator,
};
use crate::interval_arithmetic::NullableInterval;
+ use arrow::compute::CastOptions;
use arrow::datatypes::DataType;
use datafusion_common::rounding::{next_down, next_up};
use datafusion_common::{Result, ScalarValue};
+ #[test]
+ fn test_numeric_cast_out_of_range_bounds() {
+ for safe in [false, true] {
+ let options = CastOptions {
+ safe,
+ ..Default::default()
+ };
+ for source in [
+ DataType::Int64,
+ DataType::Float64,
+ DataType::Decimal128(10, 0),
+ ] {
+ for (lower, upper, expected_lower, expected_upper) in [
+ (Some(-129i64), Some(42), None, Some(42i8)),
+ (Some(-42), Some(128), Some(-42), None),
+ (Some(-129), Some(128), None, None),
+ (Some(-128), Some(127), Some(-128i8), Some(127i8)),
+ (Some(128), Some(129), None, None),
+ (Some(-130), Some(-129), None, None),
+ (None, Some(42), None, Some(42)),
+ (Some(-42), None, Some(-42), None),
+ ] {
+ let input = Interval::make(lower, upper)
+ .unwrap()
+ .cast_to(&source, &options)
+ .unwrap();
+ assert_eq!(
+ input.cast_to(&DataType::Int8, &options).unwrap(),
+ Interval::make(expected_lower, expected_upper).unwrap(),
+ "{source}: {input}, safe={safe}"
+ );
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn test_numeric_cast_bounds_across_types() {
+ for safe in [false, true] {
+ let options = CastOptions {
+ safe,
+ ..Default::default()
+ };
+ // Cover unsigned, floating-point and decimal sources and targets.
+ for source in [
+ DataType::Int64,
+ DataType::UInt64,
+ DataType::Float64,
+ DataType::Decimal128(10, 2),
+ DataType::Decimal256(30, 3),
+ ] {
+ let input = Interval::make(Some(42i64), Some(1000))
+ .unwrap()
+ .cast_to(&source, &options)
+ .unwrap();
+ for target in
+ [DataType::Int8, DataType::UInt8, DataType::Decimal128(3, 1)]
+ {
+ let lower = ScalarValue::Int64(Some(42)).cast_to(&target).unwrap();
+ assert_eq!(
+ input.cast_to(&target, &options).unwrap(),
+ Interval::try_new(lower, ScalarValue::try_from(&target).unwrap())
+ .unwrap(),
+ "{source} -> {target}, safe={safe}"
+ );
+ }
+ }
+ assert_eq!(
+ Interval::make(Some(-1i64), Some(42))
+ .unwrap()
+ .cast_to(&DataType::UInt8, &options)
+ .unwrap(),
+ Interval::make(Some(0u8), Some(42)).unwrap()
+ );
+
+ // Fractional inputs retain truncation, including with one overflowing bound.
+ for lower in [-129.5f64, -42.5] {
+ for source in [DataType::Float64, DataType::Decimal128(10, 2)] {
+ let input = Interval::make(Some(lower), Some(42.5))
+ .unwrap()
+ .cast_to(&source, &options)
+ .unwrap();
+ let expected_lower = if lower < -128.0 { None } else { Some(-42i8) };
+ assert_eq!(
+ input.cast_to(&DataType::Int8, &options).unwrap(),
+ Interval::make(expected_lower, Some(42)).unwrap()
+ );
+ }
+ }
+ // Float overflow is normalized by Interval::try_new.
+ assert_eq!(
+ Interval::make(Some(0.0f64), Some(f64::MAX))
+ .unwrap()
+ .cast_to(&DataType::Float32, &options)
+ .unwrap(),
+ Interval::make(Some(0.0f32), None).unwrap()
+ );
+ }
+ }
+
+ #[test]
+ fn test_numeric_cast_bounds_contain_values() {
+ let types = [
+ DataType::Int8,
+ DataType::Int16,
+ DataType::Int32,
+ DataType::Int64,
+ DataType::UInt8,
+ DataType::UInt16,
+ DataType::UInt32,
+ DataType::UInt64,
+ DataType::Float32,
+ DataType::Float64,
+ DataType::Decimal128(3, 0),
+ DataType::Decimal128(10, 2),
+ DataType::Decimal128(20, -2),
+ DataType::Decimal256(30, 3),
+ ];
+ let samples = [
+ i64::MIN,
+ -16777217,
+ -65537,
+ -129,
+ -128,
+ -1,
+ 0,
+ 1,
+ 42,
+ 127,
+ 128,
+ 255,
+ 256,
+ 65536,
+ 16777217,
+ i64::MAX,
+ ];
+ let safe_options = CastOptions {
+ safe: true,
+ ..Default::default()
+ };
+ let cast = |value: &ScalarValue, target: &DataType| {
+ cast_scalar_value(value, target, &safe_options)
+ };
+ for source in &types {
+ let mut values = samples
+ .iter()
+ .map(|value| cast(&ScalarValue::from(*value), source))
+ .collect::<Result<Vec<_>>>()
+ .unwrap();
+ values.retain(|value| !value.is_null());
+ for target in &types {
+ let converted = values
+ .iter()
+ .map(|value| cast(value, target))
+ .collect::<Result<Vec<_>>>()
+ .unwrap();
+ for safe in [false, true] {
+ let options = CastOptions {
+ safe,
+ ..Default::default()
+ };
+ for start in 0..values.len() {
+ for end in start..values.len() {
+ let input = Interval::try_new(
+ values[start].clone(),
+ values[end].clone(),
+ )
+ .unwrap();
+ let bounds = input.cast_to(target, &options).unwrap();
+ for value in &converted[start..=end] {
+ if !value.is_null() {
+ assert!(
+ (bounds.lower.is_null()
+ || bounds.lower <= *value)
+ && (bounds.upper.is_null()
+ || *value <= bounds.upper),
+ "{source} -> {target}, {input} -> {bounds}, value={value}, safe={safe}"
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn test_non_numeric_cast_retains_error_policy() {
+ let strict = CastOptions {
+ safe: false,
+ ..Default::default()
+ };
+ let safe = CastOptions {
+ safe: true,
+ ..strict.clone()
+ };
+ // Exercise failures at either endpoint, preserving the valid opposite bound.
+ for (lower, upper, expected_lower, expected_upper) in [
+ ("not a number", "not a number", None, None),
+ ("1", "not a number", Some(1i8), None),
+ ("!", "42", None, Some(42i8)),
+ ] {
+ let input = Interval::try_new(
+ ScalarValue::Utf8(Some(lower.into())),
+ ScalarValue::Utf8(Some(upper.into())),
+ )
+ .unwrap();
+ assert!(input.cast_to(&DataType::Int8, &strict).is_err());
+ assert_eq!(
+ input.cast_to(&DataType::Int8, &safe).unwrap(),
+ Interval::make(expected_lower, expected_upper).unwrap()
+ );
+ }
+ let numeric = Interval::make(Some(0i64), Some(1)).unwrap();
+ assert!(
+ numeric
+ .cast_to(&DataType::Struct(Default::default()), &safe)
+ .is_err()
+ );
+ }
+
#[test]
fn test_next_prev_value() -> Result<()> {
let zeros = vec![
diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.slt
index e94869e830..6d62461235 100644
--- a/datafusion/sqllogictest/test_files/cast.slt
+++ b/datafusion/sqllogictest/test_files/cast.slt
@@ -298,3 +298,37 @@ set datafusion.optimizer.expand_views_at_output = false;
statement ok
drop table t;
+
+# Regression for https://github.com/apache/datafusion/issues/25524
+
+statement ok
+CREATE TABLE narrow_column (a INT, z TINYINT) AS VALUES (1, 0);
+
+# Inferred upper bound 1000 is out of range, but all runtime values are zero.
+query I
+SELECT a FROM narrow_column WHERE CAST((a < 0)::INT * 1000 AS TINYINT) = z;
+----
+1
+
+# The same applies to the inferred lower bound -1000.
+query I
+SELECT a FROM narrow_column WHERE CAST((a < 0)::INT * -1000 AS TINYINT) = z;
+----
+1
+
+# Real overflow must still fail at execution.
+statement error Can't cast value 1000 to type Int8
+SELECT CAST(CAST(a > 0 AS INT) * 1000 AS TINYINT) FROM narrow_column;
+
+statement error Can't cast value -1000 to type Int8
+SELECT CAST(CAST(a > 0 AS INT) * -1000 AS TINYINT) FROM narrow_column;
+
+# TRY_CAST must still return NULL for real overflow.
+query II
+SELECT TRY_CAST((a > 0)::INT * 1000 AS TINYINT),
+ TRY_CAST((a > 0)::INT * -1000 AS TINYINT) FROM narrow_column;
+----
+NULL NULL
+
+statement ok
+DROP TABLE narrow_column;