Skip to content

Apache DataFusion SQL Query Engine

Various `ScalarValue` numeric method fixes & refactors (especially decimal) (#23631)

Jeffrey Vo50ae0762d393
diff --git a/datafusion/common/src/scalar/consts.rs b/datafusion/common/src/scalar/consts.rs
index 599c2523cd..df12265a37 100644
--- a/datafusion/common/src/scalar/consts.rs
+++ b/datafusion/common/src/scalar/consts.rs
@@ -17,6 +17,9 @@
// Constants defined for scalar construction.
+use arrow::datatypes::{Decimal32Type, Decimal64Type, Decimal128Type, DecimalType};
+use arrow::datatypes::{Decimal256Type, i256};
+
// Next F16 value above π (upper bound)
pub(super) const PI_UPPER_F16: half::f16 = half::f16::from_bits(0x4249);
@@ -54,3 +57,63 @@ pub(super) const NEGATIVE_FRAC_PI_2_LOWER_F32: f32 =
// Next f64 value below -π/2 (lower bound)
pub(super) const NEGATIVE_FRAC_PI_2_LOWER_F64: f64 =
(-std::f64::consts::FRAC_PI_2).next_down();
+
+// Generate lookup table for 1 values of decimals (1, 10, 100, etc.)
+macro_rules! decimal_ones_lut {
+ () => {{
+ let mut values = [1; _];
+ let mut i = 1;
+ while i < values.len() {
+ values[i] = values[i - 1] * 10;
+ i += 1;
+ }
+ values
+ }};
+}
+
+// 1, 10, 100 values meant to be indexed by scale. We omit handling for MAX_SCALE
+// itself (we don't go to MAX_SCALE + 1) since we can't represent a 1 value at
+// that scale.
+pub(super) const DECIMAL32_ONES: [i32; Decimal32Type::MAX_SCALE as usize] =
+ decimal_ones_lut!();
+pub(super) const DECIMAL64_ONES: [i64; Decimal64Type::MAX_SCALE as usize] =
+ decimal_ones_lut!();
+pub(super) const DECIMAL128_ONES: [i128; Decimal128Type::MAX_SCALE as usize] =
+ decimal_ones_lut!();
+pub(super) const DECIMAL256_ONES: [i256; Decimal256Type::MAX_SCALE as usize] = {
+ // This code was generated by codex and frankly I don't know how it works,
+ // but the test below verifies it outputs the correct values so ¯\_(ツ)_/¯
+ //
+ // This is mainly a shortcut for not needing to manually list out each value
+ // anyway.
+ //
+ // TODO: simplify this after https://github.com/apache/arrow-rs/pull/10363
+ // lands upstream
+ let mut values = [i256::ONE; _];
+ let mut i = 1;
+ while i < values.len() {
+ let (low, high) = values[i - 1].to_parts();
+ let low_product = (low as u64 as u128) * 10;
+ let high_product = (low >> 64) * 10 + (low_product >> 64);
+ let low = ((high_product as u64 as u128) << 64) | low_product as u64 as u128;
+ let carry = (high_product >> 64) as i128;
+ values[i] = i256::from_parts(low, high * 10 + carry);
+ i += 1;
+ }
+ values
+};
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_ensure_correct_decimal256_ones() {
+ for (scale, val) in DECIMAL256_ONES.iter().enumerate() {
+ let zeros = "0".repeat(scale);
+ let num = "1".to_string() + &zeros;
+ let num = i256::from_string(&num).unwrap();
+ assert_eq!(num, *val, "{scale}");
+ }
+ }
+}
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs
index ddfe32edd4..924620a930 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -54,6 +54,9 @@ use crate::cast::{
use crate::error::{_exec_err, _internal_err, _not_impl_err, DataFusionError, Result};
use crate::format::DEFAULT_CAST_OPTIONS;
use crate::hash_utils::create_hashes;
+use crate::scalar::consts::{
+ DECIMAL32_ONES, DECIMAL64_ONES, DECIMAL128_ONES, DECIMAL256_ONES,
+};
use crate::utils::SingleRowListArrayBuilder;
use crate::{_internal_datafusion_err, arrow_datafusion_err};
use arrow::array::{
@@ -83,10 +86,14 @@ use arrow::datatypes::{
Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, Field,
FieldRef, Float32Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTime,
IntervalDayTimeType, IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalUnit,
- IntervalYearMonthType, RunEndIndexType, TimeUnit, TimestampMicrosecondType,
- TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type,
- UInt16Type, UInt32Type, UInt64Type, UnionFields, UnionMode, i256,
- validate_decimal_precision_and_scale,
+ IntervalYearMonthType, MAX_DECIMAL32_FOR_EACH_PRECISION,
+ MAX_DECIMAL64_FOR_EACH_PRECISION, MAX_DECIMAL128_FOR_EACH_PRECISION,
+ MAX_DECIMAL256_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION,
+ MIN_DECIMAL64_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION,
+ MIN_DECIMAL256_FOR_EACH_PRECISION, RunEndIndexType, TimeUnit,
+ TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
+ TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionFields,
+ UnionMode, i256, validate_decimal_precision_and_scale,
};
use arrow::util::display::{ArrayFormatter, FormatOptions, array_value_to_string};
use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array};
@@ -1804,48 +1811,56 @@ impl ScalarValue {
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match 10_i32.checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal32(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL32_ONES[*scale as usize];
+ ScalarValue::Decimal32(Some(one), *precision, *scale)
}
DataType::Decimal64(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal64Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i64::from(10).checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal64(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL64_ONES[*scale as usize];
+ ScalarValue::Decimal64(Some(one), *precision, *scale)
}
DataType::Decimal128(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal128Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i128::from(10).checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal128(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL128_ONES[*scale as usize];
+ ScalarValue::Decimal128(Some(one), *precision, *scale)
}
DataType::Decimal256(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal256Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i256::from(10).checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal256(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL256_ONES[*scale as usize];
+ ScalarValue::Decimal256(Some(one), *precision, *scale)
}
_ => {
return _not_impl_err!(
@@ -1858,10 +1873,10 @@ impl ScalarValue {
/// Create a negative one value in the given type.
pub fn new_negative_one(datatype: &DataType) -> Result<ScalarValue> {
Ok(match datatype {
- DataType::Int8 | DataType::UInt8 => ScalarValue::Int8(Some(-1)),
- DataType::Int16 | DataType::UInt16 => ScalarValue::Int16(Some(-1)),
- DataType::Int32 | DataType::UInt32 => ScalarValue::Int32(Some(-1)),
- DataType::Int64 | DataType::UInt64 => ScalarValue::Int64(Some(-1)),
+ DataType::Int8 => ScalarValue::Int8(Some(-1)),
+ DataType::Int16 => ScalarValue::Int16(Some(-1)),
+ DataType::Int32 => ScalarValue::Int32(Some(-1)),
+ DataType::Int64 => ScalarValue::Int64(Some(-1)),
DataType::Float16 => ScalarValue::Float16(Some(f16::NEG_ONE)),
DataType::Float32 => ScalarValue::Float32(Some(-1.0)),
DataType::Float64 => ScalarValue::Float64(Some(-1.0)),
@@ -1870,48 +1885,56 @@ impl ScalarValue {
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match 10_i32.checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal32(Some(-value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent negative one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL32_ONES[*scale as usize];
+ ScalarValue::Decimal32(Some(-one), *precision, *scale)
}
DataType::Decimal64(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal64Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i64::from(10).checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal64(Some(-value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent negative one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL64_ONES[*scale as usize];
+ ScalarValue::Decimal64(Some(-one), *precision, *scale)
}
DataType::Decimal128(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal128Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i128::from(10).checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal128(Some(-value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent negative one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL128_ONES[*scale as usize];
+ ScalarValue::Decimal128(Some(-one), *precision, *scale)
}
DataType::Decimal256(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal256Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i256::from(10).checked_pow(*scale as u32) {
- Some(value) => {
- ScalarValue::Decimal256(Some(-value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ *precision != *scale as u8,
+ "Can't represent one at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ let one = DECIMAL256_ONES[*scale as usize];
+ ScalarValue::Decimal256(Some(-one), *precision, *scale)
}
_ => {
return _not_impl_err!(
@@ -1939,48 +1962,64 @@ impl ScalarValue {
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match 10_i32.checked_pow((*scale + 1) as u32) {
- Some(value) => {
- ScalarValue::Decimal32(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ (*precision - *scale as u8) > 1,
+ "Can't represent ten at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ // +1 safe since we validate above that scale must be less than
+ // the max possible scale
+ let ten = DECIMAL32_ONES[*scale as usize + 1];
+ ScalarValue::Decimal32(Some(ten), *precision, *scale)
}
DataType::Decimal64(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal64Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i64::from(10).checked_pow((*scale + 1) as u32) {
- Some(value) => {
- ScalarValue::Decimal64(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ (*precision - *scale as u8) > 1,
+ "Can't represent ten at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ // +1 safe since we validate above that scale must be less than
+ // the max possible scale
+ let ten = DECIMAL64_ONES[*scale as usize + 1];
+ ScalarValue::Decimal64(Some(ten), *precision, *scale)
}
DataType::Decimal128(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal128Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i128::from(10).checked_pow((*scale + 1) as u32) {
- Some(value) => {
- ScalarValue::Decimal128(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ (*precision - *scale as u8) > 1,
+ "Can't represent ten at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ // +1 safe since we validate above that scale must be less than
+ // the max possible scale
+ let ten = DECIMAL128_ONES[*scale as usize + 1];
+ ScalarValue::Decimal128(Some(ten), *precision, *scale)
}
DataType::Decimal256(precision, scale) => {
Self::validate_decimal_or_internal_err::<Decimal256Type>(
*precision, *scale,
)?;
assert_or_internal_err!(*scale >= 0, "Negative scale is not supported");
- match i256::from(10).checked_pow((*scale + 1) as u32) {
- Some(value) => {
- ScalarValue::Decimal256(Some(value), *precision, *scale)
- }
- None => return _internal_err!("Unsupported scale {scale}"),
- }
+ assert_or_internal_err!(
+ (*precision - *scale as u8) > 1,
+ "Can't represent ten at scale {} with precision {}",
+ *scale,
+ *precision
+ );
+ // +1 safe since we validate above that scale must be less than
+ // the max possible scale
+ let ten = DECIMAL256_ONES[*scale as usize + 1];
+ ScalarValue::Decimal256(Some(ten), *precision, *scale)
}
_ => {
return _not_impl_err!(
@@ -2299,7 +2338,18 @@ impl ScalarValue {
| ScalarValue::Int64(None)
| ScalarValue::Float16(None)
| ScalarValue::Float32(None)
- | ScalarValue::Float64(None) => Ok(self.clone()),
+ | ScalarValue::Float64(None)
+ | ScalarValue::IntervalYearMonth(None)
+ | ScalarValue::IntervalDayTime(None)
+ | ScalarValue::IntervalMonthDayNano(None)
+ | ScalarValue::Decimal32(None, _, _)
+ | ScalarValue::Decimal64(None, _, _)
+ | ScalarValue::Decimal128(None, _, _)
+ | ScalarValue::Decimal256(None, _, _)
+ | ScalarValue::TimestampSecond(None, _)
+ | ScalarValue::TimestampMillisecond(None, _)
+ | ScalarValue::TimestampMicrosecond(None, _)
+ | ScalarValue::TimestampNanosecond(None, _) => Ok(self.clone()),
ScalarValue::Float16(Some(v)) => Ok(ScalarValue::Float16(Some(-v))),
ScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(Some(-v))),
ScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float32(Some(-v))),
@@ -5053,28 +5103,21 @@ impl ScalarValue {
DataType::Float16 => Some(ScalarValue::Float16(Some(f16::NEG_INFINITY))),
DataType::Float32 => Some(ScalarValue::Float32(Some(f32::NEG_INFINITY))),
DataType::Float64 => Some(ScalarValue::Float64(Some(f64::NEG_INFINITY))),
+ DataType::Decimal32(precision, scale) => {
+ let min = MIN_DECIMAL32_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal32(Some(min), *precision, *scale))
+ }
+ DataType::Decimal64(precision, scale) => {
+ let min = MIN_DECIMAL64_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal64(Some(min), *precision, *scale))
+ }
DataType::Decimal128(precision, scale) => {
- // For decimal, min is -10^(precision-scale) + 10^(-scale)
- // But for simplicity, we use the minimum i128 value that fits the precision
- let max_digits = 10_i128.pow(*precision as u32) - 1;
- Some(ScalarValue::Decimal128(
- Some(-max_digits),
- *precision,
- *scale,
- ))
+ let min = MIN_DECIMAL128_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal128(Some(min), *precision, *scale))
}
DataType::Decimal256(precision, scale) => {
- // Similar to Decimal128 but with i256
- // For now, use a large negative value
- let max_digits = i256::from_i128(10_i128)
- .checked_pow(*precision as u32)
- .and_then(|v| v.checked_sub(i256::from_i128(1)))
- .unwrap_or(i256::MAX);
- Some(ScalarValue::Decimal256(
- Some(max_digits.neg_wrapping()),
- *precision,
- *scale,
- ))
+ let min = MIN_DECIMAL256_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal256(Some(min), *precision, *scale))
}
DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MIN))),
DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MIN))),
@@ -5149,27 +5192,21 @@ impl ScalarValue {
DataType::Float16 => Some(ScalarValue::Float16(Some(f16::INFINITY))),
DataType::Float32 => Some(ScalarValue::Float32(Some(f32::INFINITY))),
DataType::Float64 => Some(ScalarValue::Float64(Some(f64::INFINITY))),
+ DataType::Decimal32(precision, scale) => {
+ let max = MAX_DECIMAL32_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal32(Some(max), *precision, *scale))
+ }
+ DataType::Decimal64(precision, scale) => {
+ let max = MAX_DECIMAL64_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal64(Some(max), *precision, *scale))
+ }
DataType::Decimal128(precision, scale) => {
- // For decimal, max is 10^(precision-scale) - 10^(-scale)
- // But for simplicity, we use the maximum i128 value that fits the precision
- let max_digits = 10_i128.pow(*precision as u32) - 1;
- Some(ScalarValue::Decimal128(
- Some(max_digits),
- *precision,
- *scale,
- ))
+ let max = MAX_DECIMAL128_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal128(Some(max), *precision, *scale))
}
DataType::Decimal256(precision, scale) => {
- // Similar to Decimal128 but with i256
- let max_digits = i256::from_i128(10_i128)
- .checked_pow(*precision as u32)
- .and_then(|v| v.checked_sub(i256::from_i128(1)))
- .unwrap_or(i256::MAX);
- Some(ScalarValue::Decimal256(
- Some(max_digits),
- *precision,
- *scale,
- ))
+ let max = MAX_DECIMAL256_FOR_EACH_PRECISION[*precision as usize];
+ Some(ScalarValue::Decimal256(Some(max), *precision, *scale))
}
DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MAX))),
DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MAX))),
@@ -11470,4 +11507,41 @@ mod tests {
.unwrap();
assert_eq!(s.to_string(), "[]");
}
+
+ #[test]
+ fn test_decimal_value_bounds() {
+ fn run_tests<D: DecimalType>() {
+ // 0.1111, 0.2222, etc.
+ let max_scale = D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE);
+ // 1.111, 2.222, etc.
+ let max_scale_less_one =
+ D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 1);
+ // 11.11, 22.22, etc.
+ let max_scale_less_two =
+ D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 2);
+
+ // Invalid (can't represent the value)
+ assert!(ScalarValue::new_one(&max_scale).is_err());
+ assert!(ScalarValue::new_negative_one(&max_scale).is_err());
+ assert!(ScalarValue::new_ten(&max_scale).is_err());
+ assert!(ScalarValue::new_ten(&max_scale_less_one).is_err());
+
+ // Valid
+ let one = ScalarValue::Int32(Some(1));
+ let neg_one = ScalarValue::Int32(Some(-1));
+ let ten = ScalarValue::Int32(Some(10));
+
+ let num = ScalarValue::new_one(&max_scale_less_one).unwrap();
+ assert_eq!(num.cast_to(&DataType::Int32).unwrap(), one);
+ let num = ScalarValue::new_negative_one(&max_scale_less_one).unwrap();
+ assert_eq!(num.cast_to(&DataType::Int32).unwrap(), neg_one);
+ let num = ScalarValue::new_ten(&max_scale_less_two).unwrap();
+ assert_eq!(num.cast_to(&DataType::Int32).unwrap(), ten);
+ }
+
+ run_tests::<Decimal32Type>();
+ run_tests::<Decimal64Type>();
+ run_tests::<Decimal128Type>();
+ run_tests::<Decimal256Type>();
+ }
}
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index e4a22a3419..f5ea75dde8 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -3067,17 +3067,6 @@ mod tests {
#[test]
fn test_simplify_negated_bitwise_and() {
- // !c4 & c4 --> 0
- let expr = (-col("c4_non_null")) & col("c4_non_null");
- let expected = lit(0u32);
-
- assert_eq!(simplify(expr), expected);
- // c4 & !c4 --> 0
- let expr = col("c4_non_null") & (-col("c4_non_null"));
- let expected = lit(0u32);
-
- assert_eq!(simplify(expr), expected);
-
// !c3 & c3 --> 0
let expr = (-col("c3_non_null")) & col("c3_non_null");
let expected = lit(0i64);
@@ -3092,18 +3081,6 @@ mod tests {
#[test]
fn test_simplify_negated_bitwise_or() {
- // !c4 | c4 --> -1
- let expr = (-col("c4_non_null")) | col("c4_non_null");
- let expected = lit(-1i32);
-
- assert_eq!(simplify(expr), expected);
-
- // c4 | !c4 --> -1
- let expr = col("c4_non_null") | (-col("c4_non_null"));
- let expected = lit(-1i32);
-
- assert_eq!(simplify(expr), expected);
-
// !c3 | c3 --> -1
let expr = (-col("c3_non_null")) | col("c3_non_null");
let expected = lit(-1i64);
@@ -3119,18 +3096,6 @@ mod tests {
#[test]
fn test_simplify_negated_bitwise_xor() {
- // !c4 ^ c4 --> -1
- let expr = (-col("c4_non_null")) ^ col("c4_non_null");
- let expected = lit(-1i32);
-
- assert_eq!(simplify(expr), expected);
-
- // c4 ^ !c4 --> -1
- let expr = col("c4_non_null") ^ (-col("c4_non_null"));
- let expected = lit(-1i32);
-
- assert_eq!(simplify(expr), expected);
-
// !c3 ^ c3 --> -1
let expr = (-col("c3_non_null")) ^ col("c3_non_null");
let expected = lit(-1i64);
diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs
index b0908b4760..89bb762d59 100644
--- a/datafusion/optimizer/src/simplify_expressions/utils.rs
+++ b/datafusion/optimizer/src/simplify_expressions/utils.rs
@@ -17,7 +17,6 @@
//! Utility functions for expression simplification
-use arrow::datatypes::i256;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
Case, Expr, Like, Operator,
@@ -25,47 +24,6 @@ use datafusion_expr::{
expr_fn::{and, bitwise_and, bitwise_or, or},
};
-pub static POWS_OF_TEN: [i128; 38] = [
- 1,
- 10,
- 100,
- 1000,
- 10000,
- 100000,
- 1000000,
- 10000000,
- 100000000,
- 1000000000,
- 10000000000,
- 100000000000,
- 1000000000000,
- 10000000000000,
- 100000000000000,
- 1000000000000000,
- 10000000000000000,
- 100000000000000000,
- 1000000000000000000,
- 10000000000000000000,
- 100000000000000000000,
- 1000000000000000000000,
- 10000000000000000000000,
- 100000000000000000000000,
- 1000000000000000000000000,
- 10000000000000000000000000,
- 100000000000000000000000000,
- 1000000000000000000000000000,
- 10000000000000000000000000000,
- 100000000000000000000000000000,
- 1000000000000000000000000000000,
- 10000000000000000000000000000000,
- 100000000000000000000000000000000,
- 1000000000000000000000000000000000,
- 10000000000000000000000000000000000,
- 100000000000000000000000000000000000,
- 1000000000000000000000000000000000000,
- 10000000000000000000000000000000000000,
-];
-
/// returns true if `needle` is found in a chain of search_op
/// expressions. Such as: (A AND B) AND C
fn expr_contains_inner(expr: &Expr, needle: &Expr, search_op: Operator) -> bool {
@@ -139,54 +97,26 @@ pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) ->
}
pub fn is_zero(s: &Expr) -> bool {
- match s {
- Expr::Literal(ScalarValue::Int8(Some(0)), _)
- | Expr::Literal(ScalarValue::Int16(Some(0)), _)
- | Expr::Literal(ScalarValue::Int32(Some(0)), _)
- | Expr::Literal(ScalarValue::Int64(Some(0)), _)
- | Expr::Literal(ScalarValue::UInt8(Some(0)), _)
- | Expr::Literal(ScalarValue::UInt16(Some(0)), _)
- | Expr::Literal(ScalarValue::UInt32(Some(0)), _)
- | Expr::Literal(ScalarValue::UInt64(Some(0)), _) => true,
- Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 0. => true,
- Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 0. => true,
- Expr::Literal(ScalarValue::Decimal128(Some(v), _p, _s), _) if *v == 0 => true,
- Expr::Literal(ScalarValue::Decimal256(Some(v), _p, _s), _)
- if *v == i256::ZERO =>
- {
- true
- }
- _ => false,
+ if let Expr::Literal(sv, _) = s
+ && sv.data_type().is_numeric()
+ {
+ // unwrap safe since numeric types always have a 0 value
+ sv == &ScalarValue::new_zero(&sv.data_type()).unwrap()
+ } else {
+ false
}
}
pub fn is_one(s: &Expr) -> bool {
- match s {
- Expr::Literal(ScalarValue::Int8(Some(1)), _)
- | Expr::Literal(ScalarValue::Int16(Some(1)), _)
- | Expr::Literal(ScalarValue::Int32(Some(1)), _)
- | Expr::Literal(ScalarValue::Int64(Some(1)), _)
- | Expr::Literal(ScalarValue::UInt8(Some(1)), _)
- | Expr::Literal(ScalarValue::UInt16(Some(1)), _)
- | Expr::Literal(ScalarValue::UInt32(Some(1)), _)
- | Expr::Literal(ScalarValue::UInt64(Some(1)), _) => true,
- Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 1. => true,
- Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 1. => true,
- Expr::Literal(ScalarValue::Decimal128(Some(v), _p, s), _) => {
- *s >= 0
- && POWS_OF_TEN
- .get(*s as usize)
- .map(|x| x == v)
- .unwrap_or_default()
- }
- Expr::Literal(ScalarValue::Decimal256(Some(v), _p, s), _) => {
- *s >= 0
- && match i256::from(10).checked_pow(*s as u32) {
- Some(res) => res == *v,
- None => false,
- }
- }
- _ => false,
+ if let Expr::Literal(sv, _) = s
+ && sv.data_type().is_numeric()
+ // there are edge cases like negative scale decimals not being able to
+ // create a one value so this can fail
+ && let Ok(one) = ScalarValue::new_one(&sv.data_type())
+ {
+ sv == &one
+ } else {
+ false
}
}