Skip to content

Apache DataFusion SQL Query Engine

Revert #25790 (Decimal filter selectivity) due to TPC-H Q18 regression (#27145)

Alessandro Solimando8460f4f92c15
diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs
index 23e2e2f6ca..6aa59559b8 100644
--- a/datafusion/expr-common/src/interval_arithmetic.rs
+++ b/datafusion/expr-common/src/interval_arithmetic.rs
@@ -14,10 +14,8 @@ use crate::type_coercion::binary::{BinaryTypeCoercer, comparison_coercion};
use arrow::compute::{CastOptions, cast_with_options};
use arrow::datatypes::{
DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit,
- MAX_DECIMAL32_FOR_EACH_PRECISION, MAX_DECIMAL64_FOR_EACH_PRECISION,
MAX_DECIMAL128_FOR_EACH_PRECISION, MAX_DECIMAL256_FOR_EACH_PRECISION,
- MIN_DECIMAL32_FOR_EACH_PRECISION, MIN_DECIMAL64_FOR_EACH_PRECISION,
- MIN_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL256_FOR_EACH_PRECISION, TimeUnit, i256,
+ MIN_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL256_FOR_EACH_PRECISION, TimeUnit,
};
use datafusion_common::rounding::{alter_fp_rounding_mode, next_down, next_up};
use datafusion_common::{
@@ -26,14 +24,7 @@ use datafusion_common::{
};
macro_rules! get_extreme_value {
- (
- $extreme:ident,
- $DECIMAL32_ARRAY:ident,
- $DECIMAL64_ARRAY:ident,
- $DECIMAL128_ARRAY:ident,
- $DECIMAL256_ARRAY:ident,
- $value:expr
- ) => {
+ ($extreme:ident, $DECIMAL128_ARRAY:ident, $DECIMAL256_ARRAY:ident, $value:expr) => {
match $value {
DataType::UInt8 => ScalarValue::UInt8(Some(u8::$extreme)),
DataType::UInt16 => ScalarValue::UInt16(Some(u16::$extreme)),
@@ -80,16 +71,6 @@ macro_rules! get_extreme_value {
DataType::Interval(IntervalUnit::MonthDayNano) => {
ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::$extreme))
}
- DataType::Decimal32(precision, scale) => ScalarValue::Decimal32(
- Some($DECIMAL32_ARRAY[*precision as usize]),
- *precision,
- *scale,
- ),
- DataType::Decimal64(precision, scale) => ScalarValue::Decimal64(
- Some($DECIMAL64_ARRAY[*precision as usize]),
- *precision,
- *scale,
- ),
DataType::Decimal128(precision, scale) => ScalarValue::Decimal128(
Some($DECIMAL128_ARRAY[*precision as usize]),
*precision,
@@ -106,15 +87,7 @@ macro_rules! get_extreme_value {
}
macro_rules! value_transition {
- (
- $bound:ident,
- $direction:expr,
- $DECIMAL32_ARRAY:ident,
- $DECIMAL64_ARRAY:ident,
- $DECIMAL128_ARRAY:ident,
- $DECIMAL256_ARRAY:ident,
- $value:expr
- ) => {
+ ($bound:ident, $direction:expr, $value:expr) => {
match $value {
UInt8(Some(value)) if value == u8::$bound => UInt8(None),
UInt16(Some(value)) if value == u16::$bound => UInt16(None),
@@ -161,28 +134,6 @@ macro_rules! value_transition {
{
IntervalMonthDayNano(None)
}
- // Decimals saturate at the extreme value of their precision, not of
- // the underlying integer type:
- Decimal32(Some(value), precision, scale)
- if value == $DECIMAL32_ARRAY[precision as usize] =>
- {
- Decimal32(None, precision, scale)
- }
- Decimal64(Some(value), precision, scale)
- if value == $DECIMAL64_ARRAY[precision as usize] =>
- {
- Decimal64(None, precision, scale)
- }
- Decimal128(Some(value), precision, scale)
- if value == $DECIMAL128_ARRAY[precision as usize] =>
- {
- Decimal128(None, precision, scale)
- }
- Decimal256(Some(value), precision, scale)
- if value == $DECIMAL256_ARRAY[precision as usize] =>
- {
- Decimal256(None, precision, scale)
- }
_ => next_value_helper::<$direction>($value),
}
};
@@ -1244,8 +1195,6 @@ fn handle_overflow<const UPPER: bool>(
(true, false) => {
get_extreme_value!(
MIN,
- MIN_DECIMAL32_FOR_EACH_PRECISION,
- MIN_DECIMAL64_FOR_EACH_PRECISION,
MIN_DECIMAL128_FOR_EACH_PRECISION,
MIN_DECIMAL256_FOR_EACH_PRECISION,
dt
@@ -1254,8 +1203,6 @@ fn handle_overflow<const UPPER: bool>(
(false, true) => {
get_extreme_value!(
MAX,
- MAX_DECIMAL32_FOR_EACH_PRECISION,
- MAX_DECIMAL64_FOR_EACH_PRECISION,
MAX_DECIMAL128_FOR_EACH_PRECISION,
MAX_DECIMAL256_FOR_EACH_PRECISION,
dt
@@ -1268,30 +1215,14 @@ fn handle_overflow<const UPPER: bool>(
// used without caution.
fn next_value(value: ScalarValue) -> ScalarValue {
use ScalarValue::*;
- value_transition!(
- MAX,
- true,
- MAX_DECIMAL32_FOR_EACH_PRECISION,
- MAX_DECIMAL64_FOR_EACH_PRECISION,
- MAX_DECIMAL128_FOR_EACH_PRECISION,
- MAX_DECIMAL256_FOR_EACH_PRECISION,
- value
- )
+ value_transition!(MAX, true, value)
}
// This function should remain private since it may corrupt the an interval if
// used without caution.
fn prev_value(value: ScalarValue) -> ScalarValue {
use ScalarValue::*;
- value_transition!(
- MIN,
- false,
- MIN_DECIMAL32_FOR_EACH_PRECISION,
- MIN_DECIMAL64_FOR_EACH_PRECISION,
- MIN_DECIMAL128_FOR_EACH_PRECISION,
- MIN_DECIMAL256_FOR_EACH_PRECISION,
- value
- )
+ value_transition!(MIN, false, value)
}
trait OneTrait: Sized + std::ops::Add + std::ops::Sub {
@@ -1302,12 +1233,6 @@ macro_rules! impl_OneTrait{
}
impl_OneTrait! {u8, u16, u32, u64, i8, i16, i32, i64, i128}
-impl OneTrait for i256 {
- fn one() -> Self {
- i256::ONE
- }
-}
-
impl OneTrait for IntervalDayTime {
fn one() -> Self {
IntervalDayTime {
@@ -1362,23 +1287,6 @@ fn next_value_helper<const INC: bool>(value: ScalarValue) -> ScalarValue {
UInt16(Some(val)) => UInt16(Some(increment_decrement::<INC, u16>(val))),
UInt32(Some(val)) => UInt32(Some(increment_decrement::<INC, u32>(val))),
UInt64(Some(val)) => UInt64(Some(increment_decrement::<INC, u64>(val))),
- // Decimals step by one unit in the last place, keeping precision and scale:
- Decimal32(Some(val), precision, scale) => {
- Decimal32(Some(increment_decrement::<INC, i32>(val)), precision, scale)
- }
- Decimal64(Some(val), precision, scale) => {
- Decimal64(Some(increment_decrement::<INC, i64>(val)), precision, scale)
- }
- Decimal128(Some(val), precision, scale) => Decimal128(
- Some(increment_decrement::<INC, i128>(val)),
- precision,
- scale,
- ),
- Decimal256(Some(val), precision, scale) => Decimal256(
- Some(increment_decrement::<INC, i256>(val)),
- precision,
- scale,
- ),
DurationSecond(Some(val)) => {
DurationSecond(Some(increment_decrement::<INC, i64>(val)))
}
@@ -2360,10 +2268,7 @@ mod tests {
use crate::interval_arithmetic::NullableInterval;
use arrow::compute::CastOptions;
- use arrow::datatypes::{
- DataType, MAX_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION,
- i256,
- };
+ use arrow::datatypes::DataType;
use datafusion_common::rounding::{next_down, next_up};
use datafusion_common::{Result, ScalarValue};
@@ -2676,124 +2581,6 @@ mod tests {
Ok(())
}
- #[test]
- fn test_next_prev_value_decimal() {
- use ScalarValue::*;
-
- // Decimals step by one unit in the last place, keeping precision and scale:
- let cases = vec![
- (Decimal32(Some(2400), 9, 2), Decimal32(Some(2401), 9, 2)),
- (Decimal64(Some(2400), 18, 2), Decimal64(Some(2401), 18, 2)),
- (Decimal128(Some(2400), 15, 2), Decimal128(Some(2401), 15, 2)),
- (
- Decimal256(Some(i256::from(2400)), 40, 2),
- Decimal256(Some(i256::from(2401)), 40, 2),
- ),
- (Decimal128(Some(-1), 3, 2), Decimal128(Some(0), 3, 2)),
- ];
- for (value, next) in cases {
- assert_eq!(next_value(value.clone()), next);
- assert_eq!(prev_value(next), value);
- }
-
- // Values at the extremes of their precision (e.g. -9.99 and 9.99 for
- // precision 3, scale 2) become unbounded:
- let d128_max = MAX_DECIMAL128_FOR_EACH_PRECISION[38];
- let d128_min = MIN_DECIMAL128_FOR_EACH_PRECISION[38];
- let cases = vec![
- (
- Decimal32(Some(-999), 3, 2),
- Decimal32(Some(999), 3, 2),
- Decimal32(Some(-998), 3, 2),
- Decimal32(Some(998), 3, 2),
- Decimal32(None, 3, 2),
- ),
- (
- Decimal64(Some(-999), 3, 2),
- Decimal64(Some(999), 3, 2),
- Decimal64(Some(-998), 3, 2),
- Decimal64(Some(998), 3, 2),
- Decimal64(None, 3, 2),
- ),
- (
- Decimal128(Some(-999), 3, 2),
- Decimal128(Some(999), 3, 2),
- Decimal128(Some(-998), 3, 2),
- Decimal128(Some(998), 3, 2),
- Decimal128(None, 3, 2),
- ),
- (
- Decimal256(Some(i256::from(-999)), 3, 2),
- Decimal256(Some(i256::from(999)), 3, 2),
- Decimal256(Some(i256::from(-998)), 3, 2),
- Decimal256(Some(i256::from(998)), 3, 2),
- Decimal256(None, 3, 2),
- ),
- (
- Decimal128(Some(d128_min), 38, 0),
- Decimal128(Some(d128_max), 38, 0),
- Decimal128(Some(d128_min + 1), 38, 0),
- Decimal128(Some(d128_max - 1), 38, 0),
- Decimal128(None, 38, 0),
- ),
- ];
- for (min, max, after_min, before_max, inf) in cases {
- assert_eq!(next_value(max.clone()), inf);
- assert_eq!(prev_value(max), before_max);
-
- assert_eq!(prev_value(min.clone()), inf);
- assert_eq!(next_value(min), after_min);
-
- assert_eq!(next_value(inf.clone()), inf);
- assert_eq!(prev_value(inf.clone()), inf);
- }
- }
-
- #[test]
- fn test_satisfy_strict_comparison_decimal() -> Result<()> {
- use ScalarValue::*;
-
- // x in [0.00, 50.00], constraint 24.00 > x tightens x to [0.00, 23.99]:
- let x =
- Interval::try_new(Decimal128(Some(0), 15, 2), Decimal128(Some(5000), 15, 2))?;
- let literal = Interval::try_new(
- Decimal128(Some(2400), 15, 2),
- Decimal128(Some(2400), 15, 2),
- )?;
- let (_, x_lt) = satisfy_greater(&literal, &x, true)?.unwrap();
- assert_eq!(
- x_lt,
- Interval::try_new(Decimal128(Some(0), 15, 2), Decimal128(Some(2399), 15, 2))?
- );
-
- // Constraint x > 24.00 tightens x to [24.01, 50.00]:
- let (x_gt, _) = satisfy_greater(&x, &literal, true)?.unwrap();
- assert_eq!(
- x_gt,
- Interval::try_new(
- Decimal128(Some(2401), 15, 2),
- Decimal128(Some(5000), 15, 2),
- )?
- );
-
- // Non-strict comparisons keep the closed bound:
- let (_, x_le) = satisfy_greater(&literal, &x, false)?.unwrap();
- assert_eq!(
- x_le,
- Interval::try_new(Decimal128(Some(0), 15, 2), Decimal128(Some(2400), 15, 2))?
- );
-
- // A strict bound at the precision's extreme conservatively becomes
- // unbounded instead of stepping past it (x > 9.99 for Decimal128(3, 2)):
- let x = Interval::make_unbounded(&DataType::Decimal128(3, 2))?;
- let max =
- Interval::try_new(Decimal128(Some(999), 3, 2), Decimal128(Some(999), 3, 2))?;
- let (x_gt, _) = satisfy_greater(&x, &max, true)?.unwrap();
- assert_eq!(x_gt, Interval::make_unbounded(&DataType::Decimal128(3, 2))?);
-
- Ok(())
- }
-
#[test]
fn test_new_interval() -> Result<()> {
use ScalarValue::*;
@@ -4607,24 +4394,6 @@ mod tests {
ScalarValue::Decimal128(Some(99999999999999999999999999999999999999), 38, 35)
);
- // Test Decimal32/Decimal64 underflow handling:
- let op = Operator::Minus;
- let dt = DataType::Decimal32(3, 2);
- let lhs = ScalarValue::Decimal32(Some(-999), 3, 2);
- let rhs = ScalarValue::Decimal32(Some(1), 3, 2);
- let result = handle_overflow::<true>(&dt, op, &lhs, &rhs);
- assert_eq!(result, ScalarValue::Decimal32(Some(-999), 3, 2));
- let result = handle_overflow::<false>(&dt, op, &lhs, &rhs);
- assert_eq!(result, ScalarValue::Decimal32(None, 3, 2));
-
- let dt = DataType::Decimal64(3, 2);
- let lhs = ScalarValue::Decimal64(Some(-999), 3, 2);
- let rhs = ScalarValue::Decimal64(Some(1), 3, 2);
- let result = handle_overflow::<true>(&dt, op, &lhs, &rhs);
- assert_eq!(result, ScalarValue::Decimal64(Some(-999), 3, 2));
- let result = handle_overflow::<false>(&dt, op, &lhs, &rhs);
- assert_eq!(result, ScalarValue::Decimal64(None, 3, 2));
-
Ok(())
}
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
index e56bf4dcbc..7fbed9187b 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -699,28 +699,6 @@ fn can_propagate_cast_constraints(source: &DataType, target: &DataType) -> bool
|| (source.is_integer() && target.is_integer())
// NaN bounds are unbounded; finite Float32 values widen exactly.
|| (*source == Float32 && *target == Float64)
- || is_decimal_widening(source, target)
-}
-
-/// Whether a decimal-to-decimal cast keeps every value exactly, i.e. the target
-/// has at least as many fractional and integer digits as the source. Casting
-/// bounds back to the source then rounds to a neighboring representable value,
-/// which never excludes a valid input.
-fn is_decimal_widening(source: &DataType, target: &DataType) -> bool {
- fn precision_scale(data_type: &DataType) -> Option<(i16, i16)> {
- match data_type {
- Decimal32(p, s) | Decimal64(p, s) | Decimal128(p, s) | Decimal256(p, s) => {
- Some((*p as i16, *s as i16))
- }
- _ => None,
- }
- }
- match (precision_scale(source), precision_scale(target)) {
- (Some((source_p, source_s)), Some((target_p, target_s))) => {
- target_s >= source_s && target_p - target_s >= source_p - source_s
- }
- _ => false,
- }
}
#[cfg(feature = "proto")]
@@ -942,11 +920,6 @@ mod tests {
(Int32, Float64, true),
(Float32, Float64, true),
(Decimal128(4, 1), Decimal128(4, 1), true),
- (Decimal128(4, 1), Decimal128(6, 2), true),
- (Decimal128(15, 2), Decimal128(30, 15), true),
- (Decimal32(9, 2), Decimal128(20, 4), true),
- (Decimal128(6, 2), Decimal128(6, 3), false),
- (Decimal128(6, 2), Decimal128(8, 1), false),
] {
let schema = Schema::new(vec![Field::new("x", source.clone(), true)]);
let expr = CastExpr::new(col("x", &schema)?, target.clone(), None);
@@ -967,76 +940,6 @@ mod tests {
Ok(())
}
- #[test]
- fn test_decimal_widening_constraint_boundaries() -> Result<()> {
- fn decimal(data_type: &DataType, value: i128) -> ScalarValue {
- match data_type {
- Decimal32(p, s) => ScalarValue::Decimal32(Some(value as i32), *p, *s),
- Decimal64(p, s) => ScalarValue::Decimal64(Some(value as i64), *p, *s),
- Decimal128(p, s) => ScalarValue::Decimal128(Some(value), *p, *s),
- Decimal256(p, s) => {
- ScalarValue::Decimal256(Some(i256::from_i128(value)), *p, *s)
- }
- _ => unreachable!(),
- }
- }
-
- // (source, target, scale factor, extreme unscaled source value)
- for (source, target, factor, extreme) in [
- (Decimal128(3, 2), Decimal128(5, 3), 10, 999),
- // Cross-width widening:
- (Decimal32(4, 2), Decimal128(10, 4), 100, 9999),
- (Decimal64(5, 1), Decimal256(10, 3), 100, 99999),
- // Negative scale:
- (Decimal128(3, -1), Decimal128(5, 0), 10, 999),
- ] {
- // Source values near zero and near the extremes of the precision:
- let values: Vec<i128> = (-5..=5)
- .chain(extreme - 5..=extreme)
- .chain(-extreme..=-extreme + 5)
- .collect();
- // Output bounds at, between, and just past the widened values,
- // including past the source's extremes, so casting back rounds in
- // both directions and overflows:
- let mut bounds: Vec<i128> = values
- .iter()
- .flat_map(|v| {
- let w = v * factor;
- [w - factor, w - 1, w, w + 1, w + factor / 2, w + factor]
- })
- .collect();
- bounds.sort_unstable();
- bounds.dedup();
-
- let schema =
- Arc::new(Schema::new(vec![Field::new("x", source.clone(), false)]));
- let expr = CastExpr::new(col("x", &schema)?, target.clone(), None);
- let input = Interval::make_unbounded(&source)?;
- for (i, lower) in bounds.iter().enumerate() {
- for upper in &bounds[i..] {
- let output = Interval::try_new(
- decimal(&target, *lower),
- decimal(&target, *upper),
- )?;
- let propagated =
- expr.propagate_constraints(&output, &[&input])?.unwrap();
- assert_eq!(propagated.len(), 1);
- for value in &values {
- if output.contains_value(decimal(&target, value * factor))? {
- assert!(
- propagated[0].contains_value(decimal(&source, *value))?,
- "{source} -> {target}: input={value}, output={output}, \
- propagated={:?}",
- propagated[0]
- );
- }
- }
- }
- }
- }
- Ok(())
- }
-
#[test]
fn test_float_widening_constraint_boundaries() -> Result<()> {
let mut values = vec![
diff --git a/datafusion/physical-expr/src/intervals/utils.rs b/datafusion/physical-expr/src/intervals/utils.rs
index 8f93e49c22..35b1ee6ffd 100644
--- a/datafusion/physical-expr/src/intervals/utils.rs
+++ b/datafusion/physical-expr/src/intervals/utils.rs
@@ -23,57 +23,19 @@ use datafusion_expr::interval_arithmetic::Interval;
/// will relax as more types of `PhysicalExpr`s and `Operator`s are supported.
/// Currently, [`CastExpr`], [`NegativeExpr`], [`BinaryExpr`], [`Column`] and [`Literal`] are supported.
pub fn check_support(expr: &Arc<dyn PhysicalExpr>, schema: &SchemaRef) -> bool {
- check_support_with(expr, schema, is_datatype_supported)
-}
-
-/// Like [`check_support`], but additionally accepts decimal types. This is
-/// used when estimating filter statistics, where decimal predicates can be
-/// analyzed with interval arithmetic. Other users of [`check_support`], such
-/// as symmetric hash join pruning, do not support decimals yet.
-pub fn check_statistics_support(
- expr: &Arc<dyn PhysicalExpr>,
- schema: &SchemaRef,
-) -> bool {
- check_support_with(expr, schema, |data_type| {
- is_datatype_supported(data_type)
- || matches!(
- data_type,
- DataType::Decimal32(_, _)
- | DataType::Decimal64(_, _)
- | DataType::Decimal128(_, _)
- | DataType::Decimal256(_, _)
- )
- })
-}
-
-fn check_support_with(
- expr: &Arc<dyn PhysicalExpr>,
- schema: &SchemaRef,
- is_type_supported: fn(&DataType) -> bool,
-) -> bool {
if let Some(binary_expr) = expr.downcast_ref::<BinaryExpr>() {
- // Arithmetic is only analyzed over the types `check_support` accepts:
- // decimal arithmetic changes the precision and scale of its result, and
- // propagating constraints back to its operands would not restore their
- // types.
- let is_type_supported: fn(&DataType) -> bool =
- if binary_expr.op().is_numerical_operators() {
- is_datatype_supported
- } else {
- is_type_supported
- };
is_operator_supported(binary_expr.op())
- && check_support_with(binary_expr.left(), schema, is_type_supported)
- && check_support_with(binary_expr.right(), schema, is_type_supported)
+ && check_support(binary_expr.left(), schema)
+ && check_support(binary_expr.right(), schema)
} else if let Some(column) = expr.downcast_ref::<Column>() {
if let Ok(field) = schema.field_with_name(column.name()) {
- is_type_supported(field.data_type())
+ is_datatype_supported(field.data_type())
} else {
false
}
} else if let Some(literal) = expr.downcast_ref::<Literal>() {
if let Ok(dt) = literal.data_type(schema) {
- is_type_supported(&dt)
+ is_datatype_supported(&dt)
} else {
false
}
@@ -82,10 +44,9 @@ fn check_support_with(
// interval endpoints to a type with a different ordering (e.g. integers
// to strings) does not preserve the interval, so such expressions are
// left unanalyzed rather than producing bogus bounds.
- is_type_supported(cast.cast_type())
- && check_support_with(cast.expr(), schema, is_type_supported)
+ is_datatype_supported(cast.cast_type()) && check_support(cast.expr(), schema)
} else if let Some(negative) = expr.downcast_ref::<NegativeExpr>() {
- check_support_with(negative.arg(), schema, is_type_supported)
+ check_support(negative.arg(), schema)
} else {
false
}
@@ -223,64 +184,3 @@ fn interval_dt_to_duration_ms(dt: &IntervalDayTime) -> Result<i64> {
)
}
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- use arrow::datatypes::{Field, Schema};
-
- #[test]
- fn check_statistics_support_decimal() {
- for data_type in [
- DataType::Decimal32(9, 2),
- DataType::Decimal64(18, 2),
- DataType::Decimal128(15, 2),
- DataType::Decimal256(40, 2),
- ] {
- let schema =
- Arc::new(Schema::new(vec![Field::new("a", data_type.clone(), false)]));
- let literal = ScalarValue::try_from(&data_type).unwrap();
- let expr = Arc::new(BinaryExpr::new(
- Arc::new(Column::new("a", 0)),
- Operator::Lt,
- Arc::new(Literal::new(literal)),
- )) as Arc<dyn PhysicalExpr>;
-
- assert!(check_statistics_support(&expr, &schema));
- // Decimal support is limited to statistics estimation for now:
- assert!(!check_support(&expr, &schema));
-
- // Decimal arithmetic is not analyzed, since it changes the precision
- // and scale of its operands:
- let literal = ScalarValue::try_from(&data_type).unwrap();
- for op in [Operator::Plus, Operator::Minus, Operator::Multiply] {
- let arithmetic = Arc::new(BinaryExpr::new(
- Arc::new(Column::new("a", 0)),
- op,
- Arc::new(Literal::new(literal.clone())),
- )) as Arc<dyn PhysicalExpr>;
- let expr = Arc::new(BinaryExpr::new(
- arithmetic,
- Operator::Gt,
- Arc::new(Literal::new(literal.clone())),
- )) as Arc<dyn PhysicalExpr>;
- assert!(!check_statistics_support(&expr, &schema), "{op}");
- }
- }
-
- // Arithmetic over other supported types is unaffected:
- let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
- let expr = Arc::new(BinaryExpr::new(
- Arc::new(BinaryExpr::new(
- Arc::new(Column::new("a", 0)),
- Operator::Plus,
- Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),
- )),
- Operator::Gt,
- Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),
- )) as Arc<dyn PhysicalExpr>;
- assert!(check_statistics_support(&expr, &schema));
- assert!(check_support(&expr, &schema));
- }
-}
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 037cf65008..0e8cf991cf 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -53,7 +53,7 @@ use datafusion_physical_expr::equivalence::ProjectionMapping;
use datafusion_physical_expr::expressions::{
BinaryExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, LikeExpr, Literal, lit,
};
-use datafusion_physical_expr::intervals::utils::check_statistics_support;
+use datafusion_physical_expr::intervals::utils::check_support;
use datafusion_physical_expr::utils::collect_columns;
use datafusion_physical_expr::{
AcrossPartitions, AnalysisContext, ConstExpr, ExprBoundaries, PhysicalExpr, analyze,
@@ -389,7 +389,7 @@ impl FilterExec {
} else {
let null_rejecting_columns = collect_null_rejecting_columns(predicate);
- if check_statistics_support(predicate, schema) {
+ if check_support(predicate, schema) {
let input_analysis_ctx = AnalysisContext::try_from_statistics(
schema,
&input_stats.column_statistics,
@@ -2225,199 +2225,6 @@ mod tests {
Ok(())
}
- #[tokio::test]
- async fn test_filter_statistics_decimal_expr() -> Result<()> {
- // Table:
- // a: Decimal128(15, 2), min=0.00, max=0.10
- let decimal = |v: i128| ScalarValue::Decimal128(Some(v), 15, 2);
- let schema =
- Schema::new(vec![Field::new("a", DataType::Decimal128(15, 2), false)]);
- let input = Arc::new(StatisticsExec::new(
- Statistics {
- num_rows: Precision::Inexact(1100),
- total_byte_size: Precision::Absent,
- column_statistics: vec![ColumnStatistics {
- min_value: Precision::Inexact(decimal(0)),
- max_value: Precision::Inexact(decimal(10)),
- ..Default::default()
- }],
- },
- schema.clone(),
- ));
-
- // a >= 0.05 AND a < 0.08
- let predicate: Arc<dyn PhysicalExpr> = binary(
- binary(col("a", &schema)?, Operator::GtEq, lit(decimal(5)), &schema)?,
- Operator::And,
- binary(col("a", &schema)?, Operator::Lt, lit(decimal(8)), &schema)?,
- &schema,
- )?;
-
- // WHERE a >= 0.05 AND a < 0.08
- let filter: Arc<dyn ExecutionPlan> =
- Arc::new(FilterExec::try_new(predicate, input)?);
-
- // The strict bound tightens to 0.07, so [0.05, 0.07] keeps 3 of the 11
- // representable values in [0.00, 0.10].
- let statistics =
- StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
- assert_eq!(statistics.num_rows, Precision::Inexact(300));
- assert_eq!(
- statistics.column_statistics,
- vec![ColumnStatistics {
- null_count: Precision::Exact(0),
- min_value: Precision::Inexact(decimal(5)),
- max_value: Precision::Inexact(decimal(7)),
- ..Default::default()
- }]
- );
-
- Ok(())
- }
-
- /// Arithmetic changes the precision and scale of decimal results, so it must
- /// not narrow decimal columns: these predicates fall back to the default
- /// selectivity and keep the input bounds in the column's own type.
- #[tokio::test]
- async fn test_filter_statistics_decimal_arithmetic_expr() -> Result<()> {
- // Table:
- // a: Decimal128(15, 2), min=1.00, max=10.00
- let decimal = |v: i128| ScalarValue::Decimal128(Some(v), 15, 2);
- let schema =
- Schema::new(vec![Field::new("a", DataType::Decimal128(15, 2), false)]);
- let input_stats = Statistics {
- num_rows: Precision::Inexact(1000),
- total_byte_size: Precision::Absent,
- column_statistics: vec![ColumnStatistics {
- min_value: Precision::Inexact(decimal(100)),
- max_value: Precision::Inexact(decimal(1000)),
- ..Default::default()
- }],
- };
-
- // a * 2.00 > 5.0000 and a - 2.00 > 5.00
- for (op, rhs) in [
- (Operator::Multiply, 50_000_i128), // 5.0000 at the product's scale 4
- (Operator::Minus, 500_i128), // 5.00 at the difference's scale 2
- ] {
- let arithmetic =
- Arc::new(BinaryExpr::new(col("a", &schema)?, op, lit(decimal(200))))
- as Arc<dyn PhysicalExpr>;
- let DataType::Decimal128(precision, scale) = arithmetic.data_type(&schema)?
- else {
- unreachable!()
- };
- let predicate = Arc::new(BinaryExpr::new(
- arithmetic,
- Operator::Gt,
- lit(ScalarValue::Decimal128(Some(rhs), precision, scale)),
- ));
- let input =
- Arc::new(StatisticsExec::new(input_stats.clone(), schema.clone()));
- let filter: Arc<dyn ExecutionPlan> =
- Arc::new(FilterExec::try_new(predicate, input)?);
-
- let statistics = StatisticsContext::new()
- .compute(filter.as_ref(), &StatisticsArgs::new())?;
- assert_eq!(statistics.num_rows, Precision::Inexact(200), "{op}");
- let column = &statistics.column_statistics[0];
- assert_eq!(column.min_value, Precision::Inexact(decimal(100)), "{op}");
- assert_eq!(column.max_value, Precision::Inexact(decimal(1000)), "{op}");
- for bound in [&column.min_value, &column.max_value] {
- let value = bound.get_value().unwrap();
- assert_eq!(value.data_type(), DataType::Decimal128(15, 2), "{op}");
- }
- }
-
- // Negation keeps the decimal type, so -a >= -5.00 still narrows `a` to
- // [1.00, 5.00]: 401 of the 901 representable values in [1.00, 10.00],
- // rounded up.
- let predicate = Arc::new(BinaryExpr::new(
- Arc::new(NegativeExpr::new(col("a", &schema)?)),
- Operator::GtEq,
- lit(decimal(-500)),
- ));
- let input = Arc::new(StatisticsExec::new(input_stats, schema.clone()));
- let filter: Arc<dyn ExecutionPlan> =
- Arc::new(FilterExec::try_new(predicate, input)?);
- let statistics =
- StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
- assert_eq!(statistics.num_rows, Precision::Inexact(446));
- let column = &statistics.column_statistics[0];
- assert_eq!(column.min_value, Precision::Inexact(decimal(100)));
- assert_eq!(column.max_value, Precision::Inexact(decimal(500)));
-
- Ok(())
- }
-
- #[tokio::test]
- async fn test_filter_statistics_decimal_mixed_scale_expr() -> Result<()> {
- // Table:
- // a: Decimal128(15, 2), min=0.00, max=0.10
- let schema =
- Schema::new(vec![Field::new("a", DataType::Decimal128(15, 2), false)]);
- let input = Arc::new(StatisticsExec::new(
- Statistics {
- num_rows: Precision::Inexact(1100),
- total_byte_size: Precision::Absent,
- column_statistics: vec![ColumnStatistics {
- min_value: Precision::Inexact(ScalarValue::Decimal128(
- Some(0),
- 15,
- 2,
- )),
- max_value: Precision::Inexact(ScalarValue::Decimal128(
- Some(10),
- 15,
- 2,
- )),
- ..Default::default()
- }],
- },
- schema.clone(),
- ));
-
- // Type coercion compares `a` with the float literals as Decimal128(30, 15),
- // so the column is cast to the literals' scale:
- // CAST(a AS Decimal128(30, 15)) >= 0.055 AND CAST(a AS Decimal128(30, 15)) < 0.085
- let coerced = DataType::Decimal128(30, 15);
- let literal = |v: i128| lit(ScalarValue::Decimal128(Some(v), 30, 15));
- let predicate: Arc<dyn PhysicalExpr> = binary(
- binary(
- cast(col("a", &schema)?, &schema, coerced.clone())?,
- Operator::GtEq,
- literal(55_000_000_000_000),
- &schema,
- )?,
- Operator::And,
- binary(
- cast(col("a", &schema)?, &schema, coerced)?,
- Operator::Lt,
- literal(85_000_000_000_000),
- &schema,
- )?,
- &schema,
- )?;
- let filter: Arc<dyn ExecutionPlan> =
- Arc::new(FilterExec::try_new(predicate, input)?);
-
- let statistics =
- StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;
- // The bounds are cast back to scale 2 as [0.06, 0.08], keeping 3 of the
- // 11 representable values in [0.00, 0.10].
- assert_eq!(statistics.num_rows, Precision::Inexact(300));
- assert_eq!(
- statistics.column_statistics,
- vec![ColumnStatistics {
- min_value: Precision::Inexact(ScalarValue::Decimal128(Some(6), 15, 2)),
- max_value: Precision::Inexact(ScalarValue::Decimal128(Some(8), 15, 2)),
- ..Default::default()
- }]
- );
-
- Ok(())
- }
-
#[tokio::test]
async fn test_filter_statistics_column_level_nested() -> Result<()> {
// Table:
@@ -3061,9 +2868,9 @@ mod tests {
#[tokio::test]
async fn test_custom_filter_selectivity() -> Result<()> {
- // Need a type unsupported by interval analysis to trigger the default
- // selectivity
- let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]);
+ // Need a decimal to trigger inexact selectivity
+ let schema =
+ Schema::new(vec![Field::new("a", DataType::Decimal128(2, 3), false)]);
let input = Arc::new(StatisticsExec::new(
Statistics {
num_rows: Precision::Inexact(1000),
@@ -3072,11 +2879,11 @@ mod tests {
},
schema,
));
- // WHERE a = '10'
+ // WHERE a = 10
let predicate = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
Operator::Eq,
- Arc::new(Literal::new(ScalarValue::from("10"))),
+ Arc::new(Literal::new(ScalarValue::Decimal128(Some(10), 10, 10))),
));
let filter = FilterExec::try_new(predicate, input)?;
let statistics =
diff --git a/datafusion/sqllogictest/test_files/filter_statistics_decimal.slt b/datafusion/sqllogictest/test_files/filter_statistics_decimal.slt
deleted file mode 100644
index 09e525076f..0000000000
--- a/datafusion/sqllogictest/test_files/filter_statistics_decimal.slt
+++ /dev/null
@@ -1,91 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
-# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
-
-# Tests for filter selectivity estimates on decimal columns, using min/max
-# statistics from a parquet file.
-#
-# l_discount cycles through 0.00..0.10 and l_quantity through 1.00..50.00. Since
-# lcm(11, 50) = 550, each (l_discount, l_quantity) pair appears exactly twice
-# in the 1100 rows.
-
-query I
-COPY (
- SELECT
- CAST((value % 11) / 100.0 AS DECIMAL(15, 2)) AS l_discount,
- CAST(1 + value % 50 AS DECIMAL(15, 2)) AS l_quantity
- FROM range(1100)
-)
-TO 'test_files/scratch/filter_statistics_decimal/lineitem.parquet'
-STORED AS PARQUET;
-----
-1100
-
-statement ok
-set datafusion.explain.physical_plan_only = true;
-
-statement ok
-set datafusion.explain.show_statistics = true;
-
-statement ok
-set datafusion.execution.target_partitions = 1;
-
-statement ok
-CREATE EXTERNAL TABLE lineitem
-STORED AS PARQUET
-LOCATION 'test_files/scratch/filter_statistics_decimal/lineitem.parquet';
-
-# TPC-H Q6-style decimal ranges with fractional literals: the estimate uses
-# 3 of 11 discount values and 2300 of 4901 quantity values (1.00..23.99).
-query TT
-EXPLAIN SELECT l_discount FROM lineitem
-WHERE l_discount >= 0.05 AND l_discount <= 0.07 AND l_quantity < 24;
-----
-physical_plan
-01)FilterExec: l_discount@0 >= 0.05 AND l_discount@0 <= 0.07 AND l_quantity@1 < 24.00, projection=[l_discount@0], statistics=[Rows=Inexact(141), Bytes=Inexact(4506), [(Col[0]: Min=Inexact(Decimal128(0.05,15,2)) Max=Inexact(Decimal128(0.07,15,2)) Null=Exact(0) ScanBytes=Inexact(2253))]]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/filter_statistics_decimal/lineitem.parquet]]}, projection=[l_discount, l_quantity], file_type=parquet, predicate=l_discount@0 >= 0.05 AND l_discount@0 <= 0.07 AND l_quantity@1 < 24.00, pruning_predicate=l_discount_null_count@1 != row_count@2 AND l_discount_max@0 >= 0.05 AND l_discount_null_count@1 != row_count@2 AND l_discount_min@3 <= 0.07 AND l_quantity_null_count@5 != row_count@2 AND l_quantity_min@4 < 24.00, required_guarantees=[], statistics=[Rows=Inexact(1100), Bytes=Inexact(35200), [(Col[0]: Min=Inexact(Decimal128(0.00,15,2)) Max=Inexact(Decimal128(0.10,15,2)) Null=Inexact(0) ScanBytes=Inexact(17600)),(Col[1]: Min=Inexact(Decimal128(1.00,15,2)) Max=Inexact(Decimal128(50.00,15,2)) Null=Inexact(0) ScanBytes=Inexact(17600))]]
-
-query I
-SELECT count(*) FROM lineitem
-WHERE l_discount >= 0.05 AND l_discount <= 0.07 AND l_quantity < 24;
-----
-138
-
-# A literal with more digits than the column's scale is compared through a
-# cast to a wider decimal; the column is still narrowed to [0.06, 0.10].
-query TT
-EXPLAIN SELECT l_discount FROM lineitem WHERE l_discount >= 0.055;
-----
-physical_plan
-01)FilterExec: CAST(l_discount@0 AS Decimal128(30, 15)) >= 0.055000000000000, statistics=[Rows=Inexact(500), Bytes=Inexact(8000), [(Col[0]: Min=Inexact(Decimal128(0.06,15,2)) Max=Inexact(Decimal128(0.10,15,2)) Null=Inexact(0) ScanBytes=Inexact(8000))]]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/filter_statistics_decimal/lineitem.parquet]]}, projection=[l_discount], file_type=parquet, predicate=CAST(l_discount@0 AS Decimal128(30, 15)) >= 0.055000000000000, pruning_predicate=l_discount_null_count@1 != row_count@2 AND CAST(l_discount_max@0 AS Decimal128(30, 15);N) >= 0.055000000000000, required_guarantees=[], statistics=[Rows=Inexact(1100), Bytes=Inexact(17600), [(Col[0]: Min=Inexact(Decimal128(0.00,15,2)) Max=Inexact(Decimal128(0.10,15,2)) Null=Inexact(0) ScanBytes=Inexact(17600))]]
-
-query I
-SELECT count(*) FROM lineitem WHERE l_discount >= 0.055;
-----
-500
-
-# Decimal arithmetic is not analyzed and uses the default selectivity.
-query TT
-EXPLAIN SELECT l_discount FROM lineitem WHERE l_discount * 2 > 0.1;
-----
-physical_plan
-01)FilterExec: l_discount@0 * 2 > 0.10, statistics=[Rows=Inexact(220), Bytes=Inexact(3520), [(Col[0]: Min=Inexact(Decimal128(0.00,15,2)) Max=Inexact(Decimal128(0.10,15,2)) Null=Inexact(0) ScanBytes=Inexact(3520))]]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/filter_statistics_decimal/lineitem.parquet]]}, projection=[l_discount], file_type=parquet, predicate=l_discount@0 * 2 > 0.10, statistics=[Rows=Inexact(1100), Bytes=Inexact(17600), [(Col[0]: Min=Inexact(Decimal128(0.00,15,2)) Max=Inexact(Decimal128(0.10,15,2)) Null=Inexact(0) ScanBytes=Inexact(17600))]]
-
-query I
-SELECT count(*) FROM lineitem WHERE l_discount * 2 > 0.1;
-----
-500
-
-statement ok
-DROP TABLE lineitem;
-
-statement ok
-set datafusion.execution.target_partitions = 4;
-
-statement ok
-RESET datafusion.explain.physical_plan_only;
-
-statement ok
-RESET datafusion.explain.show_statistics;