Apache DataFusion SQL Query Engine
fix: keep decimal fractions in modulo by one (#25961)
Stefan Wangde4e344f26e7
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 506692f0df..788f1cfb03 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -828,6 +828,26 @@ impl<'a> Simplifier<'a> {}}+/// Returns whether values of the given type are guaranteed to satisfy `value % 1 == 0`.+fn modulo_one_is_zero_type(data_type: &DataType) -> bool {+ match data_type {+ DataType::Int8+ | DataType::Int16+ | DataType::Int32+ | DataType::Int64+ | DataType::UInt8+ | DataType::UInt16+ | DataType::UInt32+ | DataType::UInt64 => true,+ // A non-positive scale makes every decimal value an integer.+ DataType::Decimal32(_, scale)+ | DataType::Decimal64(_, scale)+ | DataType::Decimal128(_, scale)+ | DataType::Decimal256(_, scale) => *scale <= 0,+ _ => false,+ }+}+impl TreeNodeRewriter for Simplifier<'_> {type Node = Expr;@@ -1207,13 +1227,13 @@ impl TreeNodeRewriter for Simplifier<'_> {// Rules for Modulo//- // A % 1 --> 0 (if A is not nullable and not floating, since NAN % 1 --> NAN)+ // A % 1 --> 0 (if A is non-null and its type guarantees an integer value)Expr::BinaryExpr(BinaryExpr {left,op: Modulo,right,}) if !info.nullable(&left)?- && !info.get_data_type(&left)?.is_floating()+ && modulo_one_is_zero_type(&info.get_data_type(&left)?)&& is_one(&right) =>{Transformed::yes(Expr::Literal(@@ -3075,6 +3095,55 @@ mod tests {assert_eq!(simplify(expr), expected);}+ #[test]+ fn test_simplify_modulo_by_one_types() {+ for (data_type, folds) in [+ (DataType::Int8, true),+ (DataType::Int16, true),+ (DataType::Int32, true),+ (DataType::Int64, true),+ (DataType::UInt8, true),+ (DataType::UInt16, true),+ (DataType::UInt32, true),+ (DataType::UInt64, true),+ (DataType::Float32, false),+ (DataType::Float64, false),+ (DataType::Decimal32(9, -1), true),+ (DataType::Decimal32(9, 0), true),+ (DataType::Decimal32(9, 2), false),+ (DataType::Decimal64(18, -1), true),+ (DataType::Decimal64(18, 0), true),+ (DataType::Decimal64(18, 2), false),+ (DataType::Decimal128(38, -1), true),+ (DataType::Decimal128(38, 0), true),+ (DataType::Decimal128(38, 2), false),+ (DataType::Decimal256(76, -1), true),+ (DataType::Decimal256(76, 0), true),+ (DataType::Decimal256(76, 2), false),+ ] {+ for nullable in [false, true] {+ let schema =+ Schema::new(vec![Field::new("a", data_type.clone(), nullable)])+ .to_dfschema_ref()+ .unwrap();+ let simplifier = ExprSimplifier::new(+ SimplifyContext::builder().with_schema(schema).build(),+ );+ let expr = col("a") % lit(1);+ let expected = if folds && !nullable {+ lit(ScalarValue::new_zero(&data_type).unwrap())+ } else {+ expr.clone()+ };+ assert_eq!(+ simplifier.simplify(expr).unwrap(),+ expected,+ "{data_type:?}, nullable={nullable}"+ );+ }+ }+ }+#[test]fn test_simplify_bitwise_xor_by_null() {let null = lit(ScalarValue::Int64(None));diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.sltindex 35dd69618b..1b9fa63bf8 100644--- a/datafusion/sqllogictest/test_files/math.slt+++ b/datafusion/sqllogictest/test_files/math.slt@@ -776,6 +776,19 @@ SELECT c1%1, c2%1 FROM test_non_nullable_float0 0NaN NaN+statement ok+SET datafusion.explain.logical_plan_only = true;++query TT+EXPLAIN SELECT c2 % 1 AS remainder FROM test_non_nullable_float;+----+logical_plan+01)Projection: test_non_nullable_float.c2 % Float64(1) AS remainder+02)--TableScan: test_non_nullable_float projection=[c2]++statement ok+RESET datafusion.explain.logical_plan_only;+statement okdrop table test_non_nullable_float@@ -864,7 +877,7 @@ statement okCREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL);query I-INSERT INTO test_non_nullable_decimal VALUES(1)+INSERT INTO test_non_nullable_decimal VALUES(1.50)----1@@ -879,6 +892,29 @@ SELECT c1/0 FROM test_non_nullable_decimalquery error DataFusion error: Arrow error: Divide by zero errorSELECT c1%0 FROM test_non_nullable_decimal+query RR+SELECT c1 % 1 AS int_literal, c1 % CAST(1 AS DECIMAL(9,2)) AS decimal_literal FROM test_non_nullable_decimal+----+0.50 0.50++query I+SELECT count(*) FROM test_non_nullable_decimal WHERE c1 % 1 = 0+----+0++statement ok+SET datafusion.explain.logical_plan_only = true;++query TT+EXPLAIN SELECT c1 % CAST(1 AS DECIMAL(9,2)) AS remainder FROM test_non_nullable_decimal;+----+logical_plan+01)Projection: test_non_nullable_decimal.c1 % Decimal128(1.00,9,2) AS remainder+02)--TableScan: test_non_nullable_decimal projection=[c1]++statement ok+RESET datafusion.explain.logical_plan_only;+statement okdrop table test_non_nullable_decimal