Skip to content

Apache DataFusion SQL Query Engine

fix: keep decimal fractions in modulo by one (#25961)

Stefan Wangde4e344f26e7
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 506692f0df..788f1cfb03 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -828,6 +828,26 @@ impl<'a> Simplifier<'a> {
}
}
+/// Returns whether values of the given type are guaranteed to satisfy `value % 1 == 0`.
+fn modulo_one_is_zero_type(data_type: &DataType) -> bool {
+ match data_type {
+ DataType::Int8
+ | DataType::Int16
+ | DataType::Int32
+ | DataType::Int64
+ | DataType::UInt8
+ | DataType::UInt16
+ | DataType::UInt32
+ | DataType::UInt64 => true,
+ // A non-positive scale makes every decimal value an integer.
+ DataType::Decimal32(_, scale)
+ | DataType::Decimal64(_, scale)
+ | DataType::Decimal128(_, scale)
+ | DataType::Decimal256(_, scale) => *scale <= 0,
+ _ => false,
+ }
+}
+
impl TreeNodeRewriter for Simplifier<'_> {
type Node = Expr;
@@ -1207,13 +1227,13 @@ impl TreeNodeRewriter for Simplifier<'_> {
// Rules for Modulo
//
- // A % 1 --> 0 (if A is not nullable and not floating, since NAN % 1 --> NAN)
+ // A % 1 --> 0 (if A is non-null and its type guarantees an integer value)
Expr::BinaryExpr(BinaryExpr {
left,
op: Modulo,
right,
}) if !info.nullable(&left)?
- && !info.get_data_type(&left)?.is_floating()
+ && modulo_one_is_zero_type(&info.get_data_type(&left)?)
&& is_one(&right) =>
{
Transformed::yes(Expr::Literal(
@@ -3075,6 +3095,55 @@ mod tests {
assert_eq!(simplify(expr), expected);
}
+ #[test]
+ fn test_simplify_modulo_by_one_types() {
+ for (data_type, folds) in [
+ (DataType::Int8, true),
+ (DataType::Int16, true),
+ (DataType::Int32, true),
+ (DataType::Int64, true),
+ (DataType::UInt8, true),
+ (DataType::UInt16, true),
+ (DataType::UInt32, true),
+ (DataType::UInt64, true),
+ (DataType::Float32, false),
+ (DataType::Float64, false),
+ (DataType::Decimal32(9, -1), true),
+ (DataType::Decimal32(9, 0), true),
+ (DataType::Decimal32(9, 2), false),
+ (DataType::Decimal64(18, -1), true),
+ (DataType::Decimal64(18, 0), true),
+ (DataType::Decimal64(18, 2), false),
+ (DataType::Decimal128(38, -1), true),
+ (DataType::Decimal128(38, 0), true),
+ (DataType::Decimal128(38, 2), false),
+ (DataType::Decimal256(76, -1), true),
+ (DataType::Decimal256(76, 0), true),
+ (DataType::Decimal256(76, 2), false),
+ ] {
+ for nullable in [false, true] {
+ let schema =
+ Schema::new(vec![Field::new("a", data_type.clone(), nullable)])
+ .to_dfschema_ref()
+ .unwrap();
+ let simplifier = ExprSimplifier::new(
+ SimplifyContext::builder().with_schema(schema).build(),
+ );
+ let expr = col("a") % lit(1);
+ let expected = if folds && !nullable {
+ lit(ScalarValue::new_zero(&data_type).unwrap())
+ } else {
+ expr.clone()
+ };
+ assert_eq!(
+ simplifier.simplify(expr).unwrap(),
+ expected,
+ "{data_type:?}, nullable={nullable}"
+ );
+ }
+ }
+ }
+
#[test]
fn test_simplify_bitwise_xor_by_null() {
let null = lit(ScalarValue::Int64(None));
diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt
index 35dd69618b..1b9fa63bf8 100644
--- a/datafusion/sqllogictest/test_files/math.slt
+++ b/datafusion/sqllogictest/test_files/math.slt
@@ -776,6 +776,19 @@ SELECT c1%1, c2%1 FROM test_non_nullable_float
0 0
NaN NaN
+statement ok
+SET datafusion.explain.logical_plan_only = true;
+
+query TT
+EXPLAIN SELECT c2 % 1 AS remainder FROM test_non_nullable_float;
+----
+logical_plan
+01)Projection: test_non_nullable_float.c2 % Float64(1) AS remainder
+02)--TableScan: test_non_nullable_float projection=[c2]
+
+statement ok
+RESET datafusion.explain.logical_plan_only;
+
statement ok
drop table test_non_nullable_float
@@ -864,7 +877,7 @@ statement ok
CREATE TABLE test_non_nullable_decimal(c1 DECIMAL(9,2) NOT NULL);
query I
-INSERT INTO test_non_nullable_decimal VALUES(1)
+INSERT INTO test_non_nullable_decimal VALUES(1.50)
----
1
@@ -879,6 +892,29 @@ SELECT c1/0 FROM test_non_nullable_decimal
query error DataFusion error: Arrow error: Divide by zero error
SELECT c1%0 FROM test_non_nullable_decimal
+query RR
+SELECT c1 % 1 AS int_literal, c1 % CAST(1 AS DECIMAL(9,2)) AS decimal_literal FROM test_non_nullable_decimal
+----
+0.50 0.50
+
+query I
+SELECT count(*) FROM test_non_nullable_decimal WHERE c1 % 1 = 0
+----
+0
+
+statement ok
+SET datafusion.explain.logical_plan_only = true;
+
+query TT
+EXPLAIN SELECT c1 % CAST(1 AS DECIMAL(9,2)) AS remainder FROM test_non_nullable_decimal;
+----
+logical_plan
+01)Projection: test_non_nullable_decimal.c1 % Decimal128(1.00,9,2) AS remainder
+02)--TableScan: test_non_nullable_decimal projection=[c1]
+
+statement ok
+RESET datafusion.explain.logical_plan_only;
+
statement ok
drop table test_non_nullable_decimal