Skip to content

Apache DataFusion SQL Query Engine

fix: preserve precision in log() with mixed-width float arguments (#23676)

yoongbok leec1b23798dab6
diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rs
index 6b6b10cc2a..949361c51e 100644
--- a/datafusion/functions/src/math/log.rs
+++ b/datafusion/functions/src/math/log.rs
@@ -12,7 +12,9 @@ use arrow::datatypes::{
};
use arrow::error::ArrowError;
use arrow_buffer::i256;
-use datafusion_common::types::NativeType;
+use datafusion_common::types::{
+ NativeType, logical_float16, logical_float32, logical_float64,
+};
use datafusion_common::{Result, ScalarValue, exec_err, plan_datafusion_err, plan_err};
use datafusion_expr::interval_arithmetic::Interval;
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
@@ -60,6 +62,20 @@ impl LogFunc {
vec![TypeSignatureClass::Numeric],
NativeType::Float64,
);
+
+ // For log(base, value) with float arguments, both are widened to the
+ // widest float so a wider base is not narrowed to the value's type
+ // (issue #22581)
+ let float16 = Coercion::new_exact(TypeSignatureClass::Native(logical_float16()));
+ let float32 = Coercion::new_implicit_native(
+ logical_float32(),
+ vec![TypeSignatureClass::Native(logical_float16())],
+ );
+ let float64 = Coercion::new_implicit_native(
+ logical_float64(),
+ vec![TypeSignatureClass::Numeric],
+ );
+
Self {
signature: Signature::one_of(
// Ensure decimals have precedence over floats since we have
@@ -72,10 +88,15 @@ impl LogFunc {
TypeSignature::Coercible(vec![as_float.clone()]),
// log(base, value)
TypeSignature::Coercible(vec![
- as_float.clone(),
+ as_float,
Coercion::new_exact(TypeSignatureClass::Decimal),
]),
- TypeSignature::Coercible(vec![as_float.clone(), as_float.clone()]),
+ // Only (Float16, Float16)
+ TypeSignature::Coercible(vec![float16.clone(), float16]),
+ // Any mix of Float16 and Float32, promoted to Float32
+ TypeSignature::Coercible(vec![float32.clone(), float32]),
+ // All other numeric inputs, promoted to Float64
+ TypeSignature::Coercible(vec![float64.clone(), float64]),
],
Volatility::Immutable,
),
@@ -326,8 +347,13 @@ impl ScalarUDFImpl for LogFunc {
.collect::<Result<Vec<_>>>()?;
let return_type = self.return_type(&arg_types)?;
- // Null propagation
- if arg_types.iter().any(|dt| dt.is_null()) {
+ // Null propagation. Coercion may have already turned a NULL argument
+ // into a typed NULL literal, e.g. `log(NULL, float64_col)`
+ if arg_types.iter().any(|dt| dt.is_null())
+ || args
+ .iter()
+ .any(|arg| matches!(arg, Expr::Literal(value, _) if value.is_null()))
+ {
return Ok(ExprSimplifyResult::Simplified(lit(
ScalarValue::Null.cast_to(&return_type)?
)));
@@ -370,6 +396,8 @@ mod tests {
use arrow::datatypes::{DECIMAL256_MAX_PRECISION, Field};
use datafusion_common::cast::{as_float32_array, as_float64_array};
use datafusion_common::config::ConfigOptions;
+ use datafusion_expr::ScalarUDF;
+ use datafusion_expr::type_coercion::functions::fields_with_udf;
#[test]
fn test_log_decimal_native() {
@@ -731,6 +759,32 @@ mod tests {
}
}
+ #[test]
+ fn test_log_simplify_typed_null() {
+ // Coercion can turn a NULL argument into a typed NULL literal
+ let context = SimplifyContext::default();
+ for (args, expected) in [
+ (
+ vec![lit(ScalarValue::Float64(None)), lit(2.0)],
+ ScalarValue::Float64(None),
+ ),
+ (
+ vec![lit(2.0), lit(ScalarValue::Float64(None))],
+ ScalarValue::Float64(None),
+ ),
+ (
+ vec![lit(ScalarValue::Float32(None)), lit(2.0f32)],
+ ScalarValue::Float32(None),
+ ),
+ ] {
+ let result = LogFunc::new().simplify(args, &context).unwrap();
+ let ExprSimplifyResult::Simplified(expr) = result else {
+ panic!("Expected ExprSimplifyResult::Simplified")
+ };
+ assert_eq!(expr, lit(expected));
+ }
+ }
+
#[test]
fn test_log_output_ordering() {
// Range proving base > 1.0 and num > 1.0 (e.g. [2.0, 10.0])
@@ -1190,4 +1244,51 @@ mod tests {
}
}
}
+
+ #[test]
+ fn test_log_coerces_floats_to_widest() {
+ use DataType::*;
+ let udf = ScalarUDF::from(LogFunc::new());
+ let coerce = |types: &[DataType]| -> Vec<DataType> {
+ let fields = types
+ .iter()
+ .map(|t| Arc::new(Field::new("a", t.clone(), true)))
+ .collect::<Vec<_>>();
+ fields_with_udf(&fields, &udf)
+ .unwrap()
+ .iter()
+ .map(|f| f.data_type().clone())
+ .collect()
+ };
+
+ // A wider base must not be narrowed to the value's type (#22581)
+ assert_eq!(coerce(&[Float64, Float32]), [Float64, Float64]);
+ assert_eq!(coerce(&[Float32, Float64]), [Float64, Float64]);
+ assert_eq!(coerce(&[Float16, Float32]), [Float32, Float32]);
+ assert_eq!(coerce(&[Float32, Float16]), [Float32, Float32]);
+ assert_eq!(coerce(&[Float16, Float64]), [Float64, Float64]);
+ assert_eq!(coerce(&[Int64, Float32]), [Float64, Float64]);
+ assert_eq!(coerce(&[Float32, Int64]), [Float64, Float64]);
+ assert_eq!(coerce(&[Decimal128(10, 2), Float32]), [Float64, Float64]);
+ assert_eq!(
+ coerce(&[Float64, Dictionary(Box::new(Int32), Box::new(Float32))]),
+ [Float64, Float64]
+ );
+ // Homogeneous inputs keep their type
+ assert_eq!(coerce(&[Float16, Float16]), [Float16, Float16]);
+ assert_eq!(coerce(&[Float32, Float32]), [Float32, Float32]);
+ assert_eq!(coerce(&[Float64, Float64]), [Float64, Float64]);
+ assert_eq!(coerce(&[Float32]), [Float32]);
+ // Decimal values keep the native decimal implementation
+ assert_eq!(
+ coerce(&[Float32, Decimal128(10, 2)]),
+ [Float32, Decimal128(10, 2)]
+ );
+ assert_eq!(coerce(&[Decimal128(10, 2)]), [Decimal128(10, 2)]);
+ // A NULL base is typed by the value; a NULL value matches the decimal
+ // value signature and stays untyped
+ assert_eq!(coerce(&[Null, Float32]), [Float32, Float32]);
+ assert_eq!(coerce(&[Float32, Null]), [Float32, Null]);
+ assert_eq!(coerce(&[Null, Null]), [Null, Null]);
+ }
}
diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt
index 254616c15d..ef62708091 100644
--- a/datafusion/sqllogictest/test_files/math.slt
+++ b/datafusion/sqllogictest/test_files/math.slt
@@ -1292,22 +1292,80 @@ FROM (VALUES (2000.0::double), (-2000.0::double)) t(a);
Infinity Infinity
-Infinity -Infinity
-# Float 16/32/64 for log
+# log(base, value) widens float arguments to the wider of the two, so a wider
+# base is not narrowed to the value's type (#22581). Here base 2.5 is Float64.
query RT
SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));
----
-2.6074219 Float16
+2.606835742462 Float64
query RT
SELECT log(2.5, 10.9::float), arrow_typeof(log(2.5, 10.9::float));
----
-2.606992 Float32
+2.606992159958 Float64
query RT
SELECT log(2.5, 10.9::double), arrow_typeof(log(2.5, 10.9::double));
----
2.606992198152 Float64
+# Homogeneous float widths are preserved
+query RT
+SELECT log(arrow_cast(2.5, 'Float16'), arrow_cast(10.9, 'Float16')), arrow_typeof(log(arrow_cast(2.5, 'Float16'), arrow_cast(10.9, 'Float16')));
+----
+2.6074219 Float16
+
+# Unary Float16 log uses a Float16 default base.
+query RT
+SELECT log(arrow_cast(100.0, 'Float16')), arrow_typeof(log(arrow_cast(100.0, 'Float16')));
+----
+2 Float16
+
+query RT
+SELECT log(2.5::float, 10.9::float), arrow_typeof(log(2.5::float, 10.9::float));
+----
+2.606992 Float32
+
+# A Float16 base is widened to a Float32 value's type
+query RT
+SELECT log(arrow_cast(2.5, 'Float16'), 10.9::float), arrow_typeof(log(arrow_cast(2.5, 'Float16'), 10.9::float));
+----
+2.606992 Float32
+
+# #22581: a Float64 base must keep full precision against a Float32 value.
+# 16777217 is not representable in Float32, so an f32 computation would round it.
+query RT
+SELECT log(arrow_cast(16777217, 'Float64'), arrow_cast(2.0, 'Float32')), arrow_typeof(log(arrow_cast(16777217, 'Float64'), arrow_cast(2.0, 'Float32')));
+----
+0.041666666517 Float64
+
+# NULL arguments keep their existing result types
+query RT
+SELECT log(2.5::float, NULL), arrow_typeof(log(2.5::float, NULL));
+----
+NULL Float64
+
+query RT
+SELECT log(NULL, 2.0::float), arrow_typeof(log(NULL, 2.0::float));
+----
+NULL Float32
+
+query RT
+SELECT log(NULL, NULL), arrow_typeof(log(NULL, NULL));
+----
+NULL Float64
+
+# A Float16 base is widened to a wider value's type
+query RT
+SELECT log(arrow_cast(2.5, 'Float16'), 1.0::double), arrow_typeof(log(arrow_cast(2.5, 'Float16'), 1.0::double));
+----
+0 Float64
+
+query RT
+SELECT log(arrow_cast(2.5, 'Float16'), arrow_cast(1.0, 'Float16')), arrow_typeof(log(arrow_cast(2.5, 'Float16'), arrow_cast(1.0, 'Float16')));
+----
+0 Float16
+
# lcm with array and scalar
query I
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index fe9a417c9d..0d53180219 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -944,12 +944,12 @@ query TT
----
logical_plan
01)Sort: log_c11_base_c12 ASC NULLS LAST
-02)--Projection: log(aggregate_test_100.c12, aggregate_test_100.c11) AS log_c11_base_c12
+02)--Projection: log(aggregate_test_100.c12, CAST(aggregate_test_100.c11 AS Float64)) AS log_c11_base_c12
03)----TableScan: aggregate_test_100 projection=[c11, c12]
physical_plan
01)SortPreservingMergeExec: [log_c11_base_c12@0 ASC NULLS LAST]
02)--SortExec: expr=[log_c11_base_c12@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]
+03)----ProjectionExec: expr=[log(c12@1, CAST(c11@0 AS Float64)) as log_c11_base_c12]
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
@@ -960,12 +960,12 @@ ORDER BY log_c12_base_c11 DESC NULLS LAST;
----
logical_plan
01)Sort: log_c12_base_c11 DESC NULLS LAST
-02)--Projection: log(aggregate_test_100.c11, aggregate_test_100.c12) AS log_c12_base_c11
+02)--Projection: log(CAST(aggregate_test_100.c11 AS Float64), aggregate_test_100.c12) AS log_c12_base_c11
03)----TableScan: aggregate_test_100 projection=[c11, c12]
physical_plan
01)SortPreservingMergeExec: [log_c12_base_c11@0 DESC NULLS LAST]
02)--SortExec: expr=[log_c12_base_c11@0 DESC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]
+03)----ProjectionExec: expr=[log(CAST(c11@0 AS Float64), c12@1) as log_c12_base_c11]
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt
index 006ae86af0..b62113febe 100644
--- a/datafusion/sqllogictest/test_files/scalar.slt
+++ b/datafusion/sqllogictest/test_files/scalar.slt
@@ -705,8 +705,8 @@ select log(2,arrow_cast(a, 'Float64')), log(4,arrow_cast(b, 'Float32')) from sig
----
1 NaN
2 NULL
-NaN 3.321928
-NaN 6.643856
+NaN 3.321928094887
+NaN 6.643856189775
## log10
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 78e2a2cf90..6b33b5eded 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -709,6 +709,27 @@ value of the stride instead. See [issue #25856] for details.
[postgresql]: https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-BIN
[issue #25856]: https://github.com/apache/datafusion/issues/25856
+### `log(base, value)` widens float arguments to the wider type
+
+`log(base, value)` previously computed in the float type of `value`, so a
+wider `base` was narrowed and lost precision. Both arguments are now widened to
+the wider float type, and the result has that type. An integer or decimal
+`base` counts as `Float64`.
+
+```sql
+-- Before: 0.041666668 (Float32)
+-- After: 0.041666666517376175 (Float64)
+SELECT log(arrow_cast(16777217, 'Float64'), arrow_cast(2.0, 'Float32'));
+```
+
+For example, `log(2, float32_col)` now returns `Float64` instead of `Float32`.
+
+Calls whose arguments have the same float type, and calls with a decimal
+`value`, are unchanged. Wrap the call in `CAST` to keep the previous result
+type. See [issue #22581] for details.
+
+[issue #22581]: https://github.com/apache/datafusion/issues/22581
+
### `DataFrame::from_columns` accepts `IntoIterator`
`DataFrame::from_columns` now accepts any `IntoIterator<Item = (&str, ArrayRef)>`