Apache DataFusion SQL Query Engine
fix: preserve precision in log() with mixed-width float arguments (#23676)
yoongbok leec1b23798dab6
diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rsindex 6b6b10cc2a..949361c51e 100644--- a/datafusion/functions/src/math/log.rs+++ b/datafusion/functions/src/math/log.rs@@ -12,7 +12,9 @@ use arrow::datatypes::{};use arrow::error::ArrowError;use arrow_buffer::i256;-use datafusion_common::types::NativeType;+use datafusion_common::types::{+ NativeType, logical_float16, logical_float32, logical_float64,+};use datafusion_common::{Result, ScalarValue, exec_err, plan_datafusion_err, plan_err};use datafusion_expr::interval_arithmetic::Interval;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};@@ -60,6 +62,20 @@ impl LogFunc {vec![TypeSignatureClass::Numeric],NativeType::Float64,);++ // For log(base, value) with float arguments, both are widened to the+ // widest float so a wider base is not narrowed to the value's type+ // (issue #22581)+ let float16 = Coercion::new_exact(TypeSignatureClass::Native(logical_float16()));+ let float32 = Coercion::new_implicit_native(+ logical_float32(),+ vec![TypeSignatureClass::Native(logical_float16())],+ );+ let float64 = Coercion::new_implicit_native(+ logical_float64(),+ vec![TypeSignatureClass::Numeric],+ );+Self {signature: Signature::one_of(// Ensure decimals have precedence over floats since we have@@ -72,10 +88,15 @@ impl LogFunc {TypeSignature::Coercible(vec![as_float.clone()]),// log(base, value)TypeSignature::Coercible(vec![- as_float.clone(),+ as_float,Coercion::new_exact(TypeSignatureClass::Decimal),]),- TypeSignature::Coercible(vec![as_float.clone(), as_float.clone()]),+ // Only (Float16, Float16)+ TypeSignature::Coercible(vec![float16.clone(), float16]),+ // Any mix of Float16 and Float32, promoted to Float32+ TypeSignature::Coercible(vec![float32.clone(), float32]),+ // All other numeric inputs, promoted to Float64+ TypeSignature::Coercible(vec![float64.clone(), float64]),],Volatility::Immutable,),@@ -326,8 +347,13 @@ impl ScalarUDFImpl for LogFunc {.collect::<Result<Vec<_>>>()?;let return_type = self.return_type(&arg_types)?;- // Null propagation- if arg_types.iter().any(|dt| dt.is_null()) {+ // Null propagation. Coercion may have already turned a NULL argument+ // into a typed NULL literal, e.g. `log(NULL, float64_col)`+ if arg_types.iter().any(|dt| dt.is_null())+ || args+ .iter()+ .any(|arg| matches!(arg, Expr::Literal(value, _) if value.is_null()))+ {return Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::Null.cast_to(&return_type)?)));@@ -370,6 +396,8 @@ mod tests {use arrow::datatypes::{DECIMAL256_MAX_PRECISION, Field};use datafusion_common::cast::{as_float32_array, as_float64_array};use datafusion_common::config::ConfigOptions;+ use datafusion_expr::ScalarUDF;+ use datafusion_expr::type_coercion::functions::fields_with_udf;#[test]fn test_log_decimal_native() {@@ -731,6 +759,32 @@ mod tests {}}+ #[test]+ fn test_log_simplify_typed_null() {+ // Coercion can turn a NULL argument into a typed NULL literal+ let context = SimplifyContext::default();+ for (args, expected) in [+ (+ vec![lit(ScalarValue::Float64(None)), lit(2.0)],+ ScalarValue::Float64(None),+ ),+ (+ vec![lit(2.0), lit(ScalarValue::Float64(None))],+ ScalarValue::Float64(None),+ ),+ (+ vec![lit(ScalarValue::Float32(None)), lit(2.0f32)],+ ScalarValue::Float32(None),+ ),+ ] {+ let result = LogFunc::new().simplify(args, &context).unwrap();+ let ExprSimplifyResult::Simplified(expr) = result else {+ panic!("Expected ExprSimplifyResult::Simplified")+ };+ assert_eq!(expr, lit(expected));+ }+ }+#[test]fn test_log_output_ordering() {// Range proving base > 1.0 and num > 1.0 (e.g. [2.0, 10.0])@@ -1190,4 +1244,51 @@ mod tests {}}}++ #[test]+ fn test_log_coerces_floats_to_widest() {+ use DataType::*;+ let udf = ScalarUDF::from(LogFunc::new());+ let coerce = |types: &[DataType]| -> Vec<DataType> {+ let fields = types+ .iter()+ .map(|t| Arc::new(Field::new("a", t.clone(), true)))+ .collect::<Vec<_>>();+ fields_with_udf(&fields, &udf)+ .unwrap()+ .iter()+ .map(|f| f.data_type().clone())+ .collect()+ };++ // A wider base must not be narrowed to the value's type (#22581)+ assert_eq!(coerce(&[Float64, Float32]), [Float64, Float64]);+ assert_eq!(coerce(&[Float32, Float64]), [Float64, Float64]);+ assert_eq!(coerce(&[Float16, Float32]), [Float32, Float32]);+ assert_eq!(coerce(&[Float32, Float16]), [Float32, Float32]);+ assert_eq!(coerce(&[Float16, Float64]), [Float64, Float64]);+ assert_eq!(coerce(&[Int64, Float32]), [Float64, Float64]);+ assert_eq!(coerce(&[Float32, Int64]), [Float64, Float64]);+ assert_eq!(coerce(&[Decimal128(10, 2), Float32]), [Float64, Float64]);+ assert_eq!(+ coerce(&[Float64, Dictionary(Box::new(Int32), Box::new(Float32))]),+ [Float64, Float64]+ );+ // Homogeneous inputs keep their type+ assert_eq!(coerce(&[Float16, Float16]), [Float16, Float16]);+ assert_eq!(coerce(&[Float32, Float32]), [Float32, Float32]);+ assert_eq!(coerce(&[Float64, Float64]), [Float64, Float64]);+ assert_eq!(coerce(&[Float32]), [Float32]);+ // Decimal values keep the native decimal implementation+ assert_eq!(+ coerce(&[Float32, Decimal128(10, 2)]),+ [Float32, Decimal128(10, 2)]+ );+ assert_eq!(coerce(&[Decimal128(10, 2)]), [Decimal128(10, 2)]);+ // A NULL base is typed by the value; a NULL value matches the decimal+ // value signature and stays untyped+ assert_eq!(coerce(&[Null, Float32]), [Float32, Float32]);+ assert_eq!(coerce(&[Float32, Null]), [Float32, Null]);+ assert_eq!(coerce(&[Null, Null]), [Null, Null]);+ }}diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.sltindex 254616c15d..ef62708091 100644--- a/datafusion/sqllogictest/test_files/math.slt+++ b/datafusion/sqllogictest/test_files/math.slt@@ -1292,22 +1292,80 @@ FROM (VALUES (2000.0::double), (-2000.0::double)) t(a);Infinity Infinity-Infinity -Infinity-# Float 16/32/64 for log+# log(base, value) widens float arguments to the wider of the two, so a wider+# base is not narrowed to the value's type (#22581). Here base 2.5 is Float64.query RTSELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));-----2.6074219 Float16+2.606835742462 Float64query RTSELECT log(2.5, 10.9::float), arrow_typeof(log(2.5, 10.9::float));-----2.606992 Float32+2.606992159958 Float64query RTSELECT log(2.5, 10.9::double), arrow_typeof(log(2.5, 10.9::double));----2.606992198152 Float64+# Homogeneous float widths are preserved+query RT+SELECT log(arrow_cast(2.5, 'Float16'), arrow_cast(10.9, 'Float16')), arrow_typeof(log(arrow_cast(2.5, 'Float16'), arrow_cast(10.9, 'Float16')));+----+2.6074219 Float16++# Unary Float16 log uses a Float16 default base.+query RT+SELECT log(arrow_cast(100.0, 'Float16')), arrow_typeof(log(arrow_cast(100.0, 'Float16')));+----+2 Float16++query RT+SELECT log(2.5::float, 10.9::float), arrow_typeof(log(2.5::float, 10.9::float));+----+2.606992 Float32++# A Float16 base is widened to a Float32 value's type+query RT+SELECT log(arrow_cast(2.5, 'Float16'), 10.9::float), arrow_typeof(log(arrow_cast(2.5, 'Float16'), 10.9::float));+----+2.606992 Float32++# #22581: a Float64 base must keep full precision against a Float32 value.+# 16777217 is not representable in Float32, so an f32 computation would round it.+query RT+SELECT log(arrow_cast(16777217, 'Float64'), arrow_cast(2.0, 'Float32')), arrow_typeof(log(arrow_cast(16777217, 'Float64'), arrow_cast(2.0, 'Float32')));+----+0.041666666517 Float64++# NULL arguments keep their existing result types+query RT+SELECT log(2.5::float, NULL), arrow_typeof(log(2.5::float, NULL));+----+NULL Float64++query RT+SELECT log(NULL, 2.0::float), arrow_typeof(log(NULL, 2.0::float));+----+NULL Float32++query RT+SELECT log(NULL, NULL), arrow_typeof(log(NULL, NULL));+----+NULL Float64++# A Float16 base is widened to a wider value's type+query RT+SELECT log(arrow_cast(2.5, 'Float16'), 1.0::double), arrow_typeof(log(arrow_cast(2.5, 'Float16'), 1.0::double));+----+0 Float64++query RT+SELECT log(arrow_cast(2.5, 'Float16'), arrow_cast(1.0, 'Float16')), arrow_typeof(log(arrow_cast(2.5, 'Float16'), arrow_cast(1.0, 'Float16')));+----+0 Float16+# lcm with array and scalarquery Idiff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex fe9a417c9d..0d53180219 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -944,12 +944,12 @@ query TT----logical_plan01)Sort: log_c11_base_c12 ASC NULLS LAST-02)--Projection: log(aggregate_test_100.c12, aggregate_test_100.c11) AS log_c11_base_c12+02)--Projection: log(aggregate_test_100.c12, CAST(aggregate_test_100.c11 AS Float64)) AS log_c11_base_c1203)----TableScan: aggregate_test_100 projection=[c11, c12]physical_plan01)SortPreservingMergeExec: [log_c11_base_c12@0 ASC NULLS LAST]02)--SortExec: expr=[log_c11_base_c12@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]+03)----ProjectionExec: expr=[log(c12@1, CAST(c11@0 AS Float64)) as log_c11_base_c12]04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true@@ -960,12 +960,12 @@ ORDER BY log_c12_base_c11 DESC NULLS LAST;----logical_plan01)Sort: log_c12_base_c11 DESC NULLS LAST-02)--Projection: log(aggregate_test_100.c11, aggregate_test_100.c12) AS log_c12_base_c11+02)--Projection: log(CAST(aggregate_test_100.c11 AS Float64), aggregate_test_100.c12) AS log_c12_base_c1103)----TableScan: aggregate_test_100 projection=[c11, c12]physical_plan01)SortPreservingMergeExec: [log_c12_base_c11@0 DESC NULLS LAST]02)--SortExec: expr=[log_c12_base_c11@0 DESC NULLS LAST], preserve_partitioning=[true]-03)----ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]+03)----ProjectionExec: expr=[log(CAST(c11@0 AS Float64), c12@1) as log_c12_base_c11]04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=truediff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.sltindex 006ae86af0..b62113febe 100644--- a/datafusion/sqllogictest/test_files/scalar.slt+++ b/datafusion/sqllogictest/test_files/scalar.slt@@ -705,8 +705,8 @@ select log(2,arrow_cast(a, 'Float64')), log(4,arrow_cast(b, 'Float32')) from sig----1 NaN2 NULL-NaN 3.321928-NaN 6.643856+NaN 3.321928094887+NaN 6.643856189775## log10diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 78e2a2cf90..6b33b5eded 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -709,6 +709,27 @@ value of the stride instead. See [issue #25856] for details.[postgresql]: https://www.postgresql.org/docs/current/functions-datetime.html#FUNCTIONS-DATETIME-BIN[issue #25856]: https://github.com/apache/datafusion/issues/25856+### `log(base, value)` widens float arguments to the wider type++`log(base, value)` previously computed in the float type of `value`, so a+wider `base` was narrowed and lost precision. Both arguments are now widened to+the wider float type, and the result has that type. An integer or decimal+`base` counts as `Float64`.++```sql+-- Before: 0.041666668 (Float32)+-- After: 0.041666666517376175 (Float64)+SELECT log(arrow_cast(16777217, 'Float64'), arrow_cast(2.0, 'Float32'));+```++For example, `log(2, float32_col)` now returns `Float64` instead of `Float32`.++Calls whose arguments have the same float type, and calls with a decimal+`value`, are unchanged. Wrap the call in `CAST` to keep the previous result+type. See [issue #22581] for details.++[issue #22581]: https://github.com/apache/datafusion/issues/22581+### `DataFrame::from_columns` accepts `IntoIterator``DataFrame::from_columns` now accepts any `IntoIterator<Item = (&str, ArrayRef)>`