Apache DataFusion SQL Query Engine
Fix decimal log precision for non-power values (#20433)
Kumar Ujjawal4c961256da23
diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rsindex d1906a4bf0..3db24d67e8 100644--- a/datafusion/functions/src/math/log.rs+++ b/datafusion/functions/src/math/log.rs@@ -109,80 +109,59 @@ fn is_valid_integer_base(base: f64) -> bool {}/// Calculate logarithm for Decimal32 values.-/// For integer bases >= 2 with non-negative scale, uses the efficient u32 ilog algorithm.-/// Otherwise falls back to f64 computation.+/// For integer bases >= 2 with zero scale, return an exact integer log when the+/// value is a perfect power of the base. Otherwise falls back to f64 computation.fn log_decimal32(value: i32, scale: i8, base: f64) -> Result<f64, ArrowError> {- if is_valid_integer_base(base)- && scale >= 0- && let Some(unscaled) = unscale_to_u32(value, scale)+ if scale == 0+ && is_valid_integer_base(base)+ && let Ok(unscaled) = u32::try_from(value)+ && unscaled > 0{- return if unscaled > 0 {- Ok(unscaled.ilog(base as u32) as f64)- } else {- Ok(f64::NAN)- };+ let base_u32 = base as u32;+ let int_log = unscaled.ilog(base_u32);+ if base_u32.checked_pow(int_log) == Some(unscaled) {+ return Ok(int_log as f64);+ }}decimal_to_f64(value, scale).map(|v| v.log(base))}/// Calculate logarithm for Decimal64 values.-/// For integer bases >= 2 with non-negative scale, uses the efficient u64 ilog algorithm.-/// Otherwise falls back to f64 computation.+/// For integer bases >= 2 with zero scale, return an exact integer log when the+/// value is a perfect power of the base. Otherwise falls back to f64 computation.fn log_decimal64(value: i64, scale: i8, base: f64) -> Result<f64, ArrowError> {- if is_valid_integer_base(base)- && scale >= 0- && let Some(unscaled) = unscale_to_u64(value, scale)+ if scale == 0+ && is_valid_integer_base(base)+ && let Ok(unscaled) = u64::try_from(value)+ && unscaled > 0{- return if unscaled > 0 {- Ok(unscaled.ilog(base as u64) as f64)- } else {- Ok(f64::NAN)- };+ let base_u64 = base as u64;+ let int_log = unscaled.ilog(base_u64);+ if base_u64.checked_pow(int_log) == Some(unscaled) {+ return Ok(int_log as f64);+ }}decimal_to_f64(value, scale).map(|v| v.log(base))}/// Calculate logarithm for Decimal128 values.-/// For integer bases >= 2 with non-negative scale, uses the efficient u128 ilog algorithm.-/// Otherwise falls back to f64 computation.+/// For integer bases >= 2 with zero scale, return an exact integer log when the+/// value is a perfect power of the base. Otherwise falls back to f64 computation.fn log_decimal128(value: i128, scale: i8, base: f64) -> Result<f64, ArrowError> {- if is_valid_integer_base(base)- && scale >= 0- && let Some(unscaled) = unscale_to_u128(value, scale)+ if scale == 0+ && is_valid_integer_base(base)+ && let Ok(unscaled) = u128::try_from(value)+ && unscaled > 0{- return if unscaled > 0 {- Ok(unscaled.ilog(base as u128) as f64)- } else {- Ok(f64::NAN)- };+ let base_u128 = base as u128;+ let int_log = unscaled.ilog(base_u128);+ if base_u128.checked_pow(int_log) == Some(unscaled) {+ return Ok(int_log as f64);+ }}decimal_to_f64(value, scale).map(|v| v.log(base))}-/// Unscale a Decimal32 value to u32.-#[inline]-fn unscale_to_u32(value: i32, scale: i8) -> Option<u32> {- let value_u32 = u32::try_from(value).ok()?;- let divisor = 10u32.checked_pow(scale as u32)?;- Some(value_u32 / divisor)-}--/// Unscale a Decimal64 value to u64.-#[inline]-fn unscale_to_u64(value: i64, scale: i8) -> Option<u64> {- let value_u64 = u64::try_from(value).ok()?;- let divisor = 10u64.checked_pow(scale as u32)?;- Some(value_u64 / divisor)-}--/// Unscale a Decimal128 value to u128.-#[inline]-fn unscale_to_u128(value: i128, scale: i8) -> Option<u128> {- let value_u128 = u128::try_from(value).ok()?;- let divisor = 10u128.checked_pow(scale as u32)?;- Some(value_u128 / divisor)-}-/// Convert a scaled decimal value to f64.#[inline]fn decimal_to_f64<T: ToPrimitive + Copy>(value: T, scale: i8) -> Result<f64, ArrowError> {@@ -444,13 +423,9 @@ mod tests {#[test]fn test_log_decimal_native() {let value = 10_i128.pow(35);- assert_eq!((value as f64).log2(), 116.26748332105768);- assert_eq!(- log_decimal128(value, 0, 2.0).unwrap(),- // TODO: see we're losing our decimal points compared to above- // https://github.com/apache/datafusion/issues/18524- 116.0- );+ let expected = (value as f64).log2();+ let actual = log_decimal128(value, 0, 2.0).unwrap();+ assert!((actual - expected).abs() < 1e-10);}#[test]@@ -1012,7 +987,8 @@ mod tests {assert!((floats.value(1) - 2.0).abs() < 1e-10);assert!((floats.value(2) - 3.0).abs() < 1e-10);assert!((floats.value(3) - 4.0).abs() < 1e-10);- assert!((floats.value(4) - 4.0).abs() < 1e-10); // Integer rounding+ let expected = 12600_f64.log(10.0);+ assert!((floats.value(4) - expected).abs() < 1e-10);assert!(floats.value(5).is_nan());}ColumnarValue::Scalar(_) => {@@ -1142,13 +1118,14 @@ mod tests {.expect("failed to convert result to a Float64Array");assert_eq!(floats.len(), 7);- eprintln!("floats {:?}", &floats);assert!((floats.value(0) - 1.0).abs() < 1e-10);assert!((floats.value(1) - 2.0).abs() < 1e-10);assert!((floats.value(2) - 3.0).abs() < 1e-10);assert!((floats.value(3) - 4.0).abs() < 1e-10);- assert!((floats.value(4) - 4.0).abs() < 1e-10); // Integer rounding for float log- assert!((floats.value(5) - 38.0).abs() < 1e-10);+ let expected = 12600_f64.log(10.0);+ assert!((floats.value(4) - expected).abs() < 1e-10);+ let expected = ((i128::MAX - 1000) as f64).log(10.0);+ assert!((floats.value(5) - expected).abs() < 1e-10);assert!(floats.value(6).is_nan());}ColumnarValue::Scalar(_) => {diff --git a/datafusion/sqllogictest/test_files/decimal.slt b/datafusion/sqllogictest/test_files/decimal.sltindex eca2c88bb5..5485a5fd30 100644--- a/datafusion/sqllogictest/test_files/decimal.slt+++ b/datafusion/sqllogictest/test_files/decimal.slt@@ -804,7 +804,7 @@ select log(arrow_cast(100, 'Decimal32(9, 2)'));query Rselect log(2.0, arrow_cast(12345.67, 'Decimal32(9, 2)'));-----13+13.591717513272# log for small decimal64query R@@ -820,7 +820,7 @@ select log(arrow_cast(100, 'Decimal64(18, 2)'));query Rselect log(2.0, arrow_cast(12345.6789, 'Decimal64(15, 4)'));-----13+13.591718553311# log for small decimal128@@ -896,15 +896,13 @@ select log(10::decimal(38, 0), 100000000000000000000000000000000000::decimal(38,query Rselect log(2, 100000000000000000000000000000000000::decimal(38,0));-----116+116.267483321058# log(10^35) for decimal128 with another base-# TODO: this should be 116.267483321058, error with native decimal log impl-# https://github.com/apache/datafusion/issues/18524query Rselect log(2.0, 100000000000000000000000000000000000::decimal(38,0));-----116+116.267483321058# log with non-integer base (fallback to f64)query R@@ -1036,7 +1034,7 @@ from (values (10.0), (2.0), (3.0)) as t(base);query RSELECT log(10, arrow_cast(0.5, 'Decimal32(5, 1)'))-----NaN+-0.301029995664query RSELECT log(10, arrow_cast(1 , 'Decimal32(5, 1)'))@@ -1195,18 +1193,18 @@ select 100000000000000000000000000000000000::decimal(38,0)99999999999999996863366107917975552# log(10^35) for decimal128 with explicit decimal base-# Float parsing is rounding down+# Float parsing is rounding down, but log uses float computation so result rounds to 35query Rselect log(10, 100000000000000000000000000000000000::decimal(38,0));-----34+35# log(10^35) for large decimal128 if parsed as float-# Float parsing is rounding down+# Float parsing is rounding down, but log uses float computation so result rounds to 35query Rselect log(100000000000000000000000000000000000::decimal(38,0))-----34+35# Result is decimal since argument is decimal regardless decimals-as-floats parsingquery R