Skip to content

Apache DataFusion SQL Query Engine

Refactor `log()` signature to use coercion API + fixes (#18519)

Jeffrey Voc826009586b5
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs
index 52e0159111..fadd2e41ea 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -1734,7 +1734,7 @@ impl ScalarValue {
) {
return _internal_err!("Invalid precision and scale {err}");
}
- if *scale <= 0 {
+ if *scale < 0 {
return _internal_err!("Negative scale is not supported");
}
match 10_i32.checked_pow((*scale + 1) as u32) {
@@ -1750,7 +1750,7 @@ impl ScalarValue {
) {
return _internal_err!("Invalid precision and scale {err}");
}
- if *scale <= 0 {
+ if *scale < 0 {
return _internal_err!("Negative scale is not supported");
}
match i64::from(10).checked_pow((*scale + 1) as u32) {
@@ -4407,6 +4407,7 @@ macro_rules! impl_scalar {
impl_scalar!(f64, Float64);
impl_scalar!(f32, Float32);
+impl_scalar!(f16, Float16);
impl_scalar!(i8, Int8);
impl_scalar!(i16, Int16);
impl_scalar!(i32, Int32);
@@ -4563,6 +4564,7 @@ impl_try_from!(UInt8, u8);
impl_try_from!(UInt16, u16);
impl_try_from!(UInt32, u32);
impl_try_from!(UInt64, u64);
+impl_try_from!(Float16, f16);
impl_try_from!(Float32, f32);
impl_try_from!(Float64, f64);
impl_try_from!(Boolean, bool);
diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rs
index 8c41701ae5..a1495b779a 100644
--- a/datafusion/common/src/types/native.rs
+++ b/datafusion/common/src/types/native.rs
@@ -430,22 +430,7 @@ impl From<DataType> for NativeType {
impl NativeType {
#[inline]
pub fn is_numeric(&self) -> bool {
- use NativeType::*;
- matches!(
- self,
- UInt8
- | UInt16
- | UInt32
- | UInt64
- | Int8
- | Int16
- | Int32
- | Int64
- | Float16
- | Float32
- | Float64
- | Decimal(_, _)
- )
+ self.is_integer() || self.is_float() || self.is_decimal()
}
#[inline]
@@ -491,4 +476,14 @@ impl NativeType {
pub fn is_null(&self) -> bool {
matches!(self, NativeType::Null)
}
+
+ #[inline]
+ pub fn is_decimal(&self) -> bool {
+ matches!(self, Self::Decimal(_, _))
+ }
+
+ #[inline]
+ pub fn is_float(&self) -> bool {
+ matches!(self, Self::Float16 | Self::Float32 | Self::Float64)
+ }
}
diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs
index 2bf7092dd2..6ee1c4a2a4 100644
--- a/datafusion/expr-common/src/signature.rs
+++ b/datafusion/expr-common/src/signature.rs
@@ -21,7 +21,7 @@ use std::fmt::Display;
use std::hash::Hash;
use crate::type_coercion::aggregates::NUMERICS;
-use arrow::datatypes::{DataType, IntervalUnit, TimeUnit};
+use arrow::datatypes::{DataType, Decimal128Type, DecimalType, IntervalUnit, TimeUnit};
use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType};
use datafusion_common::utils::ListCoercion;
use datafusion_common::{internal_err, plan_err, Result};
@@ -333,9 +333,10 @@ pub enum TypeSignatureClass {
Interval,
Duration,
Native(LogicalTypeRef),
- // TODO:
- // Numeric
Integer,
+ Float,
+ Decimal,
+ Numeric,
/// Encompasses both the native Binary as well as arbitrarily sized FixedSizeBinary types
Binary,
}
@@ -378,6 +379,13 @@ impl TypeSignatureClass {
TypeSignatureClass::Binary => {
vec![DataType::Binary]
}
+ TypeSignatureClass::Decimal => vec![Decimal128Type::DEFAULT_TYPE],
+ TypeSignatureClass::Float => vec![DataType::Float64],
+ TypeSignatureClass::Numeric => vec![
+ DataType::Float64,
+ DataType::Int64,
+ Decimal128Type::DEFAULT_TYPE,
+ ],
}
}
@@ -395,6 +403,9 @@ impl TypeSignatureClass {
TypeSignatureClass::Duration if logical_type.is_duration() => true,
TypeSignatureClass::Integer if logical_type.is_integer() => true,
TypeSignatureClass::Binary if logical_type.is_binary() => true,
+ TypeSignatureClass::Decimal if logical_type.is_decimal() => true,
+ TypeSignatureClass::Float if logical_type.is_float() => true,
+ TypeSignatureClass::Numeric if logical_type.is_numeric() => true,
_ => false,
}
}
@@ -428,6 +439,15 @@ impl TypeSignatureClass {
TypeSignatureClass::Binary if native_type.is_binary() => {
Ok(origin_type.to_owned())
}
+ TypeSignatureClass::Decimal if native_type.is_decimal() => {
+ Ok(origin_type.to_owned())
+ }
+ TypeSignatureClass::Float if native_type.is_float() => {
+ Ok(origin_type.to_owned())
+ }
+ TypeSignatureClass::Numeric if native_type.is_numeric() => {
+ Ok(origin_type.to_owned())
+ }
_ if native_type.is_null() => Ok(origin_type.to_owned()),
_ => internal_err!("May miss the matching logic in `matches_native_type`"),
}
diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rs
index f66f6fcfc1..24000a3876 100644
--- a/datafusion/functions/src/math/log.rs
+++ b/datafusion/functions/src/math/log.rs
@@ -18,18 +18,18 @@
//! Math function: `log()`.
use std::any::Any;
-use std::sync::Arc;
use super::power::PowerFunc;
use crate::utils::{calculate_binary_math, decimal128_to_i128};
use arrow::array::{Array, ArrayRef};
+use arrow::compute::kernels::cast;
use arrow::datatypes::{
- DataType, Decimal128Type, Decimal256Type, Float32Type, Float64Type, Int32Type,
- Int64Type, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION,
+ DataType, Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type,
};
use arrow::error::ArrowError;
use arrow_buffer::i256;
+use datafusion_common::types::NativeType;
use datafusion_common::{
exec_err, internal_err, plan_datafusion_err, plan_err, Result, ScalarValue,
};
@@ -37,11 +37,12 @@ use datafusion_expr::expr::ScalarFunction;
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo};
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
use datafusion_expr::{
- lit, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,
- TypeSignature::*,
+ lit, Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,
+ TypeSignature, TypeSignatureClass,
};
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
use datafusion_macros::user_doc;
+use num_traits::Float;
#[user_doc(
doc_section(label = "Math Functions"),
@@ -72,37 +73,28 @@ impl Default for LogFunc {
impl LogFunc {
pub fn new() -> Self {
+ // Converts decimals & integers to float64, accepting other floats as is
+ let as_float = Coercion::new_implicit(
+ TypeSignatureClass::Float,
+ vec![TypeSignatureClass::Numeric],
+ NativeType::Float64,
+ );
Self {
signature: Signature::one_of(
+ // Ensure decimals have precedence over floats since we have
+ // a native decimal implementation for log
vec![
- Numeric(1),
- Numeric(2),
- Exact(vec![DataType::Float32, DataType::Float32]),
- Exact(vec![DataType::Float64, DataType::Float64]),
- Exact(vec![
- DataType::Int64,
- DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0),
- ]),
- Exact(vec![
- DataType::Float32,
- DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0),
- ]),
- Exact(vec![
- DataType::Float64,
- DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0),
- ]),
- Exact(vec![
- DataType::Int64,
- DataType::Decimal256(DECIMAL256_MAX_PRECISION, 0),
- ]),
- Exact(vec![
- DataType::Float32,
- DataType::Decimal256(DECIMAL256_MAX_PRECISION, 0),
- ]),
- Exact(vec![
- DataType::Float64,
- DataType::Decimal256(DECIMAL256_MAX_PRECISION, 0),
+ // log(value)
+ TypeSignature::Coercible(vec![Coercion::new_exact(
+ TypeSignatureClass::Decimal,
+ )]),
+ TypeSignature::Coercible(vec![as_float.clone()]),
+ // log(base, value)
+ TypeSignature::Coercible(vec![
+ as_float.clone(),
+ Coercion::new_exact(TypeSignatureClass::Decimal),
]),
+ TypeSignature::Coercible(vec![as_float.clone(), as_float.clone()]),
],
Volatility::Immutable,
),
@@ -160,6 +152,7 @@ impl ScalarUDFImpl for LogFunc {
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
// Check last argument (value)
match &arg_types.last().ok_or(plan_datafusion_err!("No args"))? {
+ DataType::Float16 => Ok(DataType::Float16),
DataType::Float32 => Ok(DataType::Float32),
_ => Ok(DataType::Float64),
}
@@ -192,68 +185,67 @@ impl ScalarUDFImpl for LogFunc {
// Support overloaded log(base, x) and log(x) which defaults to log(10, x)
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
- let args = ColumnarValue::values_to_arrays(&args.args)?;
+ if args.arg_fields.iter().any(|a| a.data_type().is_null()) {
+ return ColumnarValue::Scalar(ScalarValue::Null)
+ .cast_to(args.return_type(), None);
+ }
- let (base, value) = if args.len() == 2 {
- // note in f64::log params order is different than in sql. e.g in sql log(base, x) == f64::log(x, base)
- (ColumnarValue::Array(Arc::clone(&args[0])), &args[1])
+ let (base, value) = if args.args.len() == 2 {
+ (args.args[0].clone(), &args.args[1])
} else {
- // log(num) - assume base is 10
- let ret_type = if args[0].data_type().is_null() {
- &DataType::Float64
- } else {
- args[0].data_type()
- };
+ // no base specified, default to 10
(
- ColumnarValue::Array(
- ScalarValue::new_ten(ret_type)?.to_array_of_size(args[0].len())?,
- ),
- &args[0],
+ ColumnarValue::Scalar(ScalarValue::new_ten(args.return_type())?),
+ &args.args[0],
)
};
+ let value = value.to_array(args.number_rows)?;
- // All log functors have format 'log(value, base)'
- // Therefore, for `calculate_binary_math` the first type means a type of main array
- // The second type is the type of the base array (even if derived from main)
- let arr: ArrayRef = match value.data_type() {
- DataType::Float32 => calculate_binary_math::<
- Float32Type,
- Float32Type,
- Float32Type,
- _,
- >(value, &base, |x, b| Ok(f32::log(x, b)))?,
- DataType::Float64 => calculate_binary_math::<
- Float64Type,
- Float64Type,
- Float64Type,
- _,
- >(value, &base, |x, b| Ok(f64::log(x, b)))?,
- DataType::Int32 => {
- calculate_binary_math::<Int32Type, Float64Type, Float64Type, _>(
- value,
+ let output: ArrayRef = match value.data_type() {
+ DataType::Float16 => {
+ calculate_binary_math::<Float16Type, Float16Type, Float16Type, _>(
+ &value,
+ &base,
+ |value, base| Ok(value.log(base)),
+ )?
+ }
+ DataType::Float32 => {
+ calculate_binary_math::<Float32Type, Float32Type, Float32Type, _>(
+ &value,
&base,
- |x, b| Ok(f64::log(x as f64, b)),
+ |value, base| Ok(value.log(base)),
)?
}
- DataType::Int64 => {
- calculate_binary_math::<Int64Type, Float64Type, Float64Type, _>(
- value,
+ DataType::Float64 => {
+ calculate_binary_math::<Float64Type, Float64Type, Float64Type, _>(
+ &value,
&base,
- |x, b| Ok(f64::log(x as f64, b)),
+ |value, base| Ok(value.log(base)),
)?
}
- DataType::Decimal128(_precision, scale) => {
+ // TODO: native log support for decimal 32 & 64; right now upcast
+ // to decimal128 to calculate
+ // https://github.com/apache/datafusion/issues/17555
+ DataType::Decimal32(precision, scale)
+ | DataType::Decimal64(precision, scale) => {
calculate_binary_math::<Decimal128Type, Float64Type, Float64Type, _>(
- value,
+ &cast(&value, &DataType::Decimal128(*precision, *scale))?,
&base,
- |x, b| log_decimal128(x, *scale, b),
+ |value, base| log_decimal128(value, *scale, base),
)?
}
- DataType::Decimal256(_precision, scale) => {
+ DataType::Decimal128(_, scale) => {
+ calculate_binary_math::<Decimal128Type, Float64Type, Float64Type, _>(
+ &value,
+ &base,
+ |value, base| log_decimal128(value, *scale, base),
+ )?
+ }
+ DataType::Decimal256(_, scale) => {
calculate_binary_math::<Decimal256Type, Float64Type, Float64Type, _>(
- value,
+ &value,
&base,
- |x, b| log_decimal256(x, *scale, b),
+ |value, base| log_decimal256(value, *scale, base),
)?
}
other => {
@@ -261,7 +253,7 @@ impl ScalarUDFImpl for LogFunc {
}
};
- Ok(ColumnarValue::Array(arr))
+ Ok(ColumnarValue::Array(output))
}
fn documentation(&self) -> Option<&Documentation> {
@@ -277,17 +269,28 @@ impl ScalarUDFImpl for LogFunc {
mut args: Vec<Expr>,
info: &dyn SimplifyInfo,
) -> Result<ExprSimplifyResult> {
+ let mut arg_types = args
+ .iter()
+ .map(|arg| info.get_data_type(arg))
+ .collect::<Result<Vec<_>>>()?;
+ let return_type = self.return_type(&arg_types)?;
+
+ // Null propagation
+ if arg_types.iter().any(|dt| dt.is_null()) {
+ return Ok(ExprSimplifyResult::Simplified(lit(
+ ScalarValue::Null.cast_to(&return_type)?
+ )));
+ }
+
// Args are either
// log(number)
// log(base, number)
let num_args = args.len();
- if num_args > 2 {
+ if num_args != 1 && num_args != 2 {
return plan_err!("Expected log to have 1 or 2 arguments, got {num_args}");
}
- let number = args.pop().ok_or_else(|| {
- plan_datafusion_err!("Expected log to have 1 or 2 arguments, got 0")
- })?;
- let number_datatype = info.get_data_type(&number)?;
+ let number = args.pop().unwrap();
+ let number_datatype = arg_types.pop().unwrap();
// default to base 10
let base = if let Some(base) = args.pop() {
base
@@ -339,6 +342,7 @@ fn is_pow(func: &ScalarUDF) -> bool {
#[cfg(test)]
mod tests {
use std::collections::HashMap;
+ use std::sync::Arc;
use super::*;
@@ -353,6 +357,18 @@ mod tests {
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::simplify::SimplifyContext;
+ #[test]
+ fn test_log_decimal_native() {
+ let value = 10_i128.pow(35);
+ assert_eq!((value as f64).log2(), 116.26748332105768);
+ assert_eq!(
+ log_decimal128(value, 0, 2.0).unwrap(),
+ // TODO: see we're losing our decimal points compared to above
+ // https://github.com/apache/datafusion/issues/18524
+ 116.0
+ );
+ }
+
#[test]
fn test_log_invalid_base_type() {
let arg_fields = vec![
diff --git a/datafusion/sqllogictest/test_files/decimal.slt b/datafusion/sqllogictest/test_files/decimal.slt
index 502821fcc3..f350d9b3bf 100644
--- a/datafusion/sqllogictest/test_files/decimal.slt
+++ b/datafusion/sqllogictest/test_files/decimal.slt
@@ -794,15 +794,47 @@ select 100000000000000000000000000000000000::decimal(38,0), arrow_typeof(1000000
----
100000000000000000000000000000000000 Decimal128(38, 0)
+# log for small decimal32
+query R
+select log(arrow_cast(100, 'Decimal32(9, 0)'));
+----
+2
+
+query R
+select log(arrow_cast(100, 'Decimal32(9, 2)'));
+----
+2
+
+# log for small decimal64
+query R
+select log(arrow_cast(100, 'Decimal64(18, 0)'));
+----
+2
+
+query R
+select log(arrow_cast(100, 'Decimal64(18, 2)'));
+----
+2
+
# log for small decimal128
query R
-select log(100::decimal(38,0));
+select log(arrow_cast(100, 'Decimal128(38, 0)'));
+----
+2
+
+query R
+select log(arrow_cast(100, 'Decimal128(38, 2)'));
----
2
# log for small decimal256
query R
-select log(100::decimal(76,0));
+select log(arrow_cast(100, 'Decimal256(76, 0)'));
+----
+2
+
+query R
+select log(arrow_cast(100, 'Decimal256(76, 2)'));
----
2
@@ -858,10 +890,12 @@ select log(2, 100000000000000000000000000000000000::decimal(38,0));
116
# log(10^35) for decimal128 with another base
+# TODO: this should be 116.267483321058, error with native decimal log impl
+# https://github.com/apache/datafusion/issues/18524
query R
select log(2.0, 100000000000000000000000000000000000::decimal(38,0));
----
-116.267483321058
+116
# null cases
query R
diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt
index 1cb68b85b2..edba5354e0 100644
--- a/datafusion/sqllogictest/test_files/math.slt
+++ b/datafusion/sqllogictest/test_files/math.slt
@@ -705,3 +705,30 @@ select FACTORIAL(350943270);
statement ok
drop table signed_integers
+
+# Null propagation for log
+query TT
+EXPLAIN SELECT log(NULL, c2) from aggregate_simple;
+----
+logical_plan
+01)Projection: Float64(NULL) AS log(NULL,aggregate_simple.c2)
+02)--TableScan: aggregate_simple projection=[]
+physical_plan
+01)ProjectionExec: expr=[NULL as log(NULL,aggregate_simple.c2)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_simple.csv]]}, file_type=csv, has_header=true
+
+# Float 16/32/64 for log
+query RT
+SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));
+----
+2.6074219 Float16
+
+query RT
+SELECT log(2.5, 10.9::float), arrow_typeof(log(2.5, 10.9::float));
+----
+2.606992 Float32
+
+query RT
+SELECT log(2.5, 10.9::double), arrow_typeof(log(2.5, 10.9::double));
+----
+2.606992198152 Float64
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index 04a7615c76..a73f56079e 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -675,11 +675,11 @@ query TT
----
logical_plan
01)Sort: log_c11_base_c12 ASC NULLS LAST
-02)--Projection: log(aggregate_test_100.c12, CAST(aggregate_test_100.c11 AS Float64)) AS log_c11_base_c12
+02)--Projection: log(aggregate_test_100.c12, aggregate_test_100.c11) AS log_c11_base_c12
03)----TableScan: aggregate_test_100 projection=[c11, c12]
physical_plan
01)SortPreservingMergeExec: [log_c11_base_c12@0 ASC NULLS LAST]
-02)--ProjectionExec: expr=[log(c12@1, CAST(c11@0 AS Float64)) as log_c11_base_c12]
+02)--ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]
03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
@@ -690,11 +690,11 @@ ORDER BY log_c12_base_c11 DESC NULLS LAST;
----
logical_plan
01)Sort: log_c12_base_c11 DESC NULLS LAST
-02)--Projection: log(CAST(aggregate_test_100.c11 AS Float64), aggregate_test_100.c12) AS log_c12_base_c11
+02)--Projection: log(aggregate_test_100.c11, aggregate_test_100.c12) AS log_c12_base_c11
03)----TableScan: aggregate_test_100 projection=[c11, c12]
physical_plan
01)SortPreservingMergeExec: [log_c12_base_c11@0 DESC NULLS LAST]
-02)--ProjectionExec: expr=[log(CAST(c11@0 AS Float64), c12@1) as log_c12_base_c11]
+02)--ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]
03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true
@@ -1024,10 +1024,10 @@ ORDER BY SUM(column1)
# ORDER BY with a GROUP BY clause
query I
-SELECT SUM(column1)
- FROM foo
-GROUP BY column2
-ORDER BY SUM(column1)
+SELECT SUM(column1)
+ FROM foo
+GROUP BY column2
+ORDER BY SUM(column1)
----
0
2
@@ -1039,12 +1039,12 @@ ORDER BY SUM(column1)
# ORDER BY with a GROUP BY clause and a HAVING clause
query I
-SELECT
- SUM(column1)
-FROM foo
-GROUP BY column2
-HAVING SUM(column1) < 3
-ORDER BY SUM(column1)
+SELECT
+ SUM(column1)
+FROM foo
+GROUP BY column2
+HAVING SUM(column1) < 3
+ORDER BY SUM(column1)
----
0
2
@@ -1179,7 +1179,7 @@ physical_plan
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true
statement ok
-drop table ordered_table;
+drop table ordered_table;
# ABS(x) breaks the ordering if x's range contains both negative and positive values.
@@ -1215,7 +1215,7 @@ physical_plan
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true
statement ok
-drop table ordered_table;
+drop table ordered_table;
# ABS(x) preserves the ordering if x's range falls into positive values.
# Since x is defined as INT UNSIGNED, its range is assumed to be from 0 to INF.