Apache DataFusion SQL Query Engine
Refactor `log()` signature to use coercion API + fixes (#18519)
Jeffrey Voc826009586b5
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex 52e0159111..fadd2e41ea 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -1734,7 +1734,7 @@ impl ScalarValue {) {return _internal_err!("Invalid precision and scale {err}");}- if *scale <= 0 {+ if *scale < 0 {return _internal_err!("Negative scale is not supported");}match 10_i32.checked_pow((*scale + 1) as u32) {@@ -1750,7 +1750,7 @@ impl ScalarValue {) {return _internal_err!("Invalid precision and scale {err}");}- if *scale <= 0 {+ if *scale < 0 {return _internal_err!("Negative scale is not supported");}match i64::from(10).checked_pow((*scale + 1) as u32) {@@ -4407,6 +4407,7 @@ macro_rules! impl_scalar {impl_scalar!(f64, Float64);impl_scalar!(f32, Float32);+impl_scalar!(f16, Float16);impl_scalar!(i8, Int8);impl_scalar!(i16, Int16);impl_scalar!(i32, Int32);@@ -4563,6 +4564,7 @@ impl_try_from!(UInt8, u8);impl_try_from!(UInt16, u16);impl_try_from!(UInt32, u32);impl_try_from!(UInt64, u64);+impl_try_from!(Float16, f16);impl_try_from!(Float32, f32);impl_try_from!(Float64, f64);impl_try_from!(Boolean, bool);diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rsindex 8c41701ae5..a1495b779a 100644--- a/datafusion/common/src/types/native.rs+++ b/datafusion/common/src/types/native.rs@@ -430,22 +430,7 @@ impl From<DataType> for NativeType {impl NativeType {#[inline]pub fn is_numeric(&self) -> bool {- use NativeType::*;- matches!(- self,- UInt8- | UInt16- | UInt32- | UInt64- | Int8- | Int16- | Int32- | Int64- | Float16- | Float32- | Float64- | Decimal(_, _)- )+ self.is_integer() || self.is_float() || self.is_decimal()}#[inline]@@ -491,4 +476,14 @@ impl NativeType {pub fn is_null(&self) -> bool {matches!(self, NativeType::Null)}++ #[inline]+ pub fn is_decimal(&self) -> bool {+ matches!(self, Self::Decimal(_, _))+ }++ #[inline]+ pub fn is_float(&self) -> bool {+ matches!(self, Self::Float16 | Self::Float32 | Self::Float64)+ }}diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rsindex 2bf7092dd2..6ee1c4a2a4 100644--- a/datafusion/expr-common/src/signature.rs+++ b/datafusion/expr-common/src/signature.rs@@ -21,7 +21,7 @@ use std::fmt::Display;use std::hash::Hash;use crate::type_coercion::aggregates::NUMERICS;-use arrow::datatypes::{DataType, IntervalUnit, TimeUnit};+use arrow::datatypes::{DataType, Decimal128Type, DecimalType, IntervalUnit, TimeUnit};use datafusion_common::types::{LogicalType, LogicalTypeRef, NativeType};use datafusion_common::utils::ListCoercion;use datafusion_common::{internal_err, plan_err, Result};@@ -333,9 +333,10 @@ pub enum TypeSignatureClass {Interval,Duration,Native(LogicalTypeRef),- // TODO:- // NumericInteger,+ Float,+ Decimal,+ Numeric,/// Encompasses both the native Binary as well as arbitrarily sized FixedSizeBinary typesBinary,}@@ -378,6 +379,13 @@ impl TypeSignatureClass {TypeSignatureClass::Binary => {vec![DataType::Binary]}+ TypeSignatureClass::Decimal => vec![Decimal128Type::DEFAULT_TYPE],+ TypeSignatureClass::Float => vec![DataType::Float64],+ TypeSignatureClass::Numeric => vec![+ DataType::Float64,+ DataType::Int64,+ Decimal128Type::DEFAULT_TYPE,+ ],}}@@ -395,6 +403,9 @@ impl TypeSignatureClass {TypeSignatureClass::Duration if logical_type.is_duration() => true,TypeSignatureClass::Integer if logical_type.is_integer() => true,TypeSignatureClass::Binary if logical_type.is_binary() => true,+ TypeSignatureClass::Decimal if logical_type.is_decimal() => true,+ TypeSignatureClass::Float if logical_type.is_float() => true,+ TypeSignatureClass::Numeric if logical_type.is_numeric() => true,_ => false,}}@@ -428,6 +439,15 @@ impl TypeSignatureClass {TypeSignatureClass::Binary if native_type.is_binary() => {Ok(origin_type.to_owned())}+ TypeSignatureClass::Decimal if native_type.is_decimal() => {+ Ok(origin_type.to_owned())+ }+ TypeSignatureClass::Float if native_type.is_float() => {+ Ok(origin_type.to_owned())+ }+ TypeSignatureClass::Numeric if native_type.is_numeric() => {+ Ok(origin_type.to_owned())+ }_ if native_type.is_null() => Ok(origin_type.to_owned()),_ => internal_err!("May miss the matching logic in `matches_native_type`"),}diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rsindex f66f6fcfc1..24000a3876 100644--- a/datafusion/functions/src/math/log.rs+++ b/datafusion/functions/src/math/log.rs@@ -18,18 +18,18 @@//! Math function: `log()`.use std::any::Any;-use std::sync::Arc;use super::power::PowerFunc;use crate::utils::{calculate_binary_math, decimal128_to_i128};use arrow::array::{Array, ArrayRef};+use arrow::compute::kernels::cast;use arrow::datatypes::{- DataType, Decimal128Type, Decimal256Type, Float32Type, Float64Type, Int32Type,- Int64Type, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION,+ DataType, Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type,};use arrow::error::ArrowError;use arrow_buffer::i256;+use datafusion_common::types::NativeType;use datafusion_common::{exec_err, internal_err, plan_datafusion_err, plan_err, Result, ScalarValue,};@@ -37,11 +37,12 @@ use datafusion_expr::expr::ScalarFunction;use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo};use datafusion_expr::sort_properties::{ExprProperties, SortProperties};use datafusion_expr::{- lit, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,- TypeSignature::*,+ lit, Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,+ TypeSignature, TypeSignatureClass,};use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};use datafusion_macros::user_doc;+use num_traits::Float;#[user_doc(doc_section(label = "Math Functions"),@@ -72,37 +73,28 @@ impl Default for LogFunc {impl LogFunc {pub fn new() -> Self {+ // Converts decimals & integers to float64, accepting other floats as is+ let as_float = Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ );Self {signature: Signature::one_of(+ // Ensure decimals have precedence over floats since we have+ // a native decimal implementation for logvec![- Numeric(1),- Numeric(2),- Exact(vec![DataType::Float32, DataType::Float32]),- Exact(vec![DataType::Float64, DataType::Float64]),- Exact(vec![- DataType::Int64,- DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0),- ]),- Exact(vec![- DataType::Float32,- DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0),- ]),- Exact(vec![- DataType::Float64,- DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0),- ]),- Exact(vec![- DataType::Int64,- DataType::Decimal256(DECIMAL256_MAX_PRECISION, 0),- ]),- Exact(vec![- DataType::Float32,- DataType::Decimal256(DECIMAL256_MAX_PRECISION, 0),- ]),- Exact(vec![- DataType::Float64,- DataType::Decimal256(DECIMAL256_MAX_PRECISION, 0),+ // log(value)+ TypeSignature::Coercible(vec![Coercion::new_exact(+ TypeSignatureClass::Decimal,+ )]),+ TypeSignature::Coercible(vec![as_float.clone()]),+ // log(base, value)+ TypeSignature::Coercible(vec![+ as_float.clone(),+ Coercion::new_exact(TypeSignatureClass::Decimal),]),+ TypeSignature::Coercible(vec![as_float.clone(), as_float.clone()]),],Volatility::Immutable,),@@ -160,6 +152,7 @@ impl ScalarUDFImpl for LogFunc {fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {// Check last argument (value)match &arg_types.last().ok_or(plan_datafusion_err!("No args"))? {+ DataType::Float16 => Ok(DataType::Float16),DataType::Float32 => Ok(DataType::Float32),_ => Ok(DataType::Float64),}@@ -192,68 +185,67 @@ impl ScalarUDFImpl for LogFunc {// Support overloaded log(base, x) and log(x) which defaults to log(10, x)fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {- let args = ColumnarValue::values_to_arrays(&args.args)?;+ if args.arg_fields.iter().any(|a| a.data_type().is_null()) {+ return ColumnarValue::Scalar(ScalarValue::Null)+ .cast_to(args.return_type(), None);+ }- let (base, value) = if args.len() == 2 {- // note in f64::log params order is different than in sql. e.g in sql log(base, x) == f64::log(x, base)- (ColumnarValue::Array(Arc::clone(&args[0])), &args[1])+ let (base, value) = if args.args.len() == 2 {+ (args.args[0].clone(), &args.args[1])} else {- // log(num) - assume base is 10- let ret_type = if args[0].data_type().is_null() {- &DataType::Float64- } else {- args[0].data_type()- };+ // no base specified, default to 10(- ColumnarValue::Array(- ScalarValue::new_ten(ret_type)?.to_array_of_size(args[0].len())?,- ),- &args[0],+ ColumnarValue::Scalar(ScalarValue::new_ten(args.return_type())?),+ &args.args[0],)};+ let value = value.to_array(args.number_rows)?;- // All log functors have format 'log(value, base)'- // Therefore, for `calculate_binary_math` the first type means a type of main array- // The second type is the type of the base array (even if derived from main)- let arr: ArrayRef = match value.data_type() {- DataType::Float32 => calculate_binary_math::<- Float32Type,- Float32Type,- Float32Type,- _,- >(value, &base, |x, b| Ok(f32::log(x, b)))?,- DataType::Float64 => calculate_binary_math::<- Float64Type,- Float64Type,- Float64Type,- _,- >(value, &base, |x, b| Ok(f64::log(x, b)))?,- DataType::Int32 => {- calculate_binary_math::<Int32Type, Float64Type, Float64Type, _>(- value,+ let output: ArrayRef = match value.data_type() {+ DataType::Float16 => {+ calculate_binary_math::<Float16Type, Float16Type, Float16Type, _>(+ &value,+ &base,+ |value, base| Ok(value.log(base)),+ )?+ }+ DataType::Float32 => {+ calculate_binary_math::<Float32Type, Float32Type, Float32Type, _>(+ &value,&base,- |x, b| Ok(f64::log(x as f64, b)),+ |value, base| Ok(value.log(base)),)?}- DataType::Int64 => {- calculate_binary_math::<Int64Type, Float64Type, Float64Type, _>(- value,+ DataType::Float64 => {+ calculate_binary_math::<Float64Type, Float64Type, Float64Type, _>(+ &value,&base,- |x, b| Ok(f64::log(x as f64, b)),+ |value, base| Ok(value.log(base)),)?}- DataType::Decimal128(_precision, scale) => {+ // TODO: native log support for decimal 32 & 64; right now upcast+ // to decimal128 to calculate+ // https://github.com/apache/datafusion/issues/17555+ DataType::Decimal32(precision, scale)+ | DataType::Decimal64(precision, scale) => {calculate_binary_math::<Decimal128Type, Float64Type, Float64Type, _>(- value,+ &cast(&value, &DataType::Decimal128(*precision, *scale))?,&base,- |x, b| log_decimal128(x, *scale, b),+ |value, base| log_decimal128(value, *scale, base),)?}- DataType::Decimal256(_precision, scale) => {+ DataType::Decimal128(_, scale) => {+ calculate_binary_math::<Decimal128Type, Float64Type, Float64Type, _>(+ &value,+ &base,+ |value, base| log_decimal128(value, *scale, base),+ )?+ }+ DataType::Decimal256(_, scale) => {calculate_binary_math::<Decimal256Type, Float64Type, Float64Type, _>(- value,+ &value,&base,- |x, b| log_decimal256(x, *scale, b),+ |value, base| log_decimal256(value, *scale, base),)?}other => {@@ -261,7 +253,7 @@ impl ScalarUDFImpl for LogFunc {}};- Ok(ColumnarValue::Array(arr))+ Ok(ColumnarValue::Array(output))}fn documentation(&self) -> Option<&Documentation> {@@ -277,17 +269,28 @@ impl ScalarUDFImpl for LogFunc {mut args: Vec<Expr>,info: &dyn SimplifyInfo,) -> Result<ExprSimplifyResult> {+ let mut arg_types = args+ .iter()+ .map(|arg| info.get_data_type(arg))+ .collect::<Result<Vec<_>>>()?;+ let return_type = self.return_type(&arg_types)?;++ // Null propagation+ if arg_types.iter().any(|dt| dt.is_null()) {+ return Ok(ExprSimplifyResult::Simplified(lit(+ ScalarValue::Null.cast_to(&return_type)?+ )));+ }+// Args are either// log(number)// log(base, number)let num_args = args.len();- if num_args > 2 {+ if num_args != 1 && num_args != 2 {return plan_err!("Expected log to have 1 or 2 arguments, got {num_args}");}- let number = args.pop().ok_or_else(|| {- plan_datafusion_err!("Expected log to have 1 or 2 arguments, got 0")- })?;- let number_datatype = info.get_data_type(&number)?;+ let number = args.pop().unwrap();+ let number_datatype = arg_types.pop().unwrap();// default to base 10let base = if let Some(base) = args.pop() {base@@ -339,6 +342,7 @@ fn is_pow(func: &ScalarUDF) -> bool {#[cfg(test)]mod tests {use std::collections::HashMap;+ use std::sync::Arc;use super::*;@@ -353,6 +357,18 @@ mod tests {use datafusion_expr::execution_props::ExecutionProps;use datafusion_expr::simplify::SimplifyContext;+ #[test]+ fn test_log_decimal_native() {+ let value = 10_i128.pow(35);+ assert_eq!((value as f64).log2(), 116.26748332105768);+ assert_eq!(+ log_decimal128(value, 0, 2.0).unwrap(),+ // TODO: see we're losing our decimal points compared to above+ // https://github.com/apache/datafusion/issues/18524+ 116.0+ );+ }+#[test]fn test_log_invalid_base_type() {let arg_fields = vec![diff --git a/datafusion/sqllogictest/test_files/decimal.slt b/datafusion/sqllogictest/test_files/decimal.sltindex 502821fcc3..f350d9b3bf 100644--- a/datafusion/sqllogictest/test_files/decimal.slt+++ b/datafusion/sqllogictest/test_files/decimal.slt@@ -794,15 +794,47 @@ select 100000000000000000000000000000000000::decimal(38,0), arrow_typeof(1000000----100000000000000000000000000000000000 Decimal128(38, 0)+# log for small decimal32+query R+select log(arrow_cast(100, 'Decimal32(9, 0)'));+----+2++query R+select log(arrow_cast(100, 'Decimal32(9, 2)'));+----+2++# log for small decimal64+query R+select log(arrow_cast(100, 'Decimal64(18, 0)'));+----+2++query R+select log(arrow_cast(100, 'Decimal64(18, 2)'));+----+2+# log for small decimal128query R-select log(100::decimal(38,0));+select log(arrow_cast(100, 'Decimal128(38, 0)'));+----+2++query R+select log(arrow_cast(100, 'Decimal128(38, 2)'));----2# log for small decimal256query R-select log(100::decimal(76,0));+select log(arrow_cast(100, 'Decimal256(76, 0)'));+----+2++query R+select log(arrow_cast(100, 'Decimal256(76, 2)'));----2@@ -858,10 +890,12 @@ select log(2, 100000000000000000000000000000000000::decimal(38,0));116# log(10^35) for decimal128 with another base+# TODO: this should be 116.267483321058, error with native decimal log impl+# https://github.com/apache/datafusion/issues/18524query Rselect log(2.0, 100000000000000000000000000000000000::decimal(38,0));-----116.267483321058+116# null casesquery Rdiff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.sltindex 1cb68b85b2..edba5354e0 100644--- a/datafusion/sqllogictest/test_files/math.slt+++ b/datafusion/sqllogictest/test_files/math.slt@@ -705,3 +705,30 @@ select FACTORIAL(350943270);statement okdrop table signed_integers++# Null propagation for log+query TT+EXPLAIN SELECT log(NULL, c2) from aggregate_simple;+----+logical_plan+01)Projection: Float64(NULL) AS log(NULL,aggregate_simple.c2)+02)--TableScan: aggregate_simple projection=[]+physical_plan+01)ProjectionExec: expr=[NULL as log(NULL,aggregate_simple.c2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_simple.csv]]}, file_type=csv, has_header=true++# Float 16/32/64 for log+query RT+SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));+----+2.6074219 Float16++query RT+SELECT log(2.5, 10.9::float), arrow_typeof(log(2.5, 10.9::float));+----+2.606992 Float32++query RT+SELECT log(2.5, 10.9::double), arrow_typeof(log(2.5, 10.9::double));+----+2.606992198152 Float64diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 04a7615c76..a73f56079e 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -675,11 +675,11 @@ query TT----logical_plan01)Sort: log_c11_base_c12 ASC NULLS LAST-02)--Projection: log(aggregate_test_100.c12, CAST(aggregate_test_100.c11 AS Float64)) AS log_c11_base_c12+02)--Projection: log(aggregate_test_100.c12, aggregate_test_100.c11) AS log_c11_base_c1203)----TableScan: aggregate_test_100 projection=[c11, c12]physical_plan01)SortPreservingMergeExec: [log_c11_base_c12@0 ASC NULLS LAST]-02)--ProjectionExec: expr=[log(c12@1, CAST(c11@0 AS Float64)) as log_c11_base_c12]+02)--ProjectionExec: expr=[log(c12@1, c11@0) as log_c11_base_c12]03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=104)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true@@ -690,11 +690,11 @@ ORDER BY log_c12_base_c11 DESC NULLS LAST;----logical_plan01)Sort: log_c12_base_c11 DESC NULLS LAST-02)--Projection: log(CAST(aggregate_test_100.c11 AS Float64), aggregate_test_100.c12) AS log_c12_base_c11+02)--Projection: log(aggregate_test_100.c11, aggregate_test_100.c12) AS log_c12_base_c1103)----TableScan: aggregate_test_100 projection=[c11, c12]physical_plan01)SortPreservingMergeExec: [log_c12_base_c11@0 DESC NULLS LAST]-02)--ProjectionExec: expr=[log(CAST(c11@0 AS Float64), c12@1) as log_c12_base_c11]+02)--ProjectionExec: expr=[log(c11@0, c12@1) as log_c12_base_c11]03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=104)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c11, c12], output_orderings=[[c11@0 ASC NULLS LAST], [c12@1 DESC NULLS LAST]], file_type=csv, has_header=true@@ -1024,10 +1024,10 @@ ORDER BY SUM(column1)# ORDER BY with a GROUP BY clausequery I-SELECT SUM(column1)- FROM foo-GROUP BY column2-ORDER BY SUM(column1)+SELECT SUM(column1)+ FROM foo+GROUP BY column2+ORDER BY SUM(column1)----02@@ -1039,12 +1039,12 @@ ORDER BY SUM(column1)# ORDER BY with a GROUP BY clause and a HAVING clausequery I-SELECT- SUM(column1)-FROM foo-GROUP BY column2-HAVING SUM(column1) < 3-ORDER BY SUM(column1)+SELECT+ SUM(column1)+FROM foo+GROUP BY column2+HAVING SUM(column1) < 3+ORDER BY SUM(column1)----02@@ -1179,7 +1179,7 @@ physical_plan04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=truestatement ok-drop table ordered_table;+drop table ordered_table;# ABS(x) breaks the ordering if x's range contains both negative and positive values.@@ -1215,7 +1215,7 @@ physical_plan05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=truestatement ok-drop table ordered_table;+drop table ordered_table;# ABS(x) preserves the ordering if x's range falls into positive values.# Since x is defined as INT UNSIGNED, its range is assumed to be from 0 to INF.