Apache DataFusion SQL Query Engine
feat: support spark compatible floor function (#21933)
Krishna Sudarshan J015162afac7e
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex 72bca58b7a..b03a732107 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -99,3 +99,7 @@ name = "unhex"[[bench]]harness = falsename = "sha2"++[[bench]]+harness = false+name = "floor"diff --git a/datafusion/spark/benches/floor.rs b/datafusion/spark/benches/floor.rsnew file mode 100644index 0000000000..ecb1590acc--- /dev/null+++ b/datafusion/spark/benches/floor.rs@@ -0,0 +1,119 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::*;+use arrow::datatypes::*;+use criterion::{Criterion, criterion_group, criterion_main};+use datafusion_common::config::ConfigOptions;+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};+use datafusion_spark::function::math::floor::SparkFloor;+use rand::rngs::StdRng;+use rand::{Rng, SeedableRng};+use std::hint::black_box;+use std::sync::Arc;++fn seedable_rng() -> StdRng {+ StdRng::seed_from_u64(42)+}++fn generate_float64_data(size: usize, null_density: f32) -> Float64Array {+ let mut rng = seedable_rng();+ (0..size)+ .map(|_| {+ if rng.random::<f32>() < null_density {+ None+ } else {+ Some(rng.random_range::<f64, _>(-1_000_000.0..1_000_000.0))+ }+ })+ .collect()+}++fn generate_decimal128_data(size: usize, null_density: f32) -> Decimal128Array {+ let mut rng = seedable_rng();+ let array: Decimal128Array = (0..size)+ .map(|_| {+ if rng.random::<f32>() < null_density {+ None+ } else {+ Some(rng.random_range::<i128, _>(-999_999_999..999_999_999))+ }+ })+ .collect();+ array.with_precision_and_scale(18, 2).unwrap()+}++fn run_benchmark(+ c: &mut Criterion,+ name: &str,+ size: usize,+ array: Arc<dyn Array>,+ return_type: &DataType,+) {+ let floor_func = SparkFloor::new();+ let args = vec![ColumnarValue::Array(array)];+ let arg_fields: Vec<_> = args+ .iter()+ .enumerate()+ .map(|(idx, arg)| Field::new(format!("arg_{idx}"), arg.data_type(), true).into())+ .collect();+ let config_options = Arc::new(ConfigOptions::default());++ c.bench_function(&format!("{name}/size={size}"), |b| {+ b.iter(|| {+ black_box(+ floor_func+ .invoke_with_args(ScalarFunctionArgs {+ args: args.clone(),+ arg_fields: arg_fields.clone(),+ number_rows: size,+ return_field: Arc::new(Field::new(+ "f",+ return_type.clone(),+ true,+ )),+ config_options: Arc::clone(&config_options),+ })+ .unwrap(),+ )+ })+ });+}++fn criterion_benchmark(c: &mut Criterion) {+ let sizes = vec![1024, 4096, 8192];+ let null_density = 0.1;++ for &size in &sizes {+ let data = generate_float64_data(size, null_density);+ run_benchmark(c, "floor_float64", size, Arc::new(data), &DataType::Int64);+ }++ for &size in &sizes {+ let data = generate_decimal128_data(size, null_density);+ run_benchmark(+ c,+ "floor_decimal128",+ size,+ Arc::new(data),+ &DataType::Decimal128(17, 0),+ );+ }+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/datafusion/spark/src/function/math/floor.rs b/datafusion/spark/src/function/math/floor.rsnew file mode 100644index 0000000000..703f81a2c2--- /dev/null+++ b/datafusion/spark/src/function/math/floor.rs@@ -0,0 +1,182 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::cast::AsArray;+use arrow::array::types::Decimal128Type;+use arrow::array::{ArrowNativeTypeOp, Decimal128Array, Int64Array};+use arrow::compute::kernels::arity::unary;+use arrow::datatypes::{DataType, Field, FieldRef};+use datafusion_common::{DataFusionError, ScalarValue, exec_err, internal_err};+use datafusion_expr::{+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,+ Volatility,+};+use std::sync::Arc;++/// Spark-compatible `floor` function.+///+/// Differences from DataFusion's floor:+/// - Returns Int64 for float and integer inputs (while DataFusion preserves input type)+/// - For Decimal128(p, s), returns Decimal128(p-s+1, 0) with scale 0+/// (DataFusion preserves original precision and scale)+///+/// <https://spark.apache.org/docs/latest/api/sql/index.html#floor>+#[derive(Debug, PartialEq, Eq, Hash)]+pub struct SparkFloor {+ signature: Signature,+}++impl Default for SparkFloor {+ fn default() -> Self {+ Self::new()+ }+}++impl SparkFloor {+ pub fn new() -> Self {+ Self {+ signature: Signature::numeric(1, Volatility::Immutable),+ }+ }+}++impl ScalarUDFImpl for SparkFloor {+ fn name(&self) -> &str {+ "floor"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(+ &self,+ _arg_types: &[DataType],+ ) -> datafusion_common::Result<DataType> {+ internal_err!("return_field_from_args should be called instead")+ }++ fn return_field_from_args(+ &self,+ args: ReturnFieldArgs,+ ) -> datafusion_common::Result<FieldRef> {+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());+ let return_type = match args.arg_fields[0].data_type() {+ DataType::Decimal128(p, s) if *s > 0 => {+ let new_p = (*p - *s as u8 + 1).clamp(1, 38);+ DataType::Decimal128(new_p, 0)+ }+ DataType::Decimal128(p, s) => DataType::Decimal128(*p, *s),+ DataType::Float32+ | DataType::Float64+ | DataType::Int8+ | DataType::Int16+ | DataType::Int32+ | DataType::Int64 => DataType::Int64,+ _ => exec_err!(+ "found unsupported return type {:?}",+ args.arg_fields[0].data_type()+ )?,+ };+ Ok(Arc::new(Field::new(self.name(), return_type, nullable)))+ }++ fn invoke_with_args(+ &self,+ args: ScalarFunctionArgs,+ ) -> datafusion_common::Result<ColumnarValue> {+ spark_floor(&args.args, args.return_field.data_type())+ }+}++macro_rules! apply_int64 {+ ($value:expr, $arr_type:ty, $scalar_variant:path, $f:expr) => {+ match $value {+ ColumnarValue::Array(array) => {+ let result: Int64Array = unary(array.as_primitive::<$arr_type>(), $f);+ Ok(ColumnarValue::Array(Arc::new(result)))+ }+ ColumnarValue::Scalar($scalar_variant(v)) => {+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(v.map($f))))+ }+ other => internal_err!(+ "floor: data type mismatch — expected scalar of type {} but got {:?}",+ stringify!($scalar_variant),+ other.data_type()+ ),+ }+ };+}++fn spark_floor(+ args: &[ColumnarValue],+ return_type: &DataType,+) -> Result<ColumnarValue, DataFusionError> {+ let value = &args[0];+ match value.data_type() {+ DataType::Float32 => apply_int64!(+ value,+ arrow::datatypes::Float32Type,+ ScalarValue::Float32,+ |x| x.floor() as i64+ ),+ DataType::Float64 => apply_int64!(+ value,+ arrow::datatypes::Float64Type,+ ScalarValue::Float64,+ |x| x.floor() as i64+ ),+ DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => {+ value.cast_to(&DataType::Int64, None)+ }+ DataType::Decimal128(_, scale) if scale > 0 => {+ let divisor = 10_i128.pow_wrapping(scale as u32);+ let floor_decimal = |x: i128| {+ let (d, r) = (x / divisor, x % divisor);+ if r < 0 { d - 1 } else { d }+ };+ match value {+ ColumnarValue::Array(array) => {+ let result: Decimal128Array =+ unary(array.as_primitive::<Decimal128Type>(), floor_decimal);+ Ok(ColumnarValue::Array(Arc::new(+ result.with_data_type(return_type.clone()),+ )))+ }+ ColumnarValue::Scalar(ScalarValue::Decimal128(v, _, _)) => {+ let DataType::Decimal128(new_p, new_s) = return_type else {+ return internal_err!(+ "floor: data type mismatch — expected Decimal128 return type but got {:?}",+ return_type+ );+ };+ Ok(ColumnarValue::Scalar(ScalarValue::Decimal128(+ v.map(floor_decimal),+ *new_p,+ *new_s,+ )))+ }+ other => internal_err!(+ "floor: data type mismatch — expected Decimal128 scalar but got {:?}",+ other.data_type()+ ),+ }+ }+ DataType::Decimal128(_, _) => Ok(value.clone()),+ other => exec_err!("Unsupported data type {other:?} for function floor"),+ }+}diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rsindex 3f6a3a686d..896eedd033 100644--- a/datafusion/spark/src/function/math/mod.rs+++ b/datafusion/spark/src/function/math/mod.rs@@ -20,6 +20,7 @@ pub mod bin;pub mod ceil;pub mod expm1;pub mod factorial;+pub mod floor;pub mod hex;pub mod modulus;pub mod negative;@@ -37,6 +38,7 @@ make_udf_function!(abs::SparkAbs, abs);make_udf_function!(ceil::SparkCeil, ceil);make_udf_function!(expm1::SparkExpm1, expm1);make_udf_function!(factorial::SparkFactorial, factorial);+make_udf_function!(floor::SparkFloor, floor);make_udf_function!(hex::SparkHex, hex);make_udf_function!(modulus::SparkMod, modulus);make_udf_function!(modulus::SparkPmod, pmod);@@ -60,6 +62,7 @@ pub mod expr_fn {"Returns the factorial of expr. expr is [0..20]. Otherwise, null.",arg1));+ export_functions!((floor, "Returns floor of expr.", arg1));export_functions!((hex, "Computes hex value of the given column.", arg1));export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2));export_functions!((pmod, "Returns the positive remainder of division of the first argument by the second argument.", arg1 arg2));@@ -95,6 +98,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {ceil(),expm1(),factorial(),+ floor(),hex(),modulus(),pmod(),diff --git a/datafusion/sqllogictest/test_files/spark/math/floor.slt b/datafusion/sqllogictest/test_files/spark/math/floor.sltindex d39d47ab1f..fd885051af 100644--- a/datafusion/sqllogictest/test_files/spark/math/floor.slt+++ b/datafusion/sqllogictest/test_files/spark/math/floor.slt@@ -15,28 +15,282 @@# specific language governing permissions and limitations# under the License.-# This file was originally created by a porting script from:-# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function-# This file is part of the implementation of the datafusion-spark function library.-# For more information, please see:-# https://github.com/apache/datafusion/issues/15914--## Original Query: SELECT floor(-0.1);-## PySpark 3.5.5 Result: {'FLOOR(-0.1)': Decimal('-1'), 'typeof(FLOOR(-0.1))': 'decimal(1,0)', 'typeof(-0.1)': 'decimal(1,1)'}-#query-#SELECT floor(-0.1::decimal(1,1));--## Original Query: SELECT floor(3.1411, -3);-## PySpark 3.5.5 Result: {'floor(3.1411, -3)': Decimal('0'), 'typeof(floor(3.1411, -3))': 'decimal(4,0)', 'typeof(3.1411)': 'decimal(5,4)', 'typeof(-3)': 'int'}-#query-#SELECT floor(3.1411::decimal(5,4), -3::int);--## Original Query: SELECT floor(3.1411, 3);-## PySpark 3.5.5 Result: {'floor(3.1411, 3)': Decimal('3.141'), 'typeof(floor(3.1411, 3))': 'decimal(5,3)', 'typeof(3.1411)': 'decimal(5,4)', 'typeof(3)': 'int'}-#query-#SELECT floor(3.1411::decimal(5,4), 3::int);--## Original Query: SELECT floor(5);-## PySpark 3.5.5 Result: {'FLOOR(5)': 5, 'typeof(FLOOR(5))': 'bigint', 'typeof(5)': 'int'}-#query-#SELECT floor(5::int);+#############+## Scalars ##+#############++# Float64 tests - returns Int64+query IT+SELECT floor(1.1), arrow_typeof(floor(1.1));+----+1 Int64++query IT+SELECT floor(-1.1), arrow_typeof(floor(-1.1));+----+-2 Int64++query IT+SELECT floor(0.0), arrow_typeof(floor(0.0));+----+0 Int64++# Float32 tests - returns Int64+query IT+SELECT floor(arrow_cast(1.5, 'Float32')), arrow_typeof(floor(arrow_cast(1.5, 'Float32')));+----+1 Int64++query IT+SELECT floor(arrow_cast(-1.5, 'Float32')), arrow_typeof(floor(arrow_cast(-1.5, 'Float32')));+----+-2 Int64++# Integer tests - returns Int64+query IT+SELECT floor(arrow_cast(5, 'Int8')), arrow_typeof(floor(arrow_cast(5, 'Int8')));+----+5 Int64++query IT+SELECT floor(arrow_cast(-5, 'Int16')), arrow_typeof(floor(arrow_cast(-5, 'Int16')));+----+-5 Int64++query IT+SELECT floor(arrow_cast(1000, 'Int32')), arrow_typeof(floor(arrow_cast(1000, 'Int32')));+----+1000 Int64++query IT+SELECT floor(arrow_cast(-1000, 'Int64')), arrow_typeof(floor(arrow_cast(-1000, 'Int64')));+----+-1000 Int64++# Decimal128 tests - returns Decimal128(p-s+1, 0)+# Decimal(5,2) -> Decimal(4,0)+query RT+SELECT floor(arrow_cast(123.45, 'Decimal128(5,2)')), arrow_typeof(floor(arrow_cast(123.45, 'Decimal128(5,2)')));+----+123 Decimal128(4, 0)++query RT+SELECT floor(arrow_cast(-123.45, 'Decimal128(5,2)')), arrow_typeof(floor(arrow_cast(-123.45, 'Decimal128(5,2)')));+----+-124 Decimal128(4, 0)++# Decimal with scale 0 - unchanged precision+query RT+SELECT floor(arrow_cast(123, 'Decimal128(10,0)')), arrow_typeof(floor(arrow_cast(123, 'Decimal128(10,0)')));+----+123 Decimal128(10, 0)++# NULL handling+query IT+SELECT floor(NULL::double), arrow_typeof(floor(NULL::double));+----+NULL Int64++############+## Arrays ##+############++statement ok+CREATE TABLE floor_test (+ f64_col DOUBLE,+ f32_col FLOAT,+ i32_col INT,+ dec_col DECIMAL(5,2)+) AS VALUES+ (1.1, 1.5, 5, 123.45),+ (-1.1, -1.5, -5, -123.45),+ (0.0, 0.0, 0, 125.00),+ (NULL, NULL, NULL, NULL);++# Float64 array+query I+SELECT floor(f64_col) FROM floor_test;+----+1+-2+0+NULL++# Float32 array+query I+SELECT floor(f32_col) FROM floor_test;+----+1+-2+0+NULL++# Int32 array+query I+SELECT floor(i32_col) FROM floor_test;+----+5+-5+0+NULL++# Decimal array+query R+SELECT floor(dec_col) FROM floor_test;+----+123+-124+125+NULL++# Verify array result types+query TTTT+SELECT+ arrow_typeof(floor(f64_col)),+ arrow_typeof(floor(f32_col)),+ arrow_typeof(floor(i32_col)),+ arrow_typeof(floor(dec_col))+FROM floor_test LIMIT 1;+----+Int64 Int64 Int64 Decimal128(4, 0)++statement ok+DROP TABLE floor_test;++#####################+## Additional cases ##+#####################++# Float scalar — floor rounds toward negative infinity for fractional values+query II+SELECT floor(1.9::DOUBLE), floor(-1.9::DOUBLE);+----+1 -2++query II+SELECT floor(1.9::FLOAT), floor(-1.9::FLOAT);+----+1 -2++# Int8 boundaries (i8::MAX, i8::MIN) pass through as Int64+# Note: i8::MIN written via arrow_cast to dodge the -(128) parse issue (128 > i8::MAX)+query III+SELECT floor(arrow_cast(127, 'Int8')), floor(arrow_cast(-128, 'Int8')), floor(NULL::TINYINT);+----+127 -128 NULL++# Int64 boundary values pass through unchanged+query II+SELECT floor(arrow_cast(9223372036854775807, 'Int64')), floor(arrow_cast(-9223372036854775808, 'Int64'));+----+9223372036854775807 -9223372036854775808++# Decimal128 negative NULL scalar at given precision/scale+query RT+SELECT floor(NULL::DECIMAL(5,2)), arrow_typeof(floor(NULL::DECIMAL(5,2)));+----+NULL Decimal128(4, 0)++# Decimal128 array with explicit precision (5,2) — values 123.45, 125.00, -129.99+query R+SELECT floor(a) FROM (VALUES (123.45::DECIMAL(5,2)), (125.00::DECIMAL(5,2)), (-129.99::DECIMAL(5,2)), (NULL::DECIMAL(5,2))) AS t(a);+----+123+125+-130+NULL++# Decimal128 array with scale 0 — passes through unchanged+query RT+SELECT floor(a), arrow_typeof(floor(a))+FROM (VALUES (123::DECIMAL(10,0)), (-456::DECIMAL(10,0)), (NULL::DECIMAL(10,0))) AS t(a);+----+123 Decimal128(10, 0)+-456 Decimal128(10, 0)+NULL Decimal128(10, 0)++# Decimal128 scale > 0 returns Decimal128(p-s+1, 0)+query RT+SELECT floor(56.7::DECIMAL(3,1)), arrow_typeof(floor(56.7::DECIMAL(3,1)));+----+56 Decimal128(3, 0)++query RT+SELECT floor(-56.7::DECIMAL(3,1)), arrow_typeof(floor(-56.7::DECIMAL(3,1)));+----+-57 Decimal128(3, 0)++# NaN and Infinity (Float64)+# Matches Spark: (long)Math.floor(x) — NaN → 0, +Inf → Long.MAX_VALUE, -Inf → Long.MIN_VALUE+query III+SELECT floor('NaN'::DOUBLE), floor('Infinity'::DOUBLE), floor('-Infinity'::DOUBLE);+----+0 9223372036854775807 -9223372036854775808++# NaN and Infinity (Float32) — same saturating behavior+query III+SELECT floor('NaN'::FLOAT), floor('Infinity'::FLOAT), floor('-Infinity'::FLOAT);+----+0 9223372036854775807 -9223372036854775808++# NaN/Infinity in array form+query I+SELECT floor(a) FROM (VALUES ('NaN'::DOUBLE), ('Infinity'::DOUBLE), ('-Infinity'::DOUBLE), (NULL::DOUBLE)) AS t(a);+----+0+9223372036854775807+-9223372036854775808+NULL++####################+## Boundary cases ##+####################++# Int16 boundaries (i16::MAX, i16::MIN) pass through as Int64+# Note: i16::MIN via arrow_cast to dodge the -(32768) parse issue+query III+SELECT floor(arrow_cast(32767, 'Int16')), floor(arrow_cast(-32768, 'Int16')), floor(NULL::SMALLINT);+----+32767 -32768 NULL++# Int32 boundaries (i32::MAX, i32::MIN) pass through as Int64+# Note: i32::MIN via arrow_cast to dodge the -(2147483648) parse issue+query III+SELECT floor(arrow_cast(2147483647, 'Int32')), floor(arrow_cast(-2147483648, 'Int32')), floor(NULL::INT);+----+2147483647 -2147483648 NULL++# Float values exceeding Int64 range saturate at Int64 boundaries+query II+SELECT floor(1e20::DOUBLE), floor(-1e20::DOUBLE);+----+9223372036854775807 -9223372036854775808++query II+SELECT floor(1e20::FLOAT), floor(-1e20::FLOAT);+----+9223372036854775807 -9223372036854775808++# Negative zero floors to 0 (not -1) since -0.0 == 0.0+query II+SELECT floor(-0.0::DOUBLE) AS f64, floor(-0.0::FLOAT) AS f32;+----+0 0++# Tiny negative float still rounds toward negative infinity → -1+query I+SELECT floor(-1e-300::DOUBLE);+----+-1++# Decimal precision lower-bound clamp: DECIMAL(38, 38) → Decimal128(1, 0)+query RT+SELECT floor(0.5::DECIMAL(38,38)), arrow_typeof(floor(0.5::DECIMAL(38,38)));+----+0 Decimal128(1, 0)++# Decimal value at the boundary between two integers (just under .5 vs just over)+query RR+SELECT floor(0.49::DECIMAL(3,2)), floor(-0.49::DECIMAL(3,2));+----+0 -1