Skip to content

Apache DataFusion SQL Query Engine

feat: support spark compatible floor function (#21933)

Krishna Sudarshan J015162afac7e
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 72bca58b7a..b03a732107 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -99,3 +99,7 @@ name = "unhex"
[[bench]]
harness = false
name = "sha2"
+
+[[bench]]
+harness = false
+name = "floor"
diff --git a/datafusion/spark/benches/floor.rs b/datafusion/spark/benches/floor.rs
new file mode 100644
index 0000000000..ecb1590acc
--- /dev/null
+++ b/datafusion/spark/benches/floor.rs
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::*;
+use arrow::datatypes::*;
+use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_common::config::ConfigOptions;
+use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
+use datafusion_spark::function::math::floor::SparkFloor;
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use std::hint::black_box;
+use std::sync::Arc;
+
+fn seedable_rng() -> StdRng {
+ StdRng::seed_from_u64(42)
+}
+
+fn generate_float64_data(size: usize, null_density: f32) -> Float64Array {
+ let mut rng = seedable_rng();
+ (0..size)
+ .map(|_| {
+ if rng.random::<f32>() < null_density {
+ None
+ } else {
+ Some(rng.random_range::<f64, _>(-1_000_000.0..1_000_000.0))
+ }
+ })
+ .collect()
+}
+
+fn generate_decimal128_data(size: usize, null_density: f32) -> Decimal128Array {
+ let mut rng = seedable_rng();
+ let array: Decimal128Array = (0..size)
+ .map(|_| {
+ if rng.random::<f32>() < null_density {
+ None
+ } else {
+ Some(rng.random_range::<i128, _>(-999_999_999..999_999_999))
+ }
+ })
+ .collect();
+ array.with_precision_and_scale(18, 2).unwrap()
+}
+
+fn run_benchmark(
+ c: &mut Criterion,
+ name: &str,
+ size: usize,
+ array: Arc<dyn Array>,
+ return_type: &DataType,
+) {
+ let floor_func = SparkFloor::new();
+ let args = vec![ColumnarValue::Array(array)];
+ let arg_fields: Vec<_> = args
+ .iter()
+ .enumerate()
+ .map(|(idx, arg)| Field::new(format!("arg_{idx}"), arg.data_type(), true).into())
+ .collect();
+ let config_options = Arc::new(ConfigOptions::default());
+
+ c.bench_function(&format!("{name}/size={size}"), |b| {
+ b.iter(|| {
+ black_box(
+ floor_func
+ .invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Arc::new(Field::new(
+ "f",
+ return_type.clone(),
+ true,
+ )),
+ config_options: Arc::clone(&config_options),
+ })
+ .unwrap(),
+ )
+ })
+ });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+ let sizes = vec![1024, 4096, 8192];
+ let null_density = 0.1;
+
+ for &size in &sizes {
+ let data = generate_float64_data(size, null_density);
+ run_benchmark(c, "floor_float64", size, Arc::new(data), &DataType::Int64);
+ }
+
+ for &size in &sizes {
+ let data = generate_decimal128_data(size, null_density);
+ run_benchmark(
+ c,
+ "floor_decimal128",
+ size,
+ Arc::new(data),
+ &DataType::Decimal128(17, 0),
+ );
+ }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/spark/src/function/math/floor.rs b/datafusion/spark/src/function/math/floor.rs
new file mode 100644
index 0000000000..703f81a2c2
--- /dev/null
+++ b/datafusion/spark/src/function/math/floor.rs
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::cast::AsArray;
+use arrow::array::types::Decimal128Type;
+use arrow::array::{ArrowNativeTypeOp, Decimal128Array, Int64Array};
+use arrow::compute::kernels::arity::unary;
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{DataFusionError, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use std::sync::Arc;
+
+/// Spark-compatible `floor` function.
+///
+/// Differences from DataFusion's floor:
+/// - Returns Int64 for float and integer inputs (while DataFusion preserves input type)
+/// - For Decimal128(p, s), returns Decimal128(p-s+1, 0) with scale 0
+/// (DataFusion preserves original precision and scale)
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#floor>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkFloor {
+ signature: Signature,
+}
+
+impl Default for SparkFloor {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkFloor {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::numeric(1, Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkFloor {
+ fn name(&self) -> &str {
+ "floor"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(
+ &self,
+ _arg_types: &[DataType],
+ ) -> datafusion_common::Result<DataType> {
+ internal_err!("return_field_from_args should be called instead")
+ }
+
+ fn return_field_from_args(
+ &self,
+ args: ReturnFieldArgs,
+ ) -> datafusion_common::Result<FieldRef> {
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
+ let return_type = match args.arg_fields[0].data_type() {
+ DataType::Decimal128(p, s) if *s > 0 => {
+ let new_p = (*p - *s as u8 + 1).clamp(1, 38);
+ DataType::Decimal128(new_p, 0)
+ }
+ DataType::Decimal128(p, s) => DataType::Decimal128(*p, *s),
+ DataType::Float32
+ | DataType::Float64
+ | DataType::Int8
+ | DataType::Int16
+ | DataType::Int32
+ | DataType::Int64 => DataType::Int64,
+ _ => exec_err!(
+ "found unsupported return type {:?}",
+ args.arg_fields[0].data_type()
+ )?,
+ };
+ Ok(Arc::new(Field::new(self.name(), return_type, nullable)))
+ }
+
+ fn invoke_with_args(
+ &self,
+ args: ScalarFunctionArgs,
+ ) -> datafusion_common::Result<ColumnarValue> {
+ spark_floor(&args.args, args.return_field.data_type())
+ }
+}
+
+macro_rules! apply_int64 {
+ ($value:expr, $arr_type:ty, $scalar_variant:path, $f:expr) => {
+ match $value {
+ ColumnarValue::Array(array) => {
+ let result: Int64Array = unary(array.as_primitive::<$arr_type>(), $f);
+ Ok(ColumnarValue::Array(Arc::new(result)))
+ }
+ ColumnarValue::Scalar($scalar_variant(v)) => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int64(v.map($f))))
+ }
+ other => internal_err!(
+ "floor: data type mismatch — expected scalar of type {} but got {:?}",
+ stringify!($scalar_variant),
+ other.data_type()
+ ),
+ }
+ };
+}
+
+fn spark_floor(
+ args: &[ColumnarValue],
+ return_type: &DataType,
+) -> Result<ColumnarValue, DataFusionError> {
+ let value = &args[0];
+ match value.data_type() {
+ DataType::Float32 => apply_int64!(
+ value,
+ arrow::datatypes::Float32Type,
+ ScalarValue::Float32,
+ |x| x.floor() as i64
+ ),
+ DataType::Float64 => apply_int64!(
+ value,
+ arrow::datatypes::Float64Type,
+ ScalarValue::Float64,
+ |x| x.floor() as i64
+ ),
+ DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => {
+ value.cast_to(&DataType::Int64, None)
+ }
+ DataType::Decimal128(_, scale) if scale > 0 => {
+ let divisor = 10_i128.pow_wrapping(scale as u32);
+ let floor_decimal = |x: i128| {
+ let (d, r) = (x / divisor, x % divisor);
+ if r < 0 { d - 1 } else { d }
+ };
+ match value {
+ ColumnarValue::Array(array) => {
+ let result: Decimal128Array =
+ unary(array.as_primitive::<Decimal128Type>(), floor_decimal);
+ Ok(ColumnarValue::Array(Arc::new(
+ result.with_data_type(return_type.clone()),
+ )))
+ }
+ ColumnarValue::Scalar(ScalarValue::Decimal128(v, _, _)) => {
+ let DataType::Decimal128(new_p, new_s) = return_type else {
+ return internal_err!(
+ "floor: data type mismatch — expected Decimal128 return type but got {:?}",
+ return_type
+ );
+ };
+ Ok(ColumnarValue::Scalar(ScalarValue::Decimal128(
+ v.map(floor_decimal),
+ *new_p,
+ *new_s,
+ )))
+ }
+ other => internal_err!(
+ "floor: data type mismatch — expected Decimal128 scalar but got {:?}",
+ other.data_type()
+ ),
+ }
+ }
+ DataType::Decimal128(_, _) => Ok(value.clone()),
+ other => exec_err!("Unsupported data type {other:?} for function floor"),
+ }
+}
diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs
index 3f6a3a686d..896eedd033 100644
--- a/datafusion/spark/src/function/math/mod.rs
+++ b/datafusion/spark/src/function/math/mod.rs
@@ -20,6 +20,7 @@ pub mod bin;
pub mod ceil;
pub mod expm1;
pub mod factorial;
+pub mod floor;
pub mod hex;
pub mod modulus;
pub mod negative;
@@ -37,6 +38,7 @@ make_udf_function!(abs::SparkAbs, abs);
make_udf_function!(ceil::SparkCeil, ceil);
make_udf_function!(expm1::SparkExpm1, expm1);
make_udf_function!(factorial::SparkFactorial, factorial);
+make_udf_function!(floor::SparkFloor, floor);
make_udf_function!(hex::SparkHex, hex);
make_udf_function!(modulus::SparkMod, modulus);
make_udf_function!(modulus::SparkPmod, pmod);
@@ -60,6 +62,7 @@ pub mod expr_fn {
"Returns the factorial of expr. expr is [0..20]. Otherwise, null.",
arg1
));
+ export_functions!((floor, "Returns floor of expr.", arg1));
export_functions!((hex, "Computes hex value of the given column.", arg1));
export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2));
export_functions!((pmod, "Returns the positive remainder of division of the first argument by the second argument.", arg1 arg2));
@@ -95,6 +98,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
ceil(),
expm1(),
factorial(),
+ floor(),
hex(),
modulus(),
pmod(),
diff --git a/datafusion/sqllogictest/test_files/spark/math/floor.slt b/datafusion/sqllogictest/test_files/spark/math/floor.slt
index d39d47ab1f..fd885051af 100644
--- a/datafusion/sqllogictest/test_files/spark/math/floor.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/floor.slt
@@ -15,28 +15,282 @@
# specific language governing permissions and limitations
# under the License.
-# This file was originally created by a porting script from:
-# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function library.
-# For more information, please see:
-# https://github.com/apache/datafusion/issues/15914
-
-## Original Query: SELECT floor(-0.1);
-## PySpark 3.5.5 Result: {'FLOOR(-0.1)': Decimal('-1'), 'typeof(FLOOR(-0.1))': 'decimal(1,0)', 'typeof(-0.1)': 'decimal(1,1)'}
-#query
-#SELECT floor(-0.1::decimal(1,1));
-
-## Original Query: SELECT floor(3.1411, -3);
-## PySpark 3.5.5 Result: {'floor(3.1411, -3)': Decimal('0'), 'typeof(floor(3.1411, -3))': 'decimal(4,0)', 'typeof(3.1411)': 'decimal(5,4)', 'typeof(-3)': 'int'}
-#query
-#SELECT floor(3.1411::decimal(5,4), -3::int);
-
-## Original Query: SELECT floor(3.1411, 3);
-## PySpark 3.5.5 Result: {'floor(3.1411, 3)': Decimal('3.141'), 'typeof(floor(3.1411, 3))': 'decimal(5,3)', 'typeof(3.1411)': 'decimal(5,4)', 'typeof(3)': 'int'}
-#query
-#SELECT floor(3.1411::decimal(5,4), 3::int);
-
-## Original Query: SELECT floor(5);
-## PySpark 3.5.5 Result: {'FLOOR(5)': 5, 'typeof(FLOOR(5))': 'bigint', 'typeof(5)': 'int'}
-#query
-#SELECT floor(5::int);
+#############
+## Scalars ##
+#############
+
+# Float64 tests - returns Int64
+query IT
+SELECT floor(1.1), arrow_typeof(floor(1.1));
+----
+1 Int64
+
+query IT
+SELECT floor(-1.1), arrow_typeof(floor(-1.1));
+----
+-2 Int64
+
+query IT
+SELECT floor(0.0), arrow_typeof(floor(0.0));
+----
+0 Int64
+
+# Float32 tests - returns Int64
+query IT
+SELECT floor(arrow_cast(1.5, 'Float32')), arrow_typeof(floor(arrow_cast(1.5, 'Float32')));
+----
+1 Int64
+
+query IT
+SELECT floor(arrow_cast(-1.5, 'Float32')), arrow_typeof(floor(arrow_cast(-1.5, 'Float32')));
+----
+-2 Int64
+
+# Integer tests - returns Int64
+query IT
+SELECT floor(arrow_cast(5, 'Int8')), arrow_typeof(floor(arrow_cast(5, 'Int8')));
+----
+5 Int64
+
+query IT
+SELECT floor(arrow_cast(-5, 'Int16')), arrow_typeof(floor(arrow_cast(-5, 'Int16')));
+----
+-5 Int64
+
+query IT
+SELECT floor(arrow_cast(1000, 'Int32')), arrow_typeof(floor(arrow_cast(1000, 'Int32')));
+----
+1000 Int64
+
+query IT
+SELECT floor(arrow_cast(-1000, 'Int64')), arrow_typeof(floor(arrow_cast(-1000, 'Int64')));
+----
+-1000 Int64
+
+# Decimal128 tests - returns Decimal128(p-s+1, 0)
+# Decimal(5,2) -> Decimal(4,0)
+query RT
+SELECT floor(arrow_cast(123.45, 'Decimal128(5,2)')), arrow_typeof(floor(arrow_cast(123.45, 'Decimal128(5,2)')));
+----
+123 Decimal128(4, 0)
+
+query RT
+SELECT floor(arrow_cast(-123.45, 'Decimal128(5,2)')), arrow_typeof(floor(arrow_cast(-123.45, 'Decimal128(5,2)')));
+----
+-124 Decimal128(4, 0)
+
+# Decimal with scale 0 - unchanged precision
+query RT
+SELECT floor(arrow_cast(123, 'Decimal128(10,0)')), arrow_typeof(floor(arrow_cast(123, 'Decimal128(10,0)')));
+----
+123 Decimal128(10, 0)
+
+# NULL handling
+query IT
+SELECT floor(NULL::double), arrow_typeof(floor(NULL::double));
+----
+NULL Int64
+
+############
+## Arrays ##
+############
+
+statement ok
+CREATE TABLE floor_test (
+ f64_col DOUBLE,
+ f32_col FLOAT,
+ i32_col INT,
+ dec_col DECIMAL(5,2)
+) AS VALUES
+ (1.1, 1.5, 5, 123.45),
+ (-1.1, -1.5, -5, -123.45),
+ (0.0, 0.0, 0, 125.00),
+ (NULL, NULL, NULL, NULL);
+
+# Float64 array
+query I
+SELECT floor(f64_col) FROM floor_test;
+----
+1
+-2
+0
+NULL
+
+# Float32 array
+query I
+SELECT floor(f32_col) FROM floor_test;
+----
+1
+-2
+0
+NULL
+
+# Int32 array
+query I
+SELECT floor(i32_col) FROM floor_test;
+----
+5
+-5
+0
+NULL
+
+# Decimal array
+query R
+SELECT floor(dec_col) FROM floor_test;
+----
+123
+-124
+125
+NULL
+
+# Verify array result types
+query TTTT
+SELECT
+ arrow_typeof(floor(f64_col)),
+ arrow_typeof(floor(f32_col)),
+ arrow_typeof(floor(i32_col)),
+ arrow_typeof(floor(dec_col))
+FROM floor_test LIMIT 1;
+----
+Int64 Int64 Int64 Decimal128(4, 0)
+
+statement ok
+DROP TABLE floor_test;
+
+#####################
+## Additional cases ##
+#####################
+
+# Float scalar — floor rounds toward negative infinity for fractional values
+query II
+SELECT floor(1.9::DOUBLE), floor(-1.9::DOUBLE);
+----
+1 -2
+
+query II
+SELECT floor(1.9::FLOAT), floor(-1.9::FLOAT);
+----
+1 -2
+
+# Int8 boundaries (i8::MAX, i8::MIN) pass through as Int64
+# Note: i8::MIN written via arrow_cast to dodge the -(128) parse issue (128 > i8::MAX)
+query III
+SELECT floor(arrow_cast(127, 'Int8')), floor(arrow_cast(-128, 'Int8')), floor(NULL::TINYINT);
+----
+127 -128 NULL
+
+# Int64 boundary values pass through unchanged
+query II
+SELECT floor(arrow_cast(9223372036854775807, 'Int64')), floor(arrow_cast(-9223372036854775808, 'Int64'));
+----
+9223372036854775807 -9223372036854775808
+
+# Decimal128 negative NULL scalar at given precision/scale
+query RT
+SELECT floor(NULL::DECIMAL(5,2)), arrow_typeof(floor(NULL::DECIMAL(5,2)));
+----
+NULL Decimal128(4, 0)
+
+# Decimal128 array with explicit precision (5,2) — values 123.45, 125.00, -129.99
+query R
+SELECT floor(a) FROM (VALUES (123.45::DECIMAL(5,2)), (125.00::DECIMAL(5,2)), (-129.99::DECIMAL(5,2)), (NULL::DECIMAL(5,2))) AS t(a);
+----
+123
+125
+-130
+NULL
+
+# Decimal128 array with scale 0 — passes through unchanged
+query RT
+SELECT floor(a), arrow_typeof(floor(a))
+FROM (VALUES (123::DECIMAL(10,0)), (-456::DECIMAL(10,0)), (NULL::DECIMAL(10,0))) AS t(a);
+----
+123 Decimal128(10, 0)
+-456 Decimal128(10, 0)
+NULL Decimal128(10, 0)
+
+# Decimal128 scale > 0 returns Decimal128(p-s+1, 0)
+query RT
+SELECT floor(56.7::DECIMAL(3,1)), arrow_typeof(floor(56.7::DECIMAL(3,1)));
+----
+56 Decimal128(3, 0)
+
+query RT
+SELECT floor(-56.7::DECIMAL(3,1)), arrow_typeof(floor(-56.7::DECIMAL(3,1)));
+----
+-57 Decimal128(3, 0)
+
+# NaN and Infinity (Float64)
+# Matches Spark: (long)Math.floor(x) — NaN → 0, +Inf → Long.MAX_VALUE, -Inf → Long.MIN_VALUE
+query III
+SELECT floor('NaN'::DOUBLE), floor('Infinity'::DOUBLE), floor('-Infinity'::DOUBLE);
+----
+0 9223372036854775807 -9223372036854775808
+
+# NaN and Infinity (Float32) — same saturating behavior
+query III
+SELECT floor('NaN'::FLOAT), floor('Infinity'::FLOAT), floor('-Infinity'::FLOAT);
+----
+0 9223372036854775807 -9223372036854775808
+
+# NaN/Infinity in array form
+query I
+SELECT floor(a) FROM (VALUES ('NaN'::DOUBLE), ('Infinity'::DOUBLE), ('-Infinity'::DOUBLE), (NULL::DOUBLE)) AS t(a);
+----
+0
+9223372036854775807
+-9223372036854775808
+NULL
+
+####################
+## Boundary cases ##
+####################
+
+# Int16 boundaries (i16::MAX, i16::MIN) pass through as Int64
+# Note: i16::MIN via arrow_cast to dodge the -(32768) parse issue
+query III
+SELECT floor(arrow_cast(32767, 'Int16')), floor(arrow_cast(-32768, 'Int16')), floor(NULL::SMALLINT);
+----
+32767 -32768 NULL
+
+# Int32 boundaries (i32::MAX, i32::MIN) pass through as Int64
+# Note: i32::MIN via arrow_cast to dodge the -(2147483648) parse issue
+query III
+SELECT floor(arrow_cast(2147483647, 'Int32')), floor(arrow_cast(-2147483648, 'Int32')), floor(NULL::INT);
+----
+2147483647 -2147483648 NULL
+
+# Float values exceeding Int64 range saturate at Int64 boundaries
+query II
+SELECT floor(1e20::DOUBLE), floor(-1e20::DOUBLE);
+----
+9223372036854775807 -9223372036854775808
+
+query II
+SELECT floor(1e20::FLOAT), floor(-1e20::FLOAT);
+----
+9223372036854775807 -9223372036854775808
+
+# Negative zero floors to 0 (not -1) since -0.0 == 0.0
+query II
+SELECT floor(-0.0::DOUBLE) AS f64, floor(-0.0::FLOAT) AS f32;
+----
+0 0
+
+# Tiny negative float still rounds toward negative infinity → -1
+query I
+SELECT floor(-1e-300::DOUBLE);
+----
+-1
+
+# Decimal precision lower-bound clamp: DECIMAL(38, 38) → Decimal128(1, 0)
+query RT
+SELECT floor(0.5::DECIMAL(38,38)), arrow_typeof(floor(0.5::DECIMAL(38,38)));
+----
+0 Decimal128(1, 0)
+
+# Decimal value at the boundary between two integers (just under .5 vs just over)
+query RR
+SELECT floor(0.49::DECIMAL(3,2)), floor(-0.49::DECIMAL(3,2));
+----
+0 -1