Skip to content

Apache DataFusion SQL Query Engine

feat: decimal support for percentile_cont and median (#24419)

theirix46957762e3a2
diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs
index 2a2adfdc98..d2db50f374 100644
--- a/datafusion/functions-aggregate/src/median.rs
+++ b/datafusion/functions-aggregate/src/median.rs
@@ -15,48 +15,24 @@
// specific language governing permissions and limitations
// under the License.
-use std::cmp::Ordering;
-use std::fmt::{Debug, Formatter};
-use std::mem::{size_of, size_of_val};
-use std::sync::Arc;
+use std::fmt::Debug;
-use arrow::array::{
- ArrowNumericType, BooleanArray, ListArray, PrimitiveArray, PrimitiveBuilder,
- downcast_integer,
-};
-use arrow::buffer::{OffsetBuffer, ScalarBuffer};
-use arrow::{
- array::{ArrayRef, AsArray},
- datatypes::{
- DataType, Decimal128Type, Decimal256Type, Field, Float16Type, Float32Type,
- Float64Type,
- },
-};
+use arrow::datatypes::DataType;
-use arrow::array::Array;
-use arrow::array::ArrowNativeTypeOp;
-use arrow::datatypes::{
- ArrowNativeType, ArrowPrimitiveType, Decimal32Type, Decimal64Type, FieldRef,
-};
+use arrow::datatypes::FieldRef;
-use datafusion_common::hash_utils::RandomState;
-use datafusion_common::types::{NativeType, logical_float64};
-use datafusion_common::{
- DataFusionError, Result, ScalarValue, assert_eq_or_internal_err, exec_datafusion_err,
- internal_datafusion_err, internal_err,
+use crate::percentile_cont::{
+ PercentileCont, create_percentile_accumulator, create_percentile_groups_accumulator,
};
+use datafusion_common::Result;
+use datafusion_common::types::logical_float64;
+use datafusion_expr::GroupsAccumulator;
use datafusion_expr::function::StateFieldsArgs;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, Documentation, Signature, TypeSignature,
- TypeSignatureClass, Volatility, function::AccumulatorArgs, utils::format_state_name,
+ TypeSignatureClass, Volatility, function::AccumulatorArgs,
};
-use datafusion_expr::{EmitTo, GroupsAccumulator};
-use datafusion_functions_aggregate_common::aggregate::groups_accumulator::accumulate::accumulate;
-use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::filtered_null_mask;
-use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;
-use datafusion_functions_aggregate_common::utils::{GenericDistinctBuffer, Hashable};
use datafusion_macros::user_doc;
-use std::collections::HashMap;
make_udaf_expr_and_func!(
Median,
@@ -91,6 +67,7 @@ make_udaf_expr_and_func!(
#[derive(PartialEq, Eq, Hash, Debug)]
pub struct Median {
signature: Signature,
+ percentile_cont: PercentileCont,
}
impl Default for Median {
@@ -113,14 +90,14 @@ impl Median {
TypeSignature::Coercible(vec![Coercion::new_exact(
TypeSignatureClass::Float,
)]),
- TypeSignature::Coercible(vec![Coercion::new_implicit(
- TypeSignatureClass::Native(logical_float64()),
+ TypeSignature::Coercible(vec![Coercion::new_implicit_native(
+ logical_float64(),
vec![TypeSignatureClass::Integer],
- NativeType::Float64,
)]),
],
Volatility::Immutable,
),
+ percentile_cont: PercentileCont::new(),
}
}
}
@@ -135,589 +112,38 @@ impl AggregateUDFImpl for Median {
}
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- Ok(arg_types[0].clone())
+ self.percentile_cont.return_type(arg_types)
}
fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
- if args.input_fields[0].data_type().is_null() {
- return Ok(vec![
- Field::new(
- format_state_name(args.name, self.name()),
- DataType::Null,
- true,
- )
- .into(),
- ]);
- }
-
- //Intermediate state is a list of the elements we have collected so far
- let field = Field::new_list_field(args.input_fields[0].data_type().clone(), true);
- let state_name = if args.is_distinct {
- "distinct_median"
- } else {
- "median"
- };
-
- Ok(vec![
- Field::new(
- format_state_name(args.name, state_name),
- DataType::List(Arc::new(field)),
- true,
- )
- .into(),
- ])
+ self.percentile_cont.state_fields(args)
}
- fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
- macro_rules! helper {
- ($t:ty, $dt:expr) => {
- if acc_args.is_distinct {
- Ok(Box::new(DistinctMedianAccumulator::<$t> {
- data_type: $dt.clone(),
- distinct_values: GenericDistinctBuffer::new($dt),
- }))
- } else {
- Ok(Box::new(MedianAccumulator::<$t> {
- data_type: $dt.clone(),
- all_values: vec![],
- }))
- }
- };
- }
-
- let dt = acc_args.expr_fields[0].data_type().clone();
- if dt.is_null() {
- return Ok(Box::new(NoopAccumulator::default()));
- }
-
- downcast_integer! {
- dt => (helper, dt),
- DataType::Float16 => helper!(Float16Type, dt),
- DataType::Float32 => helper!(Float32Type, dt),
- DataType::Float64 => helper!(Float64Type, dt),
- DataType::Decimal32(_, _) => helper!(Decimal32Type, dt),
- DataType::Decimal64(_, _) => helper!(Decimal64Type, dt),
- DataType::Decimal128(_, _) => helper!(Decimal128Type, dt),
- DataType::Decimal256(_, _) => helper!(Decimal256Type, dt),
- _ => Err(DataFusionError::NotImplemented(format!(
- "MedianAccumulator not supported for {} with {}",
- acc_args.name,
- dt,
- ))),
- }
+ fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
+ create_percentile_accumulator(
+ self.name(),
+ 0.5,
+ args.expr_fields[0].data_type(),
+ args.is_distinct,
+ )
}
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
- !args.is_distinct && !args.expr_fields[0].data_type().is_null()
+ self.percentile_cont.groups_accumulator_supported(args)
}
fn create_groups_accumulator(
&self,
args: AccumulatorArgs,
) -> Result<Box<dyn GroupsAccumulator>> {
- let num_args = args.exprs.len();
- assert_eq_or_internal_err!(
- num_args,
- 1,
- "median should only have 1 arg, but found num args:{}",
- num_args
- );
-
- let dt = args.expr_fields[0].data_type().clone();
-
- macro_rules! helper {
- ($t:ty, $dt:expr) => {
- Ok(Box::new(MedianGroupsAccumulator::<$t>::new($dt)))
- };
- }
-
- downcast_integer! {
- dt => (helper, dt),
- DataType::Float16 => helper!(Float16Type, dt),
- DataType::Float32 => helper!(Float32Type, dt),
- DataType::Float64 => helper!(Float64Type, dt),
- DataType::Decimal32(_, _) => helper!(Decimal32Type, dt),
- DataType::Decimal64(_, _) => helper!(Decimal64Type, dt),
- DataType::Decimal128(_, _) => helper!(Decimal128Type, dt),
- DataType::Decimal256(_, _) => helper!(Decimal256Type, dt),
- _ => Err(DataFusionError::NotImplemented(format!(
- "MedianGroupsAccumulator not supported for {} with {}",
- args.name,
- dt,
- ))),
- }
+ create_percentile_groups_accumulator(
+ self.name(),
+ 0.5,
+ args.expr_fields[0].data_type(),
+ )
}
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
}
-
-/// The median accumulator accumulates the raw input values
-/// as `ScalarValue`s
-///
-/// The intermediate state is represented as a List of scalar values updated by
-/// `merge_batch` and a `Vec` of `ArrayRef` that are converted to scalar values
-/// in the final evaluation step so that we avoid expensive conversions and
-/// allocations during `update_batch`.
-struct MedianAccumulator<T: ArrowNumericType> {
- data_type: DataType,
- all_values: Vec<T::Native>,
-}
-
-impl<T: ArrowNumericType> Debug for MedianAccumulator<T> {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- write!(f, "MedianAccumulator({})", self.data_type)
- }
-}
-
-impl<T: ArrowNumericType> Accumulator for MedianAccumulator<T> {
- fn state(&mut self) -> Result<Vec<ScalarValue>> {
- // Convert `all_values` to `ListArray` and return a single List ScalarValue
-
- // Build offsets
- let offsets =
- OffsetBuffer::new(ScalarBuffer::from(vec![0, self.all_values.len() as i32]));
-
- // Build inner array
- let values_array = PrimitiveArray::<T>::new(
- ScalarBuffer::from(std::mem::take(&mut self.all_values)),
- None,
- )
- .with_data_type(self.data_type.clone());
-
- // Build the result list array
- let list_array = ListArray::new(
- Arc::new(Field::new_list_field(self.data_type.clone(), true)),
- offsets,
- Arc::new(values_array),
- None,
- );
-
- Ok(vec![ScalarValue::List(Arc::new(list_array))])
- }
-
- fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
- let values = values[0].as_primitive::<T>();
- let additional = values.len() - values.null_count();
- self.all_values.try_reserve(additional).map_err(|e| {
- exec_datafusion_err!(
- "failed to reserve {additional} values for median accumulator: {e}"
- )
- })?;
- if values.null_count() > 0 {
- self.all_values.extend(values.iter().flatten());
- } else {
- // Fast path: no nulls, so the values buffer can be appended wholesale.
- self.all_values.extend_from_slice(values.values());
- }
- Ok(())
- }
-
- fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
- let array = states[0].as_list::<i32>();
- for v in array.iter().flatten() {
- self.update_batch(&[v])?
- }
- Ok(())
- }
-
- fn evaluate(&mut self) -> Result<ScalarValue> {
- let median = calculate_median::<T>(&mut self.all_values);
- ScalarValue::new_primitive::<T>(median, &self.data_type)
- }
-
- fn size(&self) -> usize {
- size_of_val(self) + self.all_values.capacity() * size_of::<T::Native>()
- }
-
- fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
- let mut to_remove: HashMap<Hashable<T::Native>, usize, RandomState> =
- HashMap::default();
-
- let arr = values[0].as_primitive::<T>();
- if arr.null_count() > 0 {
- for value in arr.iter().flatten() {
- *to_remove.entry(Hashable(value)).or_default() += 1;
- }
- } else {
- // Fast path: no nulls, so skip the per-element validity check.
- for value in arr.values().iter() {
- *to_remove.entry(Hashable(*value)).or_default() += 1;
- }
- }
-
- let mut i = 0;
- while i < self.all_values.len() {
- let k = Hashable(self.all_values[i]);
- if let Some(count) = to_remove.get_mut(&k)
- && *count > 0
- {
- self.all_values.swap_remove(i);
- *count -= 1;
- if *count == 0 {
- to_remove.remove(&k);
- if to_remove.is_empty() {
- break;
- }
- }
- } else {
- i += 1;
- }
- }
-
- // Retracting values that are not tracked means the accumulator state
- // has diverged from the window frame; continuing would silently
- // produce wrong results, so surface it as an error.
- if !to_remove.is_empty() {
- return internal_err!(
- "median retract_batch: retracted value(s) not present in the window"
- );
- }
- Ok(())
- }
-
- fn supports_retract_batch(&self) -> bool {
- true
- }
-}
-
-/// The median groups accumulator accumulates the raw input values
-///
-/// For calculating the accurate medians of groups, we need to store all values
-/// of groups before final evaluation.
-/// So values in each group will be stored in a `Vec<T>`, and the total group values
-/// will be actually organized as a `Vec<Vec<T>>`.
-#[derive(Debug)]
-struct MedianGroupsAccumulator<T: ArrowNumericType + Send> {
- data_type: DataType,
- group_values: Vec<Vec<T::Native>>,
-}
-
-impl<T: ArrowNumericType + Send> MedianGroupsAccumulator<T> {
- pub fn new(data_type: DataType) -> Self {
- Self {
- data_type,
- group_values: Vec::new(),
- }
- }
-}
-
-impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<T> {
- fn update_batch(
- &mut self,
- values: &[ArrayRef],
- group_indices: &[usize],
- opt_filter: Option<&BooleanArray>,
- total_num_groups: usize,
- ) -> Result<()> {
- assert_eq!(values.len(), 1, "single argument to update_batch");
- let values = values[0].as_primitive::<T>();
-
- // Push the `not nulls + not filtered` row into its group
- self.group_values.resize(total_num_groups, Vec::new());
- accumulate(
- group_indices,
- values,
- opt_filter,
- |group_index, new_value| {
- self.group_values[group_index].push(new_value);
- },
- );
-
- Ok(())
- }
-
- fn merge_batch(
- &mut self,
- values: &[ArrayRef],
- group_indices: &[usize],
- total_num_groups: usize,
- ) -> Result<()> {
- assert_eq!(values.len(), 1, "one argument to merge_batch");
-
- // The merged values should be organized like as a `ListArray` which is nullable
- // (input with nulls usually generated from `convert_to_state`), but `inner array` of
- // `ListArray` is `non-nullable`.
- //
- // Following is the possible and impossible input `values`:
- //
- // # Possible values
- // ```text
- // group 0: [1, 2, 3]
- // group 1: null (list array is nullable)
- // group 2: [6, 7, 8]
- // ...
- // group n: [...]
- // ```
- //
- // # Impossible values
- // ```text
- // group x: [1, 2, null] (values in list array is non-nullable)
- // ```
- //
- let input_group_values = values[0].as_list::<i32>();
-
- // Ensure group values big enough
- self.group_values.resize(total_num_groups, Vec::new());
-
- // Extend values to related groups
- // TODO: avoid using iterator of the `ListArray`, this will lead to
- // many calls of `slice` of its ``inner array`, and `slice` is not
- // so efficient(due to the calculation of `null_count` for each `slice`).
- group_indices
- .iter()
- .zip(input_group_values.iter())
- .for_each(|(&group_index, values_opt)| {
- if let Some(values) = values_opt {
- let values = values.as_primitive::<T>();
- self.group_values[group_index].extend(values.values().iter());
- }
- });
-
- Ok(())
- }
-
- fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
- // Emit values
- let emit_group_values = emit_to.take_needed(&mut self.group_values);
-
- // Build offsets
- let mut offsets = Vec::with_capacity(self.group_values.len() + 1);
- offsets.push(0);
- let mut cur_len = 0_i32;
- for group_value in &emit_group_values {
- cur_len += group_value.len() as i32;
- offsets.push(cur_len);
- }
- // TODO: maybe we can use `OffsetBuffer::new_unchecked` like what in `convert_to_state`,
- // but safety should be considered more carefully here(and I am not sure if it can get
- // performance improvement when we introduce checks to keep the safety...).
- //
- // Can see more details in:
- // https://github.com/apache/datafusion/pull/13681#discussion_r1931209791
- //
- let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets));
-
- // Build inner array
- let flatten_group_values =
- emit_group_values.into_iter().flatten().collect::<Vec<_>>();
- let group_values_array =
- PrimitiveArray::<T>::new(ScalarBuffer::from(flatten_group_values), None)
- .with_data_type(self.data_type.clone());
-
- // Build the result list array
- let result_list_array = ListArray::new(
- Arc::new(Field::new_list_field(self.data_type.clone(), true)),
- offsets,
- Arc::new(group_values_array),
- None,
- );
-
- Ok(vec![Arc::new(result_list_array)])
- }
-
- fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
- // Emit values
- let emit_group_values = emit_to.take_needed(&mut self.group_values);
-
- // Calculate median for each group
- let mut evaluate_result_builder =
- PrimitiveBuilder::<T>::new().with_data_type(self.data_type.clone());
- for mut values in emit_group_values {
- let median = calculate_median::<T>(&mut values);
- evaluate_result_builder.append_option(median);
- }
-
- Ok(Arc::new(evaluate_result_builder.finish()))
- }
-
- fn convert_to_state(
- &self,
- values: &[ArrayRef],
- opt_filter: Option<&BooleanArray>,
- ) -> Result<Vec<ArrayRef>> {
- assert_eq!(values.len(), 1, "one argument to merge_batch");
-
- let input_array = values[0].as_primitive::<T>();
-
- // Directly convert the input array to states, each row will be
- // seen as a respective group.
- // For detail, the `input_array` will be converted to a `ListArray`.
- // And if row is `not null + not filtered`, it will be converted to a list
- // with only one element; otherwise, this row in `ListArray` will be set
- // to null.
-
- // Reuse values buffer in `input_array` to build `values` in `ListArray`
- let values = PrimitiveArray::<T>::new(input_array.values().clone(), None)
- .with_data_type(self.data_type.clone());
-
- // `offsets` in `ListArray`, each row as a list element
- let offset_end = i32::try_from(input_array.len()).map_err(|e| {
- internal_datafusion_err!(
- "cast array_len to i32 failed in convert_to_state of group median, err:{e:?}"
- )
- })?;
- let offsets = (0..=offset_end).collect::<Vec<_>>();
- // Safety: all checks in `OffsetBuffer::new` are ensured to pass
- let offsets = unsafe { OffsetBuffer::new_unchecked(ScalarBuffer::from(offsets)) };
-
- // `nulls` for converted `ListArray`
- let nulls = filtered_null_mask(opt_filter, input_array);
-
- let converted_list_array = ListArray::new(
- Arc::new(Field::new_list_field(self.data_type.clone(), true)),
- offsets,
- Arc::new(values),
- nulls,
- );
-
- Ok(vec![Arc::new(converted_list_array)])
- }
- fn size(&self) -> usize {
- self.group_values
- .iter()
- .map(|values| values.capacity() * size_of::<T::Native>())
- .sum::<usize>()
- // account for size of self.group_values too
- + self.group_values.capacity() * size_of::<Vec<T::Native>>()
- + size_of::<Vec<Vec<T::Native>>>()
- }
-}
-
-#[derive(Debug)]
-struct DistinctMedianAccumulator<T: ArrowNumericType> {
- distinct_values: GenericDistinctBuffer<T>,
- data_type: DataType,
-}
-
-impl<T: ArrowNumericType + Debug> Accumulator for DistinctMedianAccumulator<T> {
- fn state(&mut self) -> Result<Vec<ScalarValue>> {
- self.distinct_values.state()
- }
-
- fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
- self.distinct_values.update_batch(values)
- }
-
- fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
- self.distinct_values.merge_batch(states)
- }
-
- fn evaluate(&mut self) -> Result<ScalarValue> {
- let mut d: Vec<T::Native> =
- self.distinct_values.values.iter().map(|v| v.0).collect();
- let median = calculate_median::<T>(&mut d);
- ScalarValue::new_primitive::<T>(median, &self.data_type)
- }
-
- fn size(&self) -> usize {
- size_of_val(self) + self.distinct_values.size()
- }
-}
-
-/// Get maximum entry in the slice,
-fn slice_max<T>(array: &[T::Native]) -> T::Native
-where
- T: ArrowPrimitiveType,
- T::Native: PartialOrd, // Ensure the type supports PartialOrd for comparison
-{
- // Make sure that, array is not empty.
- debug_assert!(!array.is_empty());
- // `.unwrap()` is safe here as the array is supposed to be non-empty
- *array
- .iter()
- .max_by(|x, y| x.partial_cmp(y).unwrap_or(Ordering::Less))
- .unwrap()
-}
-
-fn calculate_median<T: ArrowNumericType>(values: &mut [T::Native]) -> Option<T::Native> {
- let cmp = |x: &T::Native, y: &T::Native| x.compare(*y);
-
- let len = values.len();
- if len == 0 {
- None
- } else if len % 2 == 0 {
- let (low, high, _) = values.select_nth_unstable_by(len / 2, cmp);
- // Get the maximum of the low (left side after bi-partitioning)
- let left_max = slice_max::<T>(low);
- // Calculate median as the average of the two middle values.
- // Use checked arithmetic to detect overflow and fall back to safe formula.
- let two = T::Native::usize_as(2);
- let median = match left_max.add_checked(*high) {
- Ok(sum) => sum.div_wrapping(two),
- Err(_) => {
- // Overflow detected - use safe midpoint formula:
- // a/2 + b/2 + ((a%2 + b%2) / 2)
- // This avoids overflow by dividing before adding.
- let half_left = left_max.div_wrapping(two);
- let half_right = (*high).div_wrapping(two);
- let rem_left = left_max.mod_wrapping(two);
- let rem_right = (*high).mod_wrapping(two);
- // The sum of remainders (0, 1, or 2 for unsigned; -2 to 2 for signed)
- // divided by 2 gives the correction factor (0 or 1 for unsigned; -1, 0, or 1 for signed)
- let correction = rem_left.add_wrapping(rem_right).div_wrapping(two);
- half_left.add_wrapping(half_right).add_wrapping(correction)
- }
- };
- Some(median)
- } else {
- let (_, median, _) = values.select_nth_unstable_by(len / 2, cmp);
- Some(*median)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use arrow::array::Float64Array;
-
- fn median_accumulator() -> MedianAccumulator<Float64Type> {
- MedianAccumulator {
- data_type: DataType::Float64,
- all_values: vec![],
- }
- }
-
- #[test]
- fn retract_batch_errors_on_untracked_value() {
- let mut acc = median_accumulator();
- let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));
- acc.update_batch(std::slice::from_ref(&values)).unwrap();
-
- let retract: ArrayRef = Arc::new(Float64Array::from(vec![3.0]));
- let err = acc
- .retract_batch(std::slice::from_ref(&retract))
- .unwrap_err()
- .to_string();
- assert!(
- err.contains("not present in the window"),
- "unexpected error: {err}"
- );
- }
-
- #[test]
- fn update_batch_with_and_without_nulls_agree() {
- // The null-free fast path must accumulate the same values as the
- // general path.
- let dense: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));
- let sparse: ArrayRef = Arc::new(Float64Array::from(vec![
- Some(1.0),
- None,
- Some(2.0),
- None,
- Some(3.0),
- ]));
-
- let mut dense_acc = median_accumulator();
- dense_acc
- .update_batch(std::slice::from_ref(&dense))
- .unwrap();
- let mut sparse_acc = median_accumulator();
- sparse_acc
- .update_batch(std::slice::from_ref(&sparse))
- .unwrap();
-
- assert_eq!(dense_acc.all_values, sparse_acc.all_values);
- }
-}
diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rs
index c026a3d3a5..2499e18f27 100644
--- a/datafusion/functions-aggregate/src/percentile_cont.rs
+++ b/datafusion/functions-aggregate/src/percentile_cont.rs
@@ -17,11 +17,13 @@
use std::collections::HashMap;
use std::fmt::Debug;
+use std::marker::PhantomData;
use std::mem::{size_of, size_of_val};
use std::sync::Arc;
use arrow::array::{
- ArrowNumericType, BooleanArray, ListArray, PrimitiveArray, PrimitiveBuilder,
+ ArrowNumericType, ArrowPrimitiveType, BooleanArray, ListArray, PrimitiveArray,
+ PrimitiveBuilder,
};
use arrow::buffer::{OffsetBuffer, ScalarBuffer};
use arrow::{
@@ -32,6 +34,10 @@ use arrow::{
use num_traits::AsPrimitive;
use arrow::array::ArrowNativeTypeOp;
+use arrow::datatypes::{
+ ArrowNativeType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,
+ DecimalType,
+};
use datafusion_common::hash_utils::RandomState;
use datafusion_common::internal_err;
use datafusion_common::types::{NativeType, logical_float64};
@@ -39,6 +45,7 @@ use datafusion_common::utils::memory::estimate_memory_size;
use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;
use crate::min_max::{max_udaf, min_udaf};
+use crate::utils::validate_percentile_expr;
use datafusion_common::{
Result, ScalarValue, exec_datafusion_err, internal_datafusion_err,
utils::{SingleRowListArrayBuilder, take_function_args},
@@ -46,7 +53,7 @@ use datafusion_common::{
use datafusion_expr::utils::format_state_name;
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, Documentation, Expr, Signature,
- TypeSignatureClass, Volatility,
+ TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_expr::{EmitTo, GroupsAccumulator};
use datafusion_expr::{
@@ -59,8 +66,6 @@ use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls:
use datafusion_functions_aggregate_common::utils::Hashable;
use datafusion_macros::user_doc;
-use crate::utils::validate_percentile_expr;
-
/// Precision multiplier for linear interpolation calculations.
///
/// This value of 1,000,000 was chosen to balance precision with overflow safety:
@@ -72,9 +77,9 @@ use crate::utils::validate_percentile_expr;
/// is computed as: `lower + ((upper - lower) * (fraction * PRECISION)) / PRECISION`
/// to avoid floating-point operations on integer types while maintaining precision.
///
-/// The interpolation arithmetic is performed in f64 and then cast back to the
+/// The interpolation arithmetic for floats is performed in f64 and then cast back to the
/// native type to avoid overflowing Float16 intermediates.
-const INTERPOLATION_PRECISION: f64 = 1_000_000.0;
+const INTERPOLATION_PRECISION: usize = 1_000_000;
create_func!(PercentileCont, percentile_cont_udaf);
@@ -143,18 +148,30 @@ impl Default for PercentileCont {
impl PercentileCont {
pub fn new() -> Self {
Self {
- signature: Signature::coercible(
+ signature: Signature::one_of(
vec![
- Coercion::new_implicit(
- TypeSignatureClass::Float,
- vec![TypeSignatureClass::Numeric],
- NativeType::Float64,
- ),
- Coercion::new_implicit(
- TypeSignatureClass::Native(logical_float64()),
- vec![TypeSignatureClass::Numeric],
- NativeType::Float64,
- ),
+ // Decimal signature: decimals, percentile
+ TypeSignature::Coercible(vec![
+ // value
+ Coercion::new_exact(TypeSignatureClass::Decimal),
+ // percentile
+ Coercion::new_implicit_native(
+ logical_float64(),
+ vec![TypeSignatureClass::Numeric],
+ ),
+ ]),
+ // Float signature: float, percentile
+ TypeSignature::Coercible(vec![
+ Coercion::new_implicit(
+ TypeSignatureClass::Float,
+ vec![TypeSignatureClass::Numeric],
+ NativeType::Float64,
+ ),
+ Coercion::new_implicit_native(
+ logical_float64(),
+ vec![TypeSignatureClass::Numeric],
+ ),
+ ]),
],
Volatility::Immutable,
)
@@ -179,10 +196,7 @@ impl AggregateUDFImpl for PercentileCont {
}
fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- match &arg_types[0] {
- DataType::Null => Ok(DataType::Float64),
- dt => Ok(dt.clone()),
- }
+ Ok(arg_types[0].clone())
}
fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
@@ -216,40 +230,14 @@ impl AggregateUDFImpl for PercentileCont {
}
fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {
+ // Always verify percentiles
let percentile = get_percentile(&args)?;
-
- let input_dt = args.expr_fields[0].data_type();
- if input_dt.is_null() {
- return Ok(Box::new(NoopAccumulator::new(ScalarValue::Float64(None))));
- }
-
- if args.is_distinct {
- match input_dt {
- DataType::Float16 => Ok(Box::new(DistinctPercentileContAccumulator::<
- Float16Type,
- >::new(percentile))),
- DataType::Float32 => Ok(Box::new(DistinctPercentileContAccumulator::<
- Float32Type,
- >::new(percentile))),
- DataType::Float64 => Ok(Box::new(DistinctPercentileContAccumulator::<
- Float64Type,
- >::new(percentile))),
- dt => internal_err!("Unsupported datatype for percentile cont: {dt}"),
- }
- } else {
- match input_dt {
- DataType::Float16 => Ok(Box::new(
- PercentileContAccumulator::<Float16Type>::new(percentile),
- )),
- DataType::Float32 => Ok(Box::new(
- PercentileContAccumulator::<Float32Type>::new(percentile),
- )),
- DataType::Float64 => Ok(Box::new(
- PercentileContAccumulator::<Float64Type>::new(percentile),
- )),
- dt => internal_err!("Unsupported datatype for percentile cont: {dt}"),
- }
- }
+ create_percentile_accumulator(
+ self.name(),
+ percentile,
+ args.expr_fields[0].data_type(),
+ args.is_distinct,
+ )
}
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
@@ -260,21 +248,13 @@ impl AggregateUDFImpl for PercentileCont {
&self,
args: AccumulatorArgs,
) -> Result<Box<dyn GroupsAccumulator>> {
+ // Always verify percentiles
let percentile = get_percentile(&args)?;
-
- let input_dt = args.expr_fields[0].data_type();
- match input_dt {
- DataType::Float16 => Ok(Box::new(PercentileContGroupsAccumulator::<
- Float16Type,
- >::new(percentile))),
- DataType::Float32 => Ok(Box::new(PercentileContGroupsAccumulator::<
- Float32Type,
- >::new(percentile))),
- DataType::Float64 => Ok(Box::new(PercentileContGroupsAccumulator::<
- Float64Type,
- >::new(percentile))),
- dt => internal_err!("Unsupported datatype for percentile cont: {dt}"),
- }
+ create_percentile_groups_accumulator(
+ self.name(),
+ percentile,
+ args.expr_fields[0].data_type(),
+ )
}
fn simplify(&self) -> Option<AggregateFunctionSimplification> {
@@ -310,6 +290,85 @@ fn get_percentile(args: &AccumulatorArgs) -> Result<f64> {
Ok(percentile)
}
+pub fn create_percentile_accumulator(
+ name: &str,
+ percentile: f64,
+ input_dt: &DataType,
+ is_distinct: bool,
+) -> Result<Box<dyn Accumulator>> {
+ // Null input evaluates to null
+ if input_dt.is_null() {
+ return Ok(Box::new(NoopAccumulator::default()));
+ }
+
+ macro_rules! helper {
+ ($t:ty, $i:ty, $dt:expr) => {
+ if is_distinct {
+ Ok(Box::new(DistinctPercentileContAccumulator::<$t, $i>::new(
+ percentile,
+ $dt.clone(),
+ )))
+ } else {
+ Ok(Box::new(PercentileContAccumulator::<$t, $i>::new(
+ percentile,
+ $dt.clone(),
+ )))
+ }
+ };
+ }
+ match input_dt {
+ DataType::Float16 => helper!(Float16Type, FloatInterpolator, input_dt),
+ DataType::Float32 => helper!(Float32Type, FloatInterpolator, input_dt),
+ DataType::Float64 => helper!(Float64Type, FloatInterpolator, input_dt),
+ DataType::Decimal32(_, _) => {
+ helper!(Decimal32Type, DecimalInterpolator, input_dt)
+ }
+ DataType::Decimal64(_, _) => {
+ helper!(Decimal64Type, DecimalInterpolator, input_dt)
+ }
+ DataType::Decimal128(_, _) => {
+ helper!(Decimal128Type, DecimalInterpolator, input_dt)
+ }
+ DataType::Decimal256(_, _) => {
+ helper!(Decimal256Type, DecimalInterpolator, input_dt)
+ }
+ dt => internal_err!("Unsupported datatype for {} with {}", name, dt),
+ }
+}
+
+pub fn create_percentile_groups_accumulator(
+ name: &str,
+ percentile: f64,
+ input_dt: &DataType,
+) -> Result<Box<dyn GroupsAccumulator>> {
+ macro_rules! helper {
+ ($t:ty, $i:ty, $dt:expr) => {
+ Ok(Box::new(PercentileContGroupsAccumulator::<$t, $i>::new(
+ percentile,
+ $dt.clone(),
+ )))
+ };
+ }
+ match input_dt {
+ DataType::Float16 => helper!(Float16Type, FloatInterpolator, input_dt),
+ DataType::Float32 => helper!(Float32Type, FloatInterpolator, input_dt),
+ DataType::Float64 => helper!(Float64Type, FloatInterpolator, input_dt),
+ DataType::Decimal32(_, _) => {
+ helper!(Decimal32Type, DecimalInterpolator, input_dt)
+ }
+ DataType::Decimal64(_, _) => {
+ helper!(Decimal64Type, DecimalInterpolator, input_dt)
+ }
+ DataType::Decimal128(_, _) => {
+ helper!(Decimal128Type, DecimalInterpolator, input_dt)
+ }
+ DataType::Decimal256(_, _) => {
+ helper!(Decimal256Type, DecimalInterpolator, input_dt)
+ }
+ dt => internal_err!("Unsupported datatype for {} with {}", name, dt),
+ }
+}
+
fn simplify_percentile_cont_aggregate(
aggregate_function: AggregateFunction,
info: &SimplifyContext,
@@ -377,25 +436,33 @@ fn simplify_percentile_cont_aggregate(
/// in the final evaluation step so that we avoid expensive conversions and
/// allocations during `update_batch`.
#[derive(Debug)]
-struct PercentileContAccumulator<T: ArrowNumericType + Debug> {
+struct PercentileContAccumulator<
+ T: ArrowNumericType + Debug,
+ I: PercentileInterpolator<T>,
+> {
all_values: Vec<T::Native>,
percentile: f64,
+ data_type: DataType,
+ _interpolator: PhantomData<I>,
}
-impl<T: ArrowNumericType + Debug> PercentileContAccumulator<T> {
- fn new(percentile: f64) -> Self {
+impl<T: ArrowNumericType + Debug, I: PercentileInterpolator<T>>
+ PercentileContAccumulator<T, I>
+{
+ fn new(percentile: f64, data_type: DataType) -> Self {
Self {
all_values: vec![],
percentile,
+ data_type,
+ _interpolator: PhantomData,
}
}
}
-impl<T> Accumulator for PercentileContAccumulator<T>
+impl<T, I> Accumulator for PercentileContAccumulator<T, I>
where
T: ArrowNumericType + Debug,
- T::Native: Copy + AsPrimitive<f64>,
- f64: AsPrimitive<T::Native>,
+ I: PercentileInterpolator<T> + 'static,
{
fn state(&mut self) -> Result<Vec<ScalarValue>> {
// Convert `all_values` to `ListArray` and return a single List ScalarValue
@@ -408,11 +475,12 @@ where
let values_array = PrimitiveArray::<T>::new(
ScalarBuffer::from(std::mem::take(&mut self.all_values)),
None,
- );
+ )
+ .with_data_type(self.data_type.clone());
// Build the result list array
let list_array = ListArray::new(
- Arc::new(Field::new_list_field(T::DATA_TYPE, true)),
+ Arc::new(Field::new_list_field(self.data_type.clone(), true)),
offsets,
Arc::new(values_array),
None,
@@ -440,13 +508,16 @@ where
fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {
let array = states[0].as_list::<i32>();
- self.update_batch(&[array.value(0)])?;
+ // Feed all list elements from a batch
+ for values in array.iter().flatten() {
+ self.update_batch(&[values])?;
+ }
Ok(())
}
fn evaluate(&mut self) -> Result<ScalarValue> {
- let value = calculate_percentile::<T>(&mut self.all_values, self.percentile);
- ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)
+ let value = calculate_percentile::<T, I>(&mut self.all_values, self.percentile)?;
+ ScalarValue::new_primitive::<T>(value, &self.data_type)
}
fn size(&self) -> usize {
@@ -511,25 +582,33 @@ where
/// So values in each group will be stored in a `Vec<T>`, and the total group values
/// will be actually organized as a `Vec<Vec<T>>`.
#[derive(Debug)]
-struct PercentileContGroupsAccumulator<T: ArrowNumericType + Send> {
+struct PercentileContGroupsAccumulator<
+ T: ArrowNumericType + Debug,
+ I: PercentileInterpolator<T>,
+> {
group_values: Vec<Vec<T::Native>>,
percentile: f64,
+ data_type: DataType,
+ _interpolator: PhantomData<I>,
}
-impl<T: ArrowNumericType + Send> PercentileContGroupsAccumulator<T> {
- fn new(percentile: f64) -> Self {
+impl<T: ArrowNumericType + Debug, I: PercentileInterpolator<T>>
+ PercentileContGroupsAccumulator<T, I>
+{
+ fn new(percentile: f64, data_type: DataType) -> Self {
Self {
group_values: vec![],
percentile,
+ data_type,
+ _interpolator: PhantomData,
}
}
}
-impl<T> GroupsAccumulator for PercentileContGroupsAccumulator<T>
+impl<T, I> GroupsAccumulator for PercentileContGroupsAccumulator<T, I>
where
- T: ArrowNumericType + Send,
- T::Native: Copy + AsPrimitive<f64>,
- f64: AsPrimitive<T::Native>,
+ T: ArrowNumericType + Debug + Send,
+ I: PercentileInterpolator<T> + 'static,
{
fn update_batch(
&mut self,
@@ -602,11 +681,12 @@ where
let flatten_group_values =
emit_group_values.into_iter().flatten().collect::<Vec<_>>();
let group_values_array =
- PrimitiveArray::<T>::new(ScalarBuffer::from(flatten_group_values), None);
+ PrimitiveArray::<T>::new(ScalarBuffer::from(flatten_group_values), None)
+ .with_data_type(self.data_type.clone());
// Build the result list array
let result_list_array = ListArray::new(
- Arc::new(Field::new_list_field(T::DATA_TYPE, true)),
+ Arc::new(Field::new_list_field(self.data_type.clone(), true)),
offsets,
Arc::new(group_values_array),
None,
@@ -621,9 +701,11 @@ where
// Calculate percentile for each group
let mut evaluate_result_builder =
- PrimitiveBuilder::<T>::with_capacity(emit_group_values.len());
+ PrimitiveBuilder::<T>::with_capacity(emit_group_values.len())
+ .with_data_type(self.data_type.clone());
for values in &mut emit_group_values {
- let value = calculate_percentile::<T>(values.as_mut_slice(), self.percentile);
+ let value =
+ calculate_percentile::<T, I>(values.as_mut_slice(), self.percentile)?;
evaluate_result_builder.append_option(value);
}
@@ -647,7 +729,8 @@ where
// to null.
// Reuse values buffer in `input_array` to build `values` in `ListArray`
- let values = PrimitiveArray::<T>::new(input_array.values().clone(), None);
+ let values = PrimitiveArray::<T>::new(input_array.values().clone(), None)
+ .with_data_type(self.data_type.clone());
// `offsets` in `ListArray`, each row as a list element
let offset_end = i32::try_from(input_array.len()).map_err(|e| {
@@ -668,7 +751,7 @@ where
let nulls = filtered_null_mask(opt_filter, input_array);
let converted_list_array = ListArray::new(
- Arc::new(Field::new_list_field(T::DATA_TYPE, true)),
+ Arc::new(Field::new_list_field(self.data_type.clone(), true)),
offsets,
Arc::new(values),
nulls,
@@ -694,7 +777,10 @@ where
/// left the window frame. The percentile is then computed over the set of keys
/// with a positive count.
#[derive(Debug)]
-struct DistinctPercentileContAccumulator<T: ArrowNumericType> {
+struct DistinctPercentileContAccumulator<
+ T: ArrowNumericType,
+ I: PercentileInterpolator<T>,
+> {
/// Distinct value -> number of in-window rows carrying it.
///
/// Uses the same fast (foldhash) `RandomState` as the shared
@@ -702,22 +788,27 @@ struct DistinctPercentileContAccumulator<T: ArrowNumericType> {
/// SipHash, which is considerably slower for this hot path.
counts: HashMap<Hashable<T::Native>, usize, RandomState>,
percentile: f64,
+ data_type: DataType,
+ _interpolator: PhantomData<I>,
}
-impl<T: ArrowNumericType + Debug> DistinctPercentileContAccumulator<T> {
- fn new(percentile: f64) -> Self {
+impl<T: ArrowNumericType, I: PercentileInterpolator<T>>
+ DistinctPercentileContAccumulator<T, I>
+{
+ fn new(percentile: f64, data_type: DataType) -> Self {
Self {
counts: HashMap::default(),
percentile,
+ data_type,
+ _interpolator: PhantomData,
}
}
}
-impl<T> Accumulator for DistinctPercentileContAccumulator<T>
+impl<T, I> Accumulator for DistinctPercentileContAccumulator<T, I>
where
T: ArrowNumericType + Debug,
- T::Native: Copy + AsPrimitive<f64>,
- f64: AsPrimitive<T::Native>,
+ I: PercentileInterpolator<T> + 'static,
{
fn state(&mut self) -> Result<Vec<ScalarValue>> {
// Emit the distinct keys as a single List scalar, matching the state
@@ -726,7 +817,7 @@ where
// cross-partition merges only need the distinct key set.
let arr = Arc::new(
PrimitiveArray::<T>::from_iter_values(self.counts.keys().map(|v| v.0))
- .with_data_type(T::DATA_TYPE),
+ .with_data_type(self.data_type.clone()),
);
Ok(vec![
SingleRowListArrayBuilder::new(arr).build_list_scalar(),
@@ -763,8 +854,8 @@ where
fn evaluate(&mut self) -> Result<ScalarValue> {
let mut values: Vec<T::Native> = self.counts.keys().map(|v| v.0).collect();
- let value = calculate_percentile::<T>(&mut values, self.percentile);
- ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)
+ let value = calculate_percentile::<T, I>(&mut values, self.percentile)?;
+ ScalarValue::new_primitive::<T>(value, &self.data_type)
}
fn size(&self) -> usize {
@@ -816,6 +907,126 @@ where
}
}
+/// A trait to abstract interpolation logic for percentile calculation
+/// for floats and decimals.
+trait PercentileInterpolator<T: ArrowNumericType>: Debug + Sync + Send {
+ fn interpolate(
+ lower: T::Native,
+ upper: T::Native,
+ fraction: f64,
+ ) -> Result<T::Native>;
+}
+
+#[derive(Debug)]
+struct FloatInterpolator;
+
+/// The interpolation arithmetic for floats is performed in f64 and then cast back to the
+/// native type to avoid overflowing Float16 intermediates.
+impl<T> PercentileInterpolator<T> for FloatInterpolator
+where
+ T: ArrowNumericType,
+ T::Native: AsPrimitive<f64>,
+ f64: AsPrimitive<T::Native>,
+{
+ fn interpolate(
+ lower: T::Native,
+ upper: T::Native,
+ fraction: f64,
+ ) -> Result<T::Native> {
+ // Linear interpolation.
+ // We compute a quantized interpolation weight using `FLOAT_INTERPOLATION_PRECISION` because:
+ // 1. Both values come from the input data, so (upper - lower) is bounded by the value range
+ // 2. fraction is between 0 and 1; quantizing it provides stable, predictable results
+ // 3. The result is guaranteed to be between lower_value and upper_value (modulo cast rounding)
+ // 4. Arithmetic is performed in f64 and cast back to avoid overflowing Float16 intermediates
+ let scaled = (fraction * (INTERPOLATION_PRECISION as f64)) as usize;
+ let weight = scaled as f64 / (INTERPOLATION_PRECISION as f64);
+
+ let lower_f: f64 = lower.as_();
+ let upper_f: f64 = upper.as_();
+ let interpolated_f = lower_f + (upper_f - lower_f) * weight;
+ Ok(interpolated_f.as_())
+ }
+}
+
+#[derive(Debug)]
+struct DecimalInterpolator;
+
+/// Precision multiplier for decimal linear interpolation calculations.
+fn deduce_interpolation_precision<T: DecimalType>() -> usize {
+ if T::BYTE_LENGTH == 4 {
+ // Avoid overflow in `scale_by_num` with 32-bit Decimal32
+ 10_000
+ } else {
+ INTERPOLATION_PRECISION
+ }
+}
+
+/// Compute a scaled value for interpolation using a formula `trunc(x * num / precision)`
+/// The numerical method is separating `q * num + r * num / den`
+/// where `q = x / precision` and `r = x % precision`
+fn scale_by_num<T>(x: T::Native, num: T::Native, den: T::Native) -> Result<T::Native>
+where
+ T: DecimalType,
+{
+ debug_assert!(num >= T::Native::ZERO);
+ debug_assert!(num <= den);
+
+ let q = x.div_wrapping(den);
+ let r = x.mod_wrapping(den);
+ // `q * num` cannot exceed `x`; `r * num` cannot exceed `(den - 1)^2`
+
+ let a = q.mul_checked(num)?;
+ let b = r.mul_checked(num)?.div_wrapping(den);
+ a.add_checked(b)
+ .map_err(|e| exec_datafusion_err!("Arithmetic overflow in percentile_cont: {e}"))
+}
+
+impl<T> PercentileInterpolator<T> for DecimalInterpolator
+where
+ T: DecimalType,
+{
+ fn interpolate(
+ lower: T::Native,
+ upper: T::Native,
+ fraction: f64,
+ ) -> Result<T::Native> {
+ debug_assert!((0.0..=1.0).contains(&fraction));
+ debug_assert!(lower <= upper);
+
+ let interpolation_precision = deduce_interpolation_precision::<T>();
+
+ let num =
+ T::Native::usize_as((fraction * interpolation_precision as f64) as usize);
+ let den = T::Native::usize_as(interpolation_precision);
+
+ // Happy path: `upper - lower` does not overflow
+ // (could be a case for Decimal128 with max precision)
+ if let Ok(delta) = upper.sub_checked(lower) {
+ // Calculate the interpolation weight with the formula, where den is the precision:
+ // `lower + (upper - lower) * num / den`
+ let scaled: T::Native = scale_by_num::<T>(delta, num, den)?;
+ lower.add_checked(scaled).map_err(|e| {
+ exec_datafusion_err!("Arithmetic overflow in percentile_cont: {e}")
+ })
+ } else {
+ // Avoid overflow with the subtraction - split to two additive parts
+ // `a = lower * (precision-num) / precision`
+ // `b = upper * num / precision`
+ // The weights sum to 1, so the result is bounded by max(|lower|, |upper|)
+ // and never overflows, at the cost of a second truncation (2 ULP not 1).
+ let num_a = den.sub_wrapping(num);
+ let a: T::Native = scale_by_num::<T>(lower, num_a, den)?;
+
+ let b: T::Native = scale_by_num::<T>(upper, num, den)?;
+
+ a.add_checked(b).map_err(|e| {
+ exec_datafusion_err!("Arithmetic overflow in percentile_cont: {e}")
+ })
+ }
+ }
+}
+
/// Calculate the percentile value for a given set of values.
/// This function performs an exact calculation by sorting all values.
///
@@ -827,37 +1038,33 @@ where
/// Note: This function takes a mutable slice and sorts it in place, but does not
/// consume the data. This is important for window frame queries where evaluate()
/// may be called multiple times on the same accumulator state.
-fn calculate_percentile<T: ArrowNumericType>(
+fn calculate_percentile<T: ArrowPrimitiveType, I: PercentileInterpolator<T>>(
values: &mut [T::Native],
percentile: f64,
-) -> Option<T::Native>
-where
- T::Native: Copy + AsPrimitive<f64>,
- f64: AsPrimitive<T::Native>,
-{
+) -> Result<Option<T::Native>> {
let cmp = |x: &T::Native, y: &T::Native| x.compare(*y);
let len = values.len();
if len == 0 {
- None
+ Ok(None)
} else if len == 1 {
- Some(values[0])
+ Ok(Some(values[0]))
} else if percentile == 0.0 {
// Get minimum value
- Some(
+ Ok(Some(
*values
.iter()
.min_by(|a, b| cmp(a, b))
.expect("we checked for len > 0 a few lines above"),
- )
+ ))
} else if percentile == 1.0 {
// Get maximum value
- Some(
+ Ok(Some(
*values
.iter()
.max_by(|a, b| cmp(a, b))
.expect("we checked for len > 0 a few lines above"),
- )
+ ))
} else {
// Calculate the index using the formula: p * (n - 1)
let index = percentile * ((len - 1) as f64);
@@ -867,7 +1074,7 @@ where
if lower_index == upper_index {
// Exact index, return the value at that position
let (_, value, _) = values.select_nth_unstable_by(lower_index, cmp);
- Some(*value)
+ Ok(Some(*value))
} else {
// Need to interpolate between two values
// First, partition at lower_index to get the lower value
@@ -878,20 +1085,11 @@ where
let (_, upper_value, _) = values.select_nth_unstable_by(upper_index, cmp);
let upper_value = *upper_value;
- // Linear interpolation.
- // We compute a quantized interpolation weight using `INTERPOLATION_PRECISION` because:
- // 1. Both values come from the input data, so (upper - lower) is bounded by the value range
- // 2. fraction is between 0 and 1; quantizing it provides stable, predictable results
- // 3. The result is guaranteed to be between lower_value and upper_value (modulo cast rounding)
- // 4. Arithmetic is performed in f64 and cast back to avoid overflowing Float16 intermediates
let fraction = index - (lower_index as f64);
- let scaled = (fraction * INTERPOLATION_PRECISION) as usize;
- let weight = scaled as f64 / INTERPOLATION_PRECISION;
- let lower_f: f64 = lower_value.as_();
- let upper_f: f64 = upper_value.as_();
- let interpolated_f = lower_f + (upper_f - lower_f) * weight;
- Some(interpolated_f.as_())
+ let interpolated = I::interpolate(lower_value, upper_value, fraction)?;
+
+ Ok(Some(interpolated))
}
}
}
@@ -900,11 +1098,15 @@ where
mod tests {
use super::*;
use arrow::array::Float64Array;
+ use arrow::datatypes::{Decimal64Type, Float16Type, Float64Type};
use half::f16;
#[test]
fn retract_batch_errors_on_untracked_value() {
- let mut acc = PercentileContAccumulator::<Float64Type>::new(0.5);
+ let mut acc = PercentileContAccumulator::<Float64Type, FloatInterpolator>::new(
+ 0.5,
+ DataType::Float64,
+ );
let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));
acc.update_batch(std::slice::from_ref(&values)).unwrap();
@@ -932,11 +1134,19 @@ mod tests {
Some(3.0),
]));
- let mut dense_acc = PercentileContAccumulator::<Float64Type>::new(0.5);
+ let mut dense_acc =
+ PercentileContAccumulator::<Float64Type, FloatInterpolator>::new(
+ 0.5,
+ DataType::Float64,
+ );
dense_acc
.update_batch(std::slice::from_ref(&dense))
.unwrap();
- let mut sparse_acc = PercentileContAccumulator::<Float64Type>::new(0.5);
+ let mut sparse_acc =
+ PercentileContAccumulator::<Float64Type, FloatInterpolator>::new(
+ 0.5,
+ DataType::Float64,
+ );
sparse_acc
.update_batch(std::slice::from_ref(&sparse))
.unwrap();
@@ -950,8 +1160,10 @@ mod tests {
// Interpolating between 0 and the max finite f16 value previously overflowed
// intermediate f16 computations and produced NaN.
let mut values = vec![f16::from_f32(0.0), f16::from_f32(65504.0)];
- let result = calculate_percentile::<Float16Type>(&mut values, 0.5)
- .expect("non-empty input");
+ let result =
+ calculate_percentile::<Float16Type, FloatInterpolator>(&mut values, 0.5)
+ .expect("non-empty input")
+ .expect("non-empty result");
let result_f = result.to_f32();
assert!(
!result_f.is_nan(),
@@ -963,4 +1175,59 @@ mod tests {
"unexpected result {result_f}"
);
}
+
+ #[test]
+ fn percentile_cont_decimal64() {
+ // Test values: [100.00, 200.00, 300.00, 400.00, 500.00]
+ // These are stored as i64 values scaled by 10^2
+ let mut values = vec![
+ 10000i64, // 100.00
+ 20000i64, // 200.00
+ 30000i64, // 300.00
+ 40000i64, // 400.00
+ 50000i64, // 500.00
+ ];
+
+ // Test 50th percentile (median)
+ // Should return 300.00 (30000)
+ let result =
+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 0.5)
+ .expect("evaluate failed")
+ .expect("expected Some value");
+
+ assert_eq!(result, 30000i64, "50th percentile should be 300.00");
+
+ // Test 15th percentile
+ // Should return 160.00 (16000)
+ let result =
+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 0.15)
+ .expect("evaluate failed")
+ .expect("expected Some value");
+
+ assert_eq!(result, 16000i64, "15th percentile should be 160.00");
+
+ // Test 0th percentile (minimum)
+ let mut values = vec![10000i64, 20000i64, 30000i64];
+ let result =
+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 0.0)
+ .expect("evaluate failed")
+ .expect("expected Some value");
+
+ assert_eq!(
+ result, 10000i64,
+ "0th percentile should be minimum value 100.00"
+ );
+
+ // Test 100th percentile (maximum)
+ let mut values = vec![10000i64, 20000i64, 30000i64];
+ let result =
+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 1.0)
+ .expect("evaluate failed")
+ .expect("expected Some value");
+
+ assert_eq!(
+ result, 30000i64,
+ "100th percentile should be maximum value 300.00"
+ );
+ }
}
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index e85ef5ce58..a7f08ea83a 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -3933,9 +3933,9 @@ mod tests {
// Median(a)
fn test_median_agg_expr(schema: SchemaRef) -> Result<AggregateFunctionExpr> {
- AggregateExprBuilder::new(median_udaf(), vec![col("a", &schema)?])
+ AggregateExprBuilder::new(median_udaf(), vec![col("b", &schema)?])
.schema(schema)
- .alias("MEDIAN(a)")
+ .alias("MEDIAN(b)")
.build()
}
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index 974487dac2..537c156a40 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -1172,6 +1172,39 @@ FROM distinct_pct_nulls ORDER BY id;
statement ok
DROP TABLE distinct_pct_nulls;
+# percentile_cont for shorter floats
+
+query RT
+select percentile_cont(arrow_cast(col_f32, 'Float16'), 0.5), arrow_typeof(percentile_cont(arrow_cast(col_f32, 'Float16'), 0.5)) from median_table;
+----
+2.75 Float16
+
+query RT
+select percentile_cont(col_f32, 0.5), arrow_typeof(percentile_cont(col_f32, 0.5)) from median_table;
+----
+2.75 Float32
+
+query RT
+select percentile_cont(DISTINCT arrow_cast(col_f32, 'Float16'), 0.5), arrow_typeof(percentile_cont(DISTINCT arrow_cast(col_f32, 'Float16'), 0.5)) from median_table;
+----
+2.75 Float16
+
+query RT
+select percentile_cont(DISTINCT col_f32, 0.5), arrow_typeof(percentile_cont(DISTINCT col_f32, 0.5)) from median_table;
+----
+2.75 Float32
+
+query RT
+select median(arrow_cast(col_f32, 'Float16')), arrow_typeof(median(arrow_cast(col_f32, 'Float16'))) from median_table;
+----
+2.75 Float16
+
+query RT
+select median(col_f32), arrow_typeof(median(col_f32)) from median_table;
+----
+2.75 Float32
+
+
query RT
select approx_median(arrow_cast(col_f32, 'Float16')), arrow_typeof(approx_median(arrow_cast(col_f32, 'Float16'))) from median_table;
----
@@ -4519,6 +4552,279 @@ SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY v DESC) FROM (VALUES (1), (2
2.75
+#####################
+## percentile_cont tests for decimal types
+#####################
+
+# Decimal128: typical percentiles (exact index, n=5)
+statement ok
+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (10.10), (20.20), (30.30), (40.40), (50.50);
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t
+----
+30.3 Decimal128(10, 2)
+
+query R
+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c) FROM t
+----
+20.2
+
+query R
+SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY c) FROM t
+----
+40.4
+
+query R
+SELECT percentile_cont(0.0) WITHIN GROUP (ORDER BY c) FROM t
+----
+10.1
+
+query R
+SELECT percentile_cont(1.0) WITHIN GROUP (ORDER BY c) FROM t
+----
+50.5
+
+# Alternate syntax should agree with WITHIN GROUP syntax
+query B
+SELECT percentile_cont(c, 0.5) = percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t
+----
+true
+
+statement ok
+DROP TABLE t;
+
+# Decimal128: interpolation between two values
+statement ok
+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (10.00), (20.00);
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t
+----
+15 Decimal128(10, 2)
+
+query R
+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c) FROM t
+----
+12.5
+
+query R
+SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY c) FROM t
+----
+17.5
+
+statement ok
+DROP TABLE t;
+
+# Decimal128: negative numbers and NULL handling
+statement ok
+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (-10.00), (-5.00), (0.00), (5.00), (10.00), (NULL);
+
+query R
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t
+----
+0
+
+query R
+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c) FROM t
+----
+-5
+
+statement ok
+DROP TABLE t;
+
+# Decimal128: all NULLs
+statement ok
+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (NULL);
+
+query R
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t
+----
+NULL
+
+statement ok
+DROP TABLE t;
+
+# Decimal128: single value
+statement ok
+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (42.00);
+
+query R
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t
+----
+42
+
+statement ok
+DROP TABLE t;
+
+# Decimal128: GROUP BY
+statement ok
+CREATE TABLE t (g INT, c DECIMAL(10, 2)) AS VALUES (1, 10.00), (1, 20.00), (2, 30.00), (2, 40.00), (2, 50.00);
+
+query IR
+SELECT g, percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t GROUP BY g ORDER BY g
+----
+1 15
+2 40
+
+statement ok
+DROP TABLE t;
+
+# Decimal128: DISTINCT
+statement ok
+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (10.00), (10.00), (10.00), (10.00), (20.00);
+
+query R
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t
+----
+10
+
+query RT
+SELECT percentile_cont(DISTINCT c, 0.5), arrow_typeof(percentile_cont(DISTINCT c, 0.5)) FROM t
+----
+15 Decimal128(10, 2)
+
+statement ok
+DROP TABLE t;
+
+# Other decimals
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))
+FROM (VALUES (arrow_cast(10.00, 'Decimal32(9, 2)')), (arrow_cast(20.00, 'Decimal32(9, 2)'))) as t (v)
+----
+15 Decimal32(9, 2)
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))
+FROM (VALUES (arrow_cast(10.00, 'Decimal64(18, 2)')), (arrow_cast(20.00, 'Decimal64(18, 2)'))) as t (v)
+----
+15 Decimal64(18, 2)
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))
+FROM (VALUES (arrow_cast(10.00, 'Decimal256(54, 6)')), (arrow_cast(20.00, 'Decimal256(54, 6)'))) as t (v)
+----
+15 Decimal256(54, 6)
+
+# Other decimals: GROUP BY and DISTINCT
+
+statement ok
+CREATE TABLE t (g INT, c DECIMAL(10, 2)) AS VALUES
+ (1, 10.00), (1, 10.00), (1, 10.00), (1, 20.00), (2, 30.00), (2, 40.00);
+
+query IRT
+SELECT g,
+ percentile_cont(arrow_cast(c, 'Decimal32(9, 2)'), 0.5),
+ arrow_typeof(percentile_cont(arrow_cast(c, 'Decimal32(9, 2)'), 0.5))
+FROM t GROUP BY g ORDER BY g
+----
+1 10 Decimal32(9, 2)
+2 35 Decimal32(9, 2)
+
+query RRT
+SELECT
+ percentile_cont(arrow_cast(c, 'Decimal32(9, 2)'), 0.5),
+ percentile_cont(DISTINCT arrow_cast(c, 'Decimal32(9, 2)'), 0.5),
+ arrow_typeof(percentile_cont(DISTINCT arrow_cast(c, 'Decimal32(9, 2)'), 0.5))
+FROM t
+----
+15 25 Decimal32(9, 2)
+
+query IRT
+SELECT g,
+ percentile_cont(arrow_cast(c, 'Decimal64(18, 2)'), 0.5),
+ arrow_typeof(percentile_cont(arrow_cast(c, 'Decimal64(18, 2)'), 0.5))
+FROM t GROUP BY g ORDER BY g
+----
+1 10 Decimal64(18, 2)
+2 35 Decimal64(18, 2)
+
+query RRT
+SELECT
+ percentile_cont(arrow_cast(c, 'Decimal64(18, 2)'), 0.5),
+ percentile_cont(DISTINCT arrow_cast(c, 'Decimal64(18, 2)'), 0.5),
+ arrow_typeof(percentile_cont(DISTINCT arrow_cast(c, 'Decimal64(18, 2)'), 0.5))
+FROM t
+----
+15 25 Decimal64(18, 2)
+
+query IRT
+SELECT g,
+ percentile_cont(arrow_cast(c, 'Decimal256(54, 6)'), 0.5),
+ arrow_typeof(percentile_cont(arrow_cast(c, 'Decimal256(54, 6)'), 0.5))
+FROM t GROUP BY g ORDER BY g
+----
+1 10 Decimal256(54, 6)
+2 35 Decimal256(54, 6)
+
+query RRT
+SELECT
+ percentile_cont(arrow_cast(c, 'Decimal256(54, 6)'), 0.5),
+ percentile_cont(DISTINCT arrow_cast(c, 'Decimal256(54, 6)'), 0.5),
+ arrow_typeof(percentile_cont(DISTINCT arrow_cast(c, 'Decimal256(54, 6)'), 0.5))
+FROM t
+----
+15 25 Decimal256(54, 6)
+
+statement ok
+DROP TABLE t;
+
+statement ok
+CREATE TABLE t (c DECIMAL(50, 2)) AS VALUES (10.00), (20.00);
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t
+----
+15 Decimal256(50, 2)
+
+statement ok
+DROP TABLE t;
+
+# Decimal128 overflow cases
+statement ok
+CREATE TABLE t (c DECIMAL(38, 0)) AS VALUES
+ (arrow_cast('-90000000000000000000000000000000000000', 'Decimal128(38,0)')),
+ (arrow_cast('90000000000000000000000000000000000000', 'Decimal128(38,0)'));
+
+query RT
+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.25) WITHIN GROUP (ORDER BY c)) FROM t
+----
+-45000000000000000000000000000000000000 Decimal128(38, 0)
+
+statement ok
+DROP TABLE t;
+
+statement ok
+CREATE TABLE t (c DECIMAL(38, 0)) AS VALUES
+ (arrow_cast('-99999999999999999999999999999999999999', 'Decimal128(38,0)')),
+ (arrow_cast('99999999999999999999999999999999999999', 'Decimal128(38,0)'));
+
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t
+----
+0 Decimal128(38, 0)
+
+statement ok
+DROP TABLE t;
+
+# Decimal32 overflow cases
+query RT
+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))
+FROM (VALUES (arrow_cast(-999999999, 'Decimal32(9, 0)')), (arrow_cast(999999999, 'Decimal32(9, 0)'))) as t (v)
+----
+0 Decimal32(9, 0)
+
+# A fraction lower than `1 / den` returns zero numerator, so the interpolation
+# floors to the lower value
+query RT
+SELECT percentile_cont(0.0000001) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.0000001) WITHIN GROUP (ORDER BY v))
+FROM (VALUES (arrow_cast(10.00, 'Decimal64(18, 2)')), (arrow_cast(20.00, 'Decimal64(18, 2)'))) as t (v)
+----
+10 Decimal64(18, 2)
+
+
+
# variance_single_value
query RRRR
select var(sq.column1), var_pop(sq.column1), stddev(sq.column1), stddev_pop(sq.column1) from (values (1.0)) as sq;
@@ -8875,10 +9181,11 @@ NULL NULL NULL NULL
statement ok
drop table distinct_avg;
-query R
-select percentile_cont(null, 0.5);
+# Null evaluates to null
+query ?T
+select percentile_cont(null, 0.5), arrow_typeof(percentile_cont(null, 0.5));
----
-NULL
+NULL Null
# Test string_agg window frame behavior (fix for issue #19612)
statement ok