Apache DataFusion SQL Query Engine
feat: decimal support for percentile_cont and median (#24419)
theirix46957762e3a2
diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rsindex 2a2adfdc98..d2db50f374 100644--- a/datafusion/functions-aggregate/src/median.rs+++ b/datafusion/functions-aggregate/src/median.rs@@ -15,48 +15,24 @@// specific language governing permissions and limitations// under the License.-use std::cmp::Ordering;-use std::fmt::{Debug, Formatter};-use std::mem::{size_of, size_of_val};-use std::sync::Arc;+use std::fmt::Debug;-use arrow::array::{- ArrowNumericType, BooleanArray, ListArray, PrimitiveArray, PrimitiveBuilder,- downcast_integer,-};-use arrow::buffer::{OffsetBuffer, ScalarBuffer};-use arrow::{- array::{ArrayRef, AsArray},- datatypes::{- DataType, Decimal128Type, Decimal256Type, Field, Float16Type, Float32Type,- Float64Type,- },-};+use arrow::datatypes::DataType;-use arrow::array::Array;-use arrow::array::ArrowNativeTypeOp;-use arrow::datatypes::{- ArrowNativeType, ArrowPrimitiveType, Decimal32Type, Decimal64Type, FieldRef,-};+use arrow::datatypes::FieldRef;-use datafusion_common::hash_utils::RandomState;-use datafusion_common::types::{NativeType, logical_float64};-use datafusion_common::{- DataFusionError, Result, ScalarValue, assert_eq_or_internal_err, exec_datafusion_err,- internal_datafusion_err, internal_err,+use crate::percentile_cont::{+ PercentileCont, create_percentile_accumulator, create_percentile_groups_accumulator,};+use datafusion_common::Result;+use datafusion_common::types::logical_float64;+use datafusion_expr::GroupsAccumulator;use datafusion_expr::function::StateFieldsArgs;use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, Documentation, Signature, TypeSignature,- TypeSignatureClass, Volatility, function::AccumulatorArgs, utils::format_state_name,+ TypeSignatureClass, Volatility, function::AccumulatorArgs,};-use datafusion_expr::{EmitTo, GroupsAccumulator};-use datafusion_functions_aggregate_common::aggregate::groups_accumulator::accumulate::accumulate;-use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::filtered_null_mask;-use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;-use datafusion_functions_aggregate_common::utils::{GenericDistinctBuffer, Hashable};use datafusion_macros::user_doc;-use std::collections::HashMap;make_udaf_expr_and_func!(Median,@@ -91,6 +67,7 @@ make_udaf_expr_and_func!(#[derive(PartialEq, Eq, Hash, Debug)]pub struct Median {signature: Signature,+ percentile_cont: PercentileCont,}impl Default for Median {@@ -113,14 +90,14 @@ impl Median {TypeSignature::Coercible(vec![Coercion::new_exact(TypeSignatureClass::Float,)]),- TypeSignature::Coercible(vec![Coercion::new_implicit(- TypeSignatureClass::Native(logical_float64()),+ TypeSignature::Coercible(vec![Coercion::new_implicit_native(+ logical_float64(),vec![TypeSignatureClass::Integer],- NativeType::Float64,)]),],Volatility::Immutable,),+ percentile_cont: PercentileCont::new(),}}}@@ -135,589 +112,38 @@ impl AggregateUDFImpl for Median {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- Ok(arg_types[0].clone())+ self.percentile_cont.return_type(arg_types)}fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {- if args.input_fields[0].data_type().is_null() {- return Ok(vec![- Field::new(- format_state_name(args.name, self.name()),- DataType::Null,- true,- )- .into(),- ]);- }-- //Intermediate state is a list of the elements we have collected so far- let field = Field::new_list_field(args.input_fields[0].data_type().clone(), true);- let state_name = if args.is_distinct {- "distinct_median"- } else {- "median"- };-- Ok(vec![- Field::new(- format_state_name(args.name, state_name),- DataType::List(Arc::new(field)),- true,- )- .into(),- ])+ self.percentile_cont.state_fields(args)}- fn accumulator(&self, acc_args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {- macro_rules! helper {- ($t:ty, $dt:expr) => {- if acc_args.is_distinct {- Ok(Box::new(DistinctMedianAccumulator::<$t> {- data_type: $dt.clone(),- distinct_values: GenericDistinctBuffer::new($dt),- }))- } else {- Ok(Box::new(MedianAccumulator::<$t> {- data_type: $dt.clone(),- all_values: vec![],- }))- }- };- }-- let dt = acc_args.expr_fields[0].data_type().clone();- if dt.is_null() {- return Ok(Box::new(NoopAccumulator::default()));- }-- downcast_integer! {- dt => (helper, dt),- DataType::Float16 => helper!(Float16Type, dt),- DataType::Float32 => helper!(Float32Type, dt),- DataType::Float64 => helper!(Float64Type, dt),- DataType::Decimal32(_, _) => helper!(Decimal32Type, dt),- DataType::Decimal64(_, _) => helper!(Decimal64Type, dt),- DataType::Decimal128(_, _) => helper!(Decimal128Type, dt),- DataType::Decimal256(_, _) => helper!(Decimal256Type, dt),- _ => Err(DataFusionError::NotImplemented(format!(- "MedianAccumulator not supported for {} with {}",- acc_args.name,- dt,- ))),- }+ fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ create_percentile_accumulator(+ self.name(),+ 0.5,+ args.expr_fields[0].data_type(),+ args.is_distinct,+ )}fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {- !args.is_distinct && !args.expr_fields[0].data_type().is_null()+ self.percentile_cont.groups_accumulator_supported(args)}fn create_groups_accumulator(&self,args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {- let num_args = args.exprs.len();- assert_eq_or_internal_err!(- num_args,- 1,- "median should only have 1 arg, but found num args:{}",- num_args- );-- let dt = args.expr_fields[0].data_type().clone();-- macro_rules! helper {- ($t:ty, $dt:expr) => {- Ok(Box::new(MedianGroupsAccumulator::<$t>::new($dt)))- };- }-- downcast_integer! {- dt => (helper, dt),- DataType::Float16 => helper!(Float16Type, dt),- DataType::Float32 => helper!(Float32Type, dt),- DataType::Float64 => helper!(Float64Type, dt),- DataType::Decimal32(_, _) => helper!(Decimal32Type, dt),- DataType::Decimal64(_, _) => helper!(Decimal64Type, dt),- DataType::Decimal128(_, _) => helper!(Decimal128Type, dt),- DataType::Decimal256(_, _) => helper!(Decimal256Type, dt),- _ => Err(DataFusionError::NotImplemented(format!(- "MedianGroupsAccumulator not supported for {} with {}",- args.name,- dt,- ))),- }+ create_percentile_groups_accumulator(+ self.name(),+ 0.5,+ args.expr_fields[0].data_type(),+ )}fn documentation(&self) -> Option<&Documentation> {self.doc()}}--/// The median accumulator accumulates the raw input values-/// as `ScalarValue`s-///-/// The intermediate state is represented as a List of scalar values updated by-/// `merge_batch` and a `Vec` of `ArrayRef` that are converted to scalar values-/// in the final evaluation step so that we avoid expensive conversions and-/// allocations during `update_batch`.-struct MedianAccumulator<T: ArrowNumericType> {- data_type: DataType,- all_values: Vec<T::Native>,-}--impl<T: ArrowNumericType> Debug for MedianAccumulator<T> {- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {- write!(f, "MedianAccumulator({})", self.data_type)- }-}--impl<T: ArrowNumericType> Accumulator for MedianAccumulator<T> {- fn state(&mut self) -> Result<Vec<ScalarValue>> {- // Convert `all_values` to `ListArray` and return a single List ScalarValue-- // Build offsets- let offsets =- OffsetBuffer::new(ScalarBuffer::from(vec![0, self.all_values.len() as i32]));-- // Build inner array- let values_array = PrimitiveArray::<T>::new(- ScalarBuffer::from(std::mem::take(&mut self.all_values)),- None,- )- .with_data_type(self.data_type.clone());-- // Build the result list array- let list_array = ListArray::new(- Arc::new(Field::new_list_field(self.data_type.clone(), true)),- offsets,- Arc::new(values_array),- None,- );-- Ok(vec![ScalarValue::List(Arc::new(list_array))])- }-- fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- let values = values[0].as_primitive::<T>();- let additional = values.len() - values.null_count();- self.all_values.try_reserve(additional).map_err(|e| {- exec_datafusion_err!(- "failed to reserve {additional} values for median accumulator: {e}"- )- })?;- if values.null_count() > 0 {- self.all_values.extend(values.iter().flatten());- } else {- // Fast path: no nulls, so the values buffer can be appended wholesale.- self.all_values.extend_from_slice(values.values());- }- Ok(())- }-- fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {- let array = states[0].as_list::<i32>();- for v in array.iter().flatten() {- self.update_batch(&[v])?- }- Ok(())- }-- fn evaluate(&mut self) -> Result<ScalarValue> {- let median = calculate_median::<T>(&mut self.all_values);- ScalarValue::new_primitive::<T>(median, &self.data_type)- }-- fn size(&self) -> usize {- size_of_val(self) + self.all_values.capacity() * size_of::<T::Native>()- }-- fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- let mut to_remove: HashMap<Hashable<T::Native>, usize, RandomState> =- HashMap::default();-- let arr = values[0].as_primitive::<T>();- if arr.null_count() > 0 {- for value in arr.iter().flatten() {- *to_remove.entry(Hashable(value)).or_default() += 1;- }- } else {- // Fast path: no nulls, so skip the per-element validity check.- for value in arr.values().iter() {- *to_remove.entry(Hashable(*value)).or_default() += 1;- }- }-- let mut i = 0;- while i < self.all_values.len() {- let k = Hashable(self.all_values[i]);- if let Some(count) = to_remove.get_mut(&k)- && *count > 0- {- self.all_values.swap_remove(i);- *count -= 1;- if *count == 0 {- to_remove.remove(&k);- if to_remove.is_empty() {- break;- }- }- } else {- i += 1;- }- }-- // Retracting values that are not tracked means the accumulator state- // has diverged from the window frame; continuing would silently- // produce wrong results, so surface it as an error.- if !to_remove.is_empty() {- return internal_err!(- "median retract_batch: retracted value(s) not present in the window"- );- }- Ok(())- }-- fn supports_retract_batch(&self) -> bool {- true- }-}--/// The median groups accumulator accumulates the raw input values-///-/// For calculating the accurate medians of groups, we need to store all values-/// of groups before final evaluation.-/// So values in each group will be stored in a `Vec<T>`, and the total group values-/// will be actually organized as a `Vec<Vec<T>>`.-#[derive(Debug)]-struct MedianGroupsAccumulator<T: ArrowNumericType + Send> {- data_type: DataType,- group_values: Vec<Vec<T::Native>>,-}--impl<T: ArrowNumericType + Send> MedianGroupsAccumulator<T> {- pub fn new(data_type: DataType) -> Self {- Self {- data_type,- group_values: Vec::new(),- }- }-}--impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<T> {- fn update_batch(- &mut self,- values: &[ArrayRef],- group_indices: &[usize],- opt_filter: Option<&BooleanArray>,- total_num_groups: usize,- ) -> Result<()> {- assert_eq!(values.len(), 1, "single argument to update_batch");- let values = values[0].as_primitive::<T>();-- // Push the `not nulls + not filtered` row into its group- self.group_values.resize(total_num_groups, Vec::new());- accumulate(- group_indices,- values,- opt_filter,- |group_index, new_value| {- self.group_values[group_index].push(new_value);- },- );-- Ok(())- }-- fn merge_batch(- &mut self,- values: &[ArrayRef],- group_indices: &[usize],- total_num_groups: usize,- ) -> Result<()> {- assert_eq!(values.len(), 1, "one argument to merge_batch");-- // The merged values should be organized like as a `ListArray` which is nullable- // (input with nulls usually generated from `convert_to_state`), but `inner array` of- // `ListArray` is `non-nullable`.- //- // Following is the possible and impossible input `values`:- //- // # Possible values- // ```text- // group 0: [1, 2, 3]- // group 1: null (list array is nullable)- // group 2: [6, 7, 8]- // ...- // group n: [...]- // ```- //- // # Impossible values- // ```text- // group x: [1, 2, null] (values in list array is non-nullable)- // ```- //- let input_group_values = values[0].as_list::<i32>();-- // Ensure group values big enough- self.group_values.resize(total_num_groups, Vec::new());-- // Extend values to related groups- // TODO: avoid using iterator of the `ListArray`, this will lead to- // many calls of `slice` of its ``inner array`, and `slice` is not- // so efficient(due to the calculation of `null_count` for each `slice`).- group_indices- .iter()- .zip(input_group_values.iter())- .for_each(|(&group_index, values_opt)| {- if let Some(values) = values_opt {- let values = values.as_primitive::<T>();- self.group_values[group_index].extend(values.values().iter());- }- });-- Ok(())- }-- fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {- // Emit values- let emit_group_values = emit_to.take_needed(&mut self.group_values);-- // Build offsets- let mut offsets = Vec::with_capacity(self.group_values.len() + 1);- offsets.push(0);- let mut cur_len = 0_i32;- for group_value in &emit_group_values {- cur_len += group_value.len() as i32;- offsets.push(cur_len);- }- // TODO: maybe we can use `OffsetBuffer::new_unchecked` like what in `convert_to_state`,- // but safety should be considered more carefully here(and I am not sure if it can get- // performance improvement when we introduce checks to keep the safety...).- //- // Can see more details in:- // https://github.com/apache/datafusion/pull/13681#discussion_r1931209791- //- let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets));-- // Build inner array- let flatten_group_values =- emit_group_values.into_iter().flatten().collect::<Vec<_>>();- let group_values_array =- PrimitiveArray::<T>::new(ScalarBuffer::from(flatten_group_values), None)- .with_data_type(self.data_type.clone());-- // Build the result list array- let result_list_array = ListArray::new(- Arc::new(Field::new_list_field(self.data_type.clone(), true)),- offsets,- Arc::new(group_values_array),- None,- );-- Ok(vec![Arc::new(result_list_array)])- }-- fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {- // Emit values- let emit_group_values = emit_to.take_needed(&mut self.group_values);-- // Calculate median for each group- let mut evaluate_result_builder =- PrimitiveBuilder::<T>::new().with_data_type(self.data_type.clone());- for mut values in emit_group_values {- let median = calculate_median::<T>(&mut values);- evaluate_result_builder.append_option(median);- }-- Ok(Arc::new(evaluate_result_builder.finish()))- }-- fn convert_to_state(- &self,- values: &[ArrayRef],- opt_filter: Option<&BooleanArray>,- ) -> Result<Vec<ArrayRef>> {- assert_eq!(values.len(), 1, "one argument to merge_batch");-- let input_array = values[0].as_primitive::<T>();-- // Directly convert the input array to states, each row will be- // seen as a respective group.- // For detail, the `input_array` will be converted to a `ListArray`.- // And if row is `not null + not filtered`, it will be converted to a list- // with only one element; otherwise, this row in `ListArray` will be set- // to null.-- // Reuse values buffer in `input_array` to build `values` in `ListArray`- let values = PrimitiveArray::<T>::new(input_array.values().clone(), None)- .with_data_type(self.data_type.clone());-- // `offsets` in `ListArray`, each row as a list element- let offset_end = i32::try_from(input_array.len()).map_err(|e| {- internal_datafusion_err!(- "cast array_len to i32 failed in convert_to_state of group median, err:{e:?}"- )- })?;- let offsets = (0..=offset_end).collect::<Vec<_>>();- // Safety: all checks in `OffsetBuffer::new` are ensured to pass- let offsets = unsafe { OffsetBuffer::new_unchecked(ScalarBuffer::from(offsets)) };-- // `nulls` for converted `ListArray`- let nulls = filtered_null_mask(opt_filter, input_array);-- let converted_list_array = ListArray::new(- Arc::new(Field::new_list_field(self.data_type.clone(), true)),- offsets,- Arc::new(values),- nulls,- );-- Ok(vec![Arc::new(converted_list_array)])- }- fn size(&self) -> usize {- self.group_values- .iter()- .map(|values| values.capacity() * size_of::<T::Native>())- .sum::<usize>()- // account for size of self.group_values too- + self.group_values.capacity() * size_of::<Vec<T::Native>>()- + size_of::<Vec<Vec<T::Native>>>()- }-}--#[derive(Debug)]-struct DistinctMedianAccumulator<T: ArrowNumericType> {- distinct_values: GenericDistinctBuffer<T>,- data_type: DataType,-}--impl<T: ArrowNumericType + Debug> Accumulator for DistinctMedianAccumulator<T> {- fn state(&mut self) -> Result<Vec<ScalarValue>> {- self.distinct_values.state()- }-- fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {- self.distinct_values.update_batch(values)- }-- fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {- self.distinct_values.merge_batch(states)- }-- fn evaluate(&mut self) -> Result<ScalarValue> {- let mut d: Vec<T::Native> =- self.distinct_values.values.iter().map(|v| v.0).collect();- let median = calculate_median::<T>(&mut d);- ScalarValue::new_primitive::<T>(median, &self.data_type)- }-- fn size(&self) -> usize {- size_of_val(self) + self.distinct_values.size()- }-}--/// Get maximum entry in the slice,-fn slice_max<T>(array: &[T::Native]) -> T::Native-where- T: ArrowPrimitiveType,- T::Native: PartialOrd, // Ensure the type supports PartialOrd for comparison-{- // Make sure that, array is not empty.- debug_assert!(!array.is_empty());- // `.unwrap()` is safe here as the array is supposed to be non-empty- *array- .iter()- .max_by(|x, y| x.partial_cmp(y).unwrap_or(Ordering::Less))- .unwrap()-}--fn calculate_median<T: ArrowNumericType>(values: &mut [T::Native]) -> Option<T::Native> {- let cmp = |x: &T::Native, y: &T::Native| x.compare(*y);-- let len = values.len();- if len == 0 {- None- } else if len % 2 == 0 {- let (low, high, _) = values.select_nth_unstable_by(len / 2, cmp);- // Get the maximum of the low (left side after bi-partitioning)- let left_max = slice_max::<T>(low);- // Calculate median as the average of the two middle values.- // Use checked arithmetic to detect overflow and fall back to safe formula.- let two = T::Native::usize_as(2);- let median = match left_max.add_checked(*high) {- Ok(sum) => sum.div_wrapping(two),- Err(_) => {- // Overflow detected - use safe midpoint formula:- // a/2 + b/2 + ((a%2 + b%2) / 2)- // This avoids overflow by dividing before adding.- let half_left = left_max.div_wrapping(two);- let half_right = (*high).div_wrapping(two);- let rem_left = left_max.mod_wrapping(two);- let rem_right = (*high).mod_wrapping(two);- // The sum of remainders (0, 1, or 2 for unsigned; -2 to 2 for signed)- // divided by 2 gives the correction factor (0 or 1 for unsigned; -1, 0, or 1 for signed)- let correction = rem_left.add_wrapping(rem_right).div_wrapping(two);- half_left.add_wrapping(half_right).add_wrapping(correction)- }- };- Some(median)- } else {- let (_, median, _) = values.select_nth_unstable_by(len / 2, cmp);- Some(*median)- }-}--#[cfg(test)]-mod tests {- use super::*;- use arrow::array::Float64Array;-- fn median_accumulator() -> MedianAccumulator<Float64Type> {- MedianAccumulator {- data_type: DataType::Float64,- all_values: vec![],- }- }-- #[test]- fn retract_batch_errors_on_untracked_value() {- let mut acc = median_accumulator();- let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));- acc.update_batch(std::slice::from_ref(&values)).unwrap();-- let retract: ArrayRef = Arc::new(Float64Array::from(vec![3.0]));- let err = acc- .retract_batch(std::slice::from_ref(&retract))- .unwrap_err()- .to_string();- assert!(- err.contains("not present in the window"),- "unexpected error: {err}"- );- }-- #[test]- fn update_batch_with_and_without_nulls_agree() {- // The null-free fast path must accumulate the same values as the- // general path.- let dense: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));- let sparse: ArrayRef = Arc::new(Float64Array::from(vec![- Some(1.0),- None,- Some(2.0),- None,- Some(3.0),- ]));-- let mut dense_acc = median_accumulator();- dense_acc- .update_batch(std::slice::from_ref(&dense))- .unwrap();- let mut sparse_acc = median_accumulator();- sparse_acc- .update_batch(std::slice::from_ref(&sparse))- .unwrap();-- assert_eq!(dense_acc.all_values, sparse_acc.all_values);- }-}diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rsindex c026a3d3a5..2499e18f27 100644--- a/datafusion/functions-aggregate/src/percentile_cont.rs+++ b/datafusion/functions-aggregate/src/percentile_cont.rs@@ -17,11 +17,13 @@use std::collections::HashMap;use std::fmt::Debug;+use std::marker::PhantomData;use std::mem::{size_of, size_of_val};use std::sync::Arc;use arrow::array::{- ArrowNumericType, BooleanArray, ListArray, PrimitiveArray, PrimitiveBuilder,+ ArrowNumericType, ArrowPrimitiveType, BooleanArray, ListArray, PrimitiveArray,+ PrimitiveBuilder,};use arrow::buffer::{OffsetBuffer, ScalarBuffer};use arrow::{@@ -32,6 +34,10 @@ use arrow::{use num_traits::AsPrimitive;use arrow::array::ArrowNativeTypeOp;+use arrow::datatypes::{+ ArrowNativeType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type,+ DecimalType,+};use datafusion_common::hash_utils::RandomState;use datafusion_common::internal_err;use datafusion_common::types::{NativeType, logical_float64};@@ -39,6 +45,7 @@ use datafusion_common::utils::memory::estimate_memory_size;use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;use crate::min_max::{max_udaf, min_udaf};+use crate::utils::validate_percentile_expr;use datafusion_common::{Result, ScalarValue, exec_datafusion_err, internal_datafusion_err,utils::{SingleRowListArrayBuilder, take_function_args},@@ -46,7 +53,7 @@ use datafusion_common::{use datafusion_expr::utils::format_state_name;use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, Documentation, Expr, Signature,- TypeSignatureClass, Volatility,+ TypeSignature, TypeSignatureClass, Volatility,};use datafusion_expr::{EmitTo, GroupsAccumulator};use datafusion_expr::{@@ -59,8 +66,6 @@ use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls:use datafusion_functions_aggregate_common::utils::Hashable;use datafusion_macros::user_doc;-use crate::utils::validate_percentile_expr;-/// Precision multiplier for linear interpolation calculations.////// This value of 1,000,000 was chosen to balance precision with overflow safety:@@ -72,9 +77,9 @@ use crate::utils::validate_percentile_expr;/// is computed as: `lower + ((upper - lower) * (fraction * PRECISION)) / PRECISION`/// to avoid floating-point operations on integer types while maintaining precision.///-/// The interpolation arithmetic is performed in f64 and then cast back to the+/// The interpolation arithmetic for floats is performed in f64 and then cast back to the/// native type to avoid overflowing Float16 intermediates.-const INTERPOLATION_PRECISION: f64 = 1_000_000.0;+const INTERPOLATION_PRECISION: usize = 1_000_000;create_func!(PercentileCont, percentile_cont_udaf);@@ -143,18 +148,30 @@ impl Default for PercentileCont {impl PercentileCont {pub fn new() -> Self {Self {- signature: Signature::coercible(+ signature: Signature::one_of(vec![- Coercion::new_implicit(- TypeSignatureClass::Float,- vec![TypeSignatureClass::Numeric],- NativeType::Float64,- ),- Coercion::new_implicit(- TypeSignatureClass::Native(logical_float64()),- vec![TypeSignatureClass::Numeric],- NativeType::Float64,- ),+ // Decimal signature: decimals, percentile+ TypeSignature::Coercible(vec![+ // value+ Coercion::new_exact(TypeSignatureClass::Decimal),+ // percentile+ Coercion::new_implicit_native(+ logical_float64(),+ vec![TypeSignatureClass::Numeric],+ ),+ ]),+ // Float signature: float, percentile+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit_native(+ logical_float64(),+ vec![TypeSignatureClass::Numeric],+ ),+ ]),],Volatility::Immutable,)@@ -179,10 +196,7 @@ impl AggregateUDFImpl for PercentileCont {}fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {- match &arg_types[0] {- DataType::Null => Ok(DataType::Float64),- dt => Ok(dt.clone()),- }+ Ok(arg_types[0].clone())}fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {@@ -216,40 +230,14 @@ impl AggregateUDFImpl for PercentileCont {}fn accumulator(&self, args: AccumulatorArgs) -> Result<Box<dyn Accumulator>> {+ // Always verify percentileslet percentile = get_percentile(&args)?;-- let input_dt = args.expr_fields[0].data_type();- if input_dt.is_null() {- return Ok(Box::new(NoopAccumulator::new(ScalarValue::Float64(None))));- }-- if args.is_distinct {- match input_dt {- DataType::Float16 => Ok(Box::new(DistinctPercentileContAccumulator::<- Float16Type,- >::new(percentile))),- DataType::Float32 => Ok(Box::new(DistinctPercentileContAccumulator::<- Float32Type,- >::new(percentile))),- DataType::Float64 => Ok(Box::new(DistinctPercentileContAccumulator::<- Float64Type,- >::new(percentile))),- dt => internal_err!("Unsupported datatype for percentile cont: {dt}"),- }- } else {- match input_dt {- DataType::Float16 => Ok(Box::new(- PercentileContAccumulator::<Float16Type>::new(percentile),- )),- DataType::Float32 => Ok(Box::new(- PercentileContAccumulator::<Float32Type>::new(percentile),- )),- DataType::Float64 => Ok(Box::new(- PercentileContAccumulator::<Float64Type>::new(percentile),- )),- dt => internal_err!("Unsupported datatype for percentile cont: {dt}"),- }- }+ create_percentile_accumulator(+ self.name(),+ percentile,+ args.expr_fields[0].data_type(),+ args.is_distinct,+ )}fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {@@ -260,21 +248,13 @@ impl AggregateUDFImpl for PercentileCont {&self,args: AccumulatorArgs,) -> Result<Box<dyn GroupsAccumulator>> {+ // Always verify percentileslet percentile = get_percentile(&args)?;-- let input_dt = args.expr_fields[0].data_type();- match input_dt {- DataType::Float16 => Ok(Box::new(PercentileContGroupsAccumulator::<- Float16Type,- >::new(percentile))),- DataType::Float32 => Ok(Box::new(PercentileContGroupsAccumulator::<- Float32Type,- >::new(percentile))),- DataType::Float64 => Ok(Box::new(PercentileContGroupsAccumulator::<- Float64Type,- >::new(percentile))),- dt => internal_err!("Unsupported datatype for percentile cont: {dt}"),- }+ create_percentile_groups_accumulator(+ self.name(),+ percentile,+ args.expr_fields[0].data_type(),+ )}fn simplify(&self) -> Option<AggregateFunctionSimplification> {@@ -310,6 +290,85 @@ fn get_percentile(args: &AccumulatorArgs) -> Result<f64> {Ok(percentile)}+pub fn create_percentile_accumulator(+ name: &str,+ percentile: f64,+ input_dt: &DataType,+ is_distinct: bool,+) -> Result<Box<dyn Accumulator>> {+ // Null input evaluates to null+ if input_dt.is_null() {+ return Ok(Box::new(NoopAccumulator::default()));+ }++ macro_rules! helper {+ ($t:ty, $i:ty, $dt:expr) => {+ if is_distinct {+ Ok(Box::new(DistinctPercentileContAccumulator::<$t, $i>::new(+ percentile,+ $dt.clone(),+ )))+ } else {+ Ok(Box::new(PercentileContAccumulator::<$t, $i>::new(+ percentile,+ $dt.clone(),+ )))+ }+ };+ }+ match input_dt {+ DataType::Float16 => helper!(Float16Type, FloatInterpolator, input_dt),+ DataType::Float32 => helper!(Float32Type, FloatInterpolator, input_dt),+ DataType::Float64 => helper!(Float64Type, FloatInterpolator, input_dt),+ DataType::Decimal32(_, _) => {+ helper!(Decimal32Type, DecimalInterpolator, input_dt)+ }+ DataType::Decimal64(_, _) => {+ helper!(Decimal64Type, DecimalInterpolator, input_dt)+ }+ DataType::Decimal128(_, _) => {+ helper!(Decimal128Type, DecimalInterpolator, input_dt)+ }+ DataType::Decimal256(_, _) => {+ helper!(Decimal256Type, DecimalInterpolator, input_dt)+ }+ dt => internal_err!("Unsupported datatype for {} with {}", name, dt),+ }+}++pub fn create_percentile_groups_accumulator(+ name: &str,+ percentile: f64,+ input_dt: &DataType,+) -> Result<Box<dyn GroupsAccumulator>> {+ macro_rules! helper {+ ($t:ty, $i:ty, $dt:expr) => {+ Ok(Box::new(PercentileContGroupsAccumulator::<$t, $i>::new(+ percentile,+ $dt.clone(),+ )))+ };+ }+ match input_dt {+ DataType::Float16 => helper!(Float16Type, FloatInterpolator, input_dt),+ DataType::Float32 => helper!(Float32Type, FloatInterpolator, input_dt),+ DataType::Float64 => helper!(Float64Type, FloatInterpolator, input_dt),+ DataType::Decimal32(_, _) => {+ helper!(Decimal32Type, DecimalInterpolator, input_dt)+ }+ DataType::Decimal64(_, _) => {+ helper!(Decimal64Type, DecimalInterpolator, input_dt)+ }+ DataType::Decimal128(_, _) => {+ helper!(Decimal128Type, DecimalInterpolator, input_dt)+ }+ DataType::Decimal256(_, _) => {+ helper!(Decimal256Type, DecimalInterpolator, input_dt)+ }+ dt => internal_err!("Unsupported datatype for {} with {}", name, dt),+ }+}+fn simplify_percentile_cont_aggregate(aggregate_function: AggregateFunction,info: &SimplifyContext,@@ -377,25 +436,33 @@ fn simplify_percentile_cont_aggregate(/// in the final evaluation step so that we avoid expensive conversions and/// allocations during `update_batch`.#[derive(Debug)]-struct PercentileContAccumulator<T: ArrowNumericType + Debug> {+struct PercentileContAccumulator<+ T: ArrowNumericType + Debug,+ I: PercentileInterpolator<T>,+> {all_values: Vec<T::Native>,percentile: f64,+ data_type: DataType,+ _interpolator: PhantomData<I>,}-impl<T: ArrowNumericType + Debug> PercentileContAccumulator<T> {- fn new(percentile: f64) -> Self {+impl<T: ArrowNumericType + Debug, I: PercentileInterpolator<T>>+ PercentileContAccumulator<T, I>+{+ fn new(percentile: f64, data_type: DataType) -> Self {Self {all_values: vec![],percentile,+ data_type,+ _interpolator: PhantomData,}}}-impl<T> Accumulator for PercentileContAccumulator<T>+impl<T, I> Accumulator for PercentileContAccumulator<T, I>whereT: ArrowNumericType + Debug,- T::Native: Copy + AsPrimitive<f64>,- f64: AsPrimitive<T::Native>,+ I: PercentileInterpolator<T> + 'static,{fn state(&mut self) -> Result<Vec<ScalarValue>> {// Convert `all_values` to `ListArray` and return a single List ScalarValue@@ -408,11 +475,12 @@ wherelet values_array = PrimitiveArray::<T>::new(ScalarBuffer::from(std::mem::take(&mut self.all_values)),None,- );+ )+ .with_data_type(self.data_type.clone());// Build the result list arraylet list_array = ListArray::new(- Arc::new(Field::new_list_field(T::DATA_TYPE, true)),+ Arc::new(Field::new_list_field(self.data_type.clone(), true)),offsets,Arc::new(values_array),None,@@ -440,13 +508,16 @@ wherefn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> {let array = states[0].as_list::<i32>();- self.update_batch(&[array.value(0)])?;+ // Feed all list elements from a batch+ for values in array.iter().flatten() {+ self.update_batch(&[values])?;+ }Ok(())}fn evaluate(&mut self) -> Result<ScalarValue> {- let value = calculate_percentile::<T>(&mut self.all_values, self.percentile);- ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)+ let value = calculate_percentile::<T, I>(&mut self.all_values, self.percentile)?;+ ScalarValue::new_primitive::<T>(value, &self.data_type)}fn size(&self) -> usize {@@ -511,25 +582,33 @@ where/// So values in each group will be stored in a `Vec<T>`, and the total group values/// will be actually organized as a `Vec<Vec<T>>`.#[derive(Debug)]-struct PercentileContGroupsAccumulator<T: ArrowNumericType + Send> {+struct PercentileContGroupsAccumulator<+ T: ArrowNumericType + Debug,+ I: PercentileInterpolator<T>,+> {group_values: Vec<Vec<T::Native>>,percentile: f64,+ data_type: DataType,+ _interpolator: PhantomData<I>,}-impl<T: ArrowNumericType + Send> PercentileContGroupsAccumulator<T> {- fn new(percentile: f64) -> Self {+impl<T: ArrowNumericType + Debug, I: PercentileInterpolator<T>>+ PercentileContGroupsAccumulator<T, I>+{+ fn new(percentile: f64, data_type: DataType) -> Self {Self {group_values: vec![],percentile,+ data_type,+ _interpolator: PhantomData,}}}-impl<T> GroupsAccumulator for PercentileContGroupsAccumulator<T>+impl<T, I> GroupsAccumulator for PercentileContGroupsAccumulator<T, I>where- T: ArrowNumericType + Send,- T::Native: Copy + AsPrimitive<f64>,- f64: AsPrimitive<T::Native>,+ T: ArrowNumericType + Debug + Send,+ I: PercentileInterpolator<T> + 'static,{fn update_batch(&mut self,@@ -602,11 +681,12 @@ wherelet flatten_group_values =emit_group_values.into_iter().flatten().collect::<Vec<_>>();let group_values_array =- PrimitiveArray::<T>::new(ScalarBuffer::from(flatten_group_values), None);+ PrimitiveArray::<T>::new(ScalarBuffer::from(flatten_group_values), None)+ .with_data_type(self.data_type.clone());// Build the result list arraylet result_list_array = ListArray::new(- Arc::new(Field::new_list_field(T::DATA_TYPE, true)),+ Arc::new(Field::new_list_field(self.data_type.clone(), true)),offsets,Arc::new(group_values_array),None,@@ -621,9 +701,11 @@ where// Calculate percentile for each grouplet mut evaluate_result_builder =- PrimitiveBuilder::<T>::with_capacity(emit_group_values.len());+ PrimitiveBuilder::<T>::with_capacity(emit_group_values.len())+ .with_data_type(self.data_type.clone());for values in &mut emit_group_values {- let value = calculate_percentile::<T>(values.as_mut_slice(), self.percentile);+ let value =+ calculate_percentile::<T, I>(values.as_mut_slice(), self.percentile)?;evaluate_result_builder.append_option(value);}@@ -647,7 +729,8 @@ where// to null.// Reuse values buffer in `input_array` to build `values` in `ListArray`- let values = PrimitiveArray::<T>::new(input_array.values().clone(), None);+ let values = PrimitiveArray::<T>::new(input_array.values().clone(), None)+ .with_data_type(self.data_type.clone());// `offsets` in `ListArray`, each row as a list elementlet offset_end = i32::try_from(input_array.len()).map_err(|e| {@@ -668,7 +751,7 @@ wherelet nulls = filtered_null_mask(opt_filter, input_array);let converted_list_array = ListArray::new(- Arc::new(Field::new_list_field(T::DATA_TYPE, true)),+ Arc::new(Field::new_list_field(self.data_type.clone(), true)),offsets,Arc::new(values),nulls,@@ -694,7 +777,10 @@ where/// left the window frame. The percentile is then computed over the set of keys/// with a positive count.#[derive(Debug)]-struct DistinctPercentileContAccumulator<T: ArrowNumericType> {+struct DistinctPercentileContAccumulator<+ T: ArrowNumericType,+ I: PercentileInterpolator<T>,+> {/// Distinct value -> number of in-window rows carrying it.////// Uses the same fast (foldhash) `RandomState` as the shared@@ -702,22 +788,27 @@ struct DistinctPercentileContAccumulator<T: ArrowNumericType> {/// SipHash, which is considerably slower for this hot path.counts: HashMap<Hashable<T::Native>, usize, RandomState>,percentile: f64,+ data_type: DataType,+ _interpolator: PhantomData<I>,}-impl<T: ArrowNumericType + Debug> DistinctPercentileContAccumulator<T> {- fn new(percentile: f64) -> Self {+impl<T: ArrowNumericType, I: PercentileInterpolator<T>>+ DistinctPercentileContAccumulator<T, I>+{+ fn new(percentile: f64, data_type: DataType) -> Self {Self {counts: HashMap::default(),percentile,+ data_type,+ _interpolator: PhantomData,}}}-impl<T> Accumulator for DistinctPercentileContAccumulator<T>+impl<T, I> Accumulator for DistinctPercentileContAccumulator<T, I>whereT: ArrowNumericType + Debug,- T::Native: Copy + AsPrimitive<f64>,- f64: AsPrimitive<T::Native>,+ I: PercentileInterpolator<T> + 'static,{fn state(&mut self) -> Result<Vec<ScalarValue>> {// Emit the distinct keys as a single List scalar, matching the state@@ -726,7 +817,7 @@ where// cross-partition merges only need the distinct key set.let arr = Arc::new(PrimitiveArray::<T>::from_iter_values(self.counts.keys().map(|v| v.0))- .with_data_type(T::DATA_TYPE),+ .with_data_type(self.data_type.clone()),);Ok(vec![SingleRowListArrayBuilder::new(arr).build_list_scalar(),@@ -763,8 +854,8 @@ wherefn evaluate(&mut self) -> Result<ScalarValue> {let mut values: Vec<T::Native> = self.counts.keys().map(|v| v.0).collect();- let value = calculate_percentile::<T>(&mut values, self.percentile);- ScalarValue::new_primitive::<T>(value, &T::DATA_TYPE)+ let value = calculate_percentile::<T, I>(&mut values, self.percentile)?;+ ScalarValue::new_primitive::<T>(value, &self.data_type)}fn size(&self) -> usize {@@ -816,6 +907,126 @@ where}}+/// A trait to abstract interpolation logic for percentile calculation+/// for floats and decimals.+trait PercentileInterpolator<T: ArrowNumericType>: Debug + Sync + Send {+ fn interpolate(+ lower: T::Native,+ upper: T::Native,+ fraction: f64,+ ) -> Result<T::Native>;+}++#[derive(Debug)]+struct FloatInterpolator;++/// The interpolation arithmetic for floats is performed in f64 and then cast back to the+/// native type to avoid overflowing Float16 intermediates.+impl<T> PercentileInterpolator<T> for FloatInterpolator+where+ T: ArrowNumericType,+ T::Native: AsPrimitive<f64>,+ f64: AsPrimitive<T::Native>,+{+ fn interpolate(+ lower: T::Native,+ upper: T::Native,+ fraction: f64,+ ) -> Result<T::Native> {+ // Linear interpolation.+ // We compute a quantized interpolation weight using `FLOAT_INTERPOLATION_PRECISION` because:+ // 1. Both values come from the input data, so (upper - lower) is bounded by the value range+ // 2. fraction is between 0 and 1; quantizing it provides stable, predictable results+ // 3. The result is guaranteed to be between lower_value and upper_value (modulo cast rounding)+ // 4. Arithmetic is performed in f64 and cast back to avoid overflowing Float16 intermediates+ let scaled = (fraction * (INTERPOLATION_PRECISION as f64)) as usize;+ let weight = scaled as f64 / (INTERPOLATION_PRECISION as f64);++ let lower_f: f64 = lower.as_();+ let upper_f: f64 = upper.as_();+ let interpolated_f = lower_f + (upper_f - lower_f) * weight;+ Ok(interpolated_f.as_())+ }+}++#[derive(Debug)]+struct DecimalInterpolator;++/// Precision multiplier for decimal linear interpolation calculations.+fn deduce_interpolation_precision<T: DecimalType>() -> usize {+ if T::BYTE_LENGTH == 4 {+ // Avoid overflow in `scale_by_num` with 32-bit Decimal32+ 10_000+ } else {+ INTERPOLATION_PRECISION+ }+}++/// Compute a scaled value for interpolation using a formula `trunc(x * num / precision)`+/// The numerical method is separating `q * num + r * num / den`+/// where `q = x / precision` and `r = x % precision`+fn scale_by_num<T>(x: T::Native, num: T::Native, den: T::Native) -> Result<T::Native>+where+ T: DecimalType,+{+ debug_assert!(num >= T::Native::ZERO);+ debug_assert!(num <= den);++ let q = x.div_wrapping(den);+ let r = x.mod_wrapping(den);+ // `q * num` cannot exceed `x`; `r * num` cannot exceed `(den - 1)^2`++ let a = q.mul_checked(num)?;+ let b = r.mul_checked(num)?.div_wrapping(den);+ a.add_checked(b)+ .map_err(|e| exec_datafusion_err!("Arithmetic overflow in percentile_cont: {e}"))+}++impl<T> PercentileInterpolator<T> for DecimalInterpolator+where+ T: DecimalType,+{+ fn interpolate(+ lower: T::Native,+ upper: T::Native,+ fraction: f64,+ ) -> Result<T::Native> {+ debug_assert!((0.0..=1.0).contains(&fraction));+ debug_assert!(lower <= upper);++ let interpolation_precision = deduce_interpolation_precision::<T>();++ let num =+ T::Native::usize_as((fraction * interpolation_precision as f64) as usize);+ let den = T::Native::usize_as(interpolation_precision);++ // Happy path: `upper - lower` does not overflow+ // (could be a case for Decimal128 with max precision)+ if let Ok(delta) = upper.sub_checked(lower) {+ // Calculate the interpolation weight with the formula, where den is the precision:+ // `lower + (upper - lower) * num / den`+ let scaled: T::Native = scale_by_num::<T>(delta, num, den)?;+ lower.add_checked(scaled).map_err(|e| {+ exec_datafusion_err!("Arithmetic overflow in percentile_cont: {e}")+ })+ } else {+ // Avoid overflow with the subtraction - split to two additive parts+ // `a = lower * (precision-num) / precision`+ // `b = upper * num / precision`+ // The weights sum to 1, so the result is bounded by max(|lower|, |upper|)+ // and never overflows, at the cost of a second truncation (2 ULP not 1).+ let num_a = den.sub_wrapping(num);+ let a: T::Native = scale_by_num::<T>(lower, num_a, den)?;++ let b: T::Native = scale_by_num::<T>(upper, num, den)?;++ a.add_checked(b).map_err(|e| {+ exec_datafusion_err!("Arithmetic overflow in percentile_cont: {e}")+ })+ }+ }+}+/// Calculate the percentile value for a given set of values./// This function performs an exact calculation by sorting all values.///@@ -827,37 +1038,33 @@ where/// Note: This function takes a mutable slice and sorts it in place, but does not/// consume the data. This is important for window frame queries where evaluate()/// may be called multiple times on the same accumulator state.-fn calculate_percentile<T: ArrowNumericType>(+fn calculate_percentile<T: ArrowPrimitiveType, I: PercentileInterpolator<T>>(values: &mut [T::Native],percentile: f64,-) -> Option<T::Native>-where- T::Native: Copy + AsPrimitive<f64>,- f64: AsPrimitive<T::Native>,-{+) -> Result<Option<T::Native>> {let cmp = |x: &T::Native, y: &T::Native| x.compare(*y);let len = values.len();if len == 0 {- None+ Ok(None)} else if len == 1 {- Some(values[0])+ Ok(Some(values[0]))} else if percentile == 0.0 {// Get minimum value- Some(+ Ok(Some(*values.iter().min_by(|a, b| cmp(a, b)).expect("we checked for len > 0 a few lines above"),- )+ ))} else if percentile == 1.0 {// Get maximum value- Some(+ Ok(Some(*values.iter().max_by(|a, b| cmp(a, b)).expect("we checked for len > 0 a few lines above"),- )+ ))} else {// Calculate the index using the formula: p * (n - 1)let index = percentile * ((len - 1) as f64);@@ -867,7 +1074,7 @@ whereif lower_index == upper_index {// Exact index, return the value at that positionlet (_, value, _) = values.select_nth_unstable_by(lower_index, cmp);- Some(*value)+ Ok(Some(*value))} else {// Need to interpolate between two values// First, partition at lower_index to get the lower value@@ -878,20 +1085,11 @@ wherelet (_, upper_value, _) = values.select_nth_unstable_by(upper_index, cmp);let upper_value = *upper_value;- // Linear interpolation.- // We compute a quantized interpolation weight using `INTERPOLATION_PRECISION` because:- // 1. Both values come from the input data, so (upper - lower) is bounded by the value range- // 2. fraction is between 0 and 1; quantizing it provides stable, predictable results- // 3. The result is guaranteed to be between lower_value and upper_value (modulo cast rounding)- // 4. Arithmetic is performed in f64 and cast back to avoid overflowing Float16 intermediateslet fraction = index - (lower_index as f64);- let scaled = (fraction * INTERPOLATION_PRECISION) as usize;- let weight = scaled as f64 / INTERPOLATION_PRECISION;- let lower_f: f64 = lower_value.as_();- let upper_f: f64 = upper_value.as_();- let interpolated_f = lower_f + (upper_f - lower_f) * weight;- Some(interpolated_f.as_())+ let interpolated = I::interpolate(lower_value, upper_value, fraction)?;++ Ok(Some(interpolated))}}}@@ -900,11 +1098,15 @@ wheremod tests {use super::*;use arrow::array::Float64Array;+ use arrow::datatypes::{Decimal64Type, Float16Type, Float64Type};use half::f16;#[test]fn retract_batch_errors_on_untracked_value() {- let mut acc = PercentileContAccumulator::<Float64Type>::new(0.5);+ let mut acc = PercentileContAccumulator::<Float64Type, FloatInterpolator>::new(+ 0.5,+ DataType::Float64,+ );let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));acc.update_batch(std::slice::from_ref(&values)).unwrap();@@ -932,11 +1134,19 @@ mod tests {Some(3.0),]));- let mut dense_acc = PercentileContAccumulator::<Float64Type>::new(0.5);+ let mut dense_acc =+ PercentileContAccumulator::<Float64Type, FloatInterpolator>::new(+ 0.5,+ DataType::Float64,+ );dense_acc.update_batch(std::slice::from_ref(&dense)).unwrap();- let mut sparse_acc = PercentileContAccumulator::<Float64Type>::new(0.5);+ let mut sparse_acc =+ PercentileContAccumulator::<Float64Type, FloatInterpolator>::new(+ 0.5,+ DataType::Float64,+ );sparse_acc.update_batch(std::slice::from_ref(&sparse)).unwrap();@@ -950,8 +1160,10 @@ mod tests {// Interpolating between 0 and the max finite f16 value previously overflowed// intermediate f16 computations and produced NaN.let mut values = vec![f16::from_f32(0.0), f16::from_f32(65504.0)];- let result = calculate_percentile::<Float16Type>(&mut values, 0.5)- .expect("non-empty input");+ let result =+ calculate_percentile::<Float16Type, FloatInterpolator>(&mut values, 0.5)+ .expect("non-empty input")+ .expect("non-empty result");let result_f = result.to_f32();assert!(!result_f.is_nan(),@@ -963,4 +1175,59 @@ mod tests {"unexpected result {result_f}");}++ #[test]+ fn percentile_cont_decimal64() {+ // Test values: [100.00, 200.00, 300.00, 400.00, 500.00]+ // These are stored as i64 values scaled by 10^2+ let mut values = vec![+ 10000i64, // 100.00+ 20000i64, // 200.00+ 30000i64, // 300.00+ 40000i64, // 400.00+ 50000i64, // 500.00+ ];++ // Test 50th percentile (median)+ // Should return 300.00 (30000)+ let result =+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 0.5)+ .expect("evaluate failed")+ .expect("expected Some value");++ assert_eq!(result, 30000i64, "50th percentile should be 300.00");++ // Test 15th percentile+ // Should return 160.00 (16000)+ let result =+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 0.15)+ .expect("evaluate failed")+ .expect("expected Some value");++ assert_eq!(result, 16000i64, "15th percentile should be 160.00");++ // Test 0th percentile (minimum)+ let mut values = vec![10000i64, 20000i64, 30000i64];+ let result =+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 0.0)+ .expect("evaluate failed")+ .expect("expected Some value");++ assert_eq!(+ result, 10000i64,+ "0th percentile should be minimum value 100.00"+ );++ // Test 100th percentile (maximum)+ let mut values = vec![10000i64, 20000i64, 30000i64];+ let result =+ calculate_percentile::<Decimal64Type, DecimalInterpolator>(&mut values, 1.0)+ .expect("evaluate failed")+ .expect("expected Some value");++ assert_eq!(+ result, 30000i64,+ "100th percentile should be maximum value 300.00"+ );+ }}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex e85ef5ce58..a7f08ea83a 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -3933,9 +3933,9 @@ mod tests {// Median(a)fn test_median_agg_expr(schema: SchemaRef) -> Result<AggregateFunctionExpr> {- AggregateExprBuilder::new(median_udaf(), vec![col("a", &schema)?])+ AggregateExprBuilder::new(median_udaf(), vec![col("b", &schema)?]).schema(schema)- .alias("MEDIAN(a)")+ .alias("MEDIAN(b)").build()}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 974487dac2..537c156a40 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -1172,6 +1172,39 @@ FROM distinct_pct_nulls ORDER BY id;statement okDROP TABLE distinct_pct_nulls;+# percentile_cont for shorter floats++query RT+select percentile_cont(arrow_cast(col_f32, 'Float16'), 0.5), arrow_typeof(percentile_cont(arrow_cast(col_f32, 'Float16'), 0.5)) from median_table;+----+2.75 Float16++query RT+select percentile_cont(col_f32, 0.5), arrow_typeof(percentile_cont(col_f32, 0.5)) from median_table;+----+2.75 Float32++query RT+select percentile_cont(DISTINCT arrow_cast(col_f32, 'Float16'), 0.5), arrow_typeof(percentile_cont(DISTINCT arrow_cast(col_f32, 'Float16'), 0.5)) from median_table;+----+2.75 Float16++query RT+select percentile_cont(DISTINCT col_f32, 0.5), arrow_typeof(percentile_cont(DISTINCT col_f32, 0.5)) from median_table;+----+2.75 Float32++query RT+select median(arrow_cast(col_f32, 'Float16')), arrow_typeof(median(arrow_cast(col_f32, 'Float16'))) from median_table;+----+2.75 Float16++query RT+select median(col_f32), arrow_typeof(median(col_f32)) from median_table;+----+2.75 Float32++query RTselect approx_median(arrow_cast(col_f32, 'Float16')), arrow_typeof(approx_median(arrow_cast(col_f32, 'Float16'))) from median_table;----@@ -4519,6 +4552,279 @@ SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY v DESC) FROM (VALUES (1), (22.75+#####################+## percentile_cont tests for decimal types+#####################++# Decimal128: typical percentiles (exact index, n=5)+statement ok+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (10.10), (20.20), (30.30), (40.40), (50.50);++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t+----+30.3 Decimal128(10, 2)++query R+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c) FROM t+----+20.2++query R+SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY c) FROM t+----+40.4++query R+SELECT percentile_cont(0.0) WITHIN GROUP (ORDER BY c) FROM t+----+10.1++query R+SELECT percentile_cont(1.0) WITHIN GROUP (ORDER BY c) FROM t+----+50.5++# Alternate syntax should agree with WITHIN GROUP syntax+query B+SELECT percentile_cont(c, 0.5) = percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t+----+true++statement ok+DROP TABLE t;++# Decimal128: interpolation between two values+statement ok+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (10.00), (20.00);++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t+----+15 Decimal128(10, 2)++query R+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c) FROM t+----+12.5++query R+SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY c) FROM t+----+17.5++statement ok+DROP TABLE t;++# Decimal128: negative numbers and NULL handling+statement ok+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (-10.00), (-5.00), (0.00), (5.00), (10.00), (NULL);++query R+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t+----+0++query R+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c) FROM t+----+-5++statement ok+DROP TABLE t;++# Decimal128: all NULLs+statement ok+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (NULL);++query R+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t+----+NULL++statement ok+DROP TABLE t;++# Decimal128: single value+statement ok+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (42.00);++query R+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t+----+42++statement ok+DROP TABLE t;++# Decimal128: GROUP BY+statement ok+CREATE TABLE t (g INT, c DECIMAL(10, 2)) AS VALUES (1, 10.00), (1, 20.00), (2, 30.00), (2, 40.00), (2, 50.00);++query IR+SELECT g, percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t GROUP BY g ORDER BY g+----+1 15+2 40++statement ok+DROP TABLE t;++# Decimal128: DISTINCT+statement ok+CREATE TABLE t (c DECIMAL(10, 2)) AS VALUES (10.00), (10.00), (10.00), (10.00), (20.00);++query R+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c) FROM t+----+10++query RT+SELECT percentile_cont(DISTINCT c, 0.5), arrow_typeof(percentile_cont(DISTINCT c, 0.5)) FROM t+----+15 Decimal128(10, 2)++statement ok+DROP TABLE t;++# Other decimals++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))+FROM (VALUES (arrow_cast(10.00, 'Decimal32(9, 2)')), (arrow_cast(20.00, 'Decimal32(9, 2)'))) as t (v)+----+15 Decimal32(9, 2)++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))+FROM (VALUES (arrow_cast(10.00, 'Decimal64(18, 2)')), (arrow_cast(20.00, 'Decimal64(18, 2)'))) as t (v)+----+15 Decimal64(18, 2)++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))+FROM (VALUES (arrow_cast(10.00, 'Decimal256(54, 6)')), (arrow_cast(20.00, 'Decimal256(54, 6)'))) as t (v)+----+15 Decimal256(54, 6)++# Other decimals: GROUP BY and DISTINCT++statement ok+CREATE TABLE t (g INT, c DECIMAL(10, 2)) AS VALUES+ (1, 10.00), (1, 10.00), (1, 10.00), (1, 20.00), (2, 30.00), (2, 40.00);++query IRT+SELECT g,+ percentile_cont(arrow_cast(c, 'Decimal32(9, 2)'), 0.5),+ arrow_typeof(percentile_cont(arrow_cast(c, 'Decimal32(9, 2)'), 0.5))+FROM t GROUP BY g ORDER BY g+----+1 10 Decimal32(9, 2)+2 35 Decimal32(9, 2)++query RRT+SELECT+ percentile_cont(arrow_cast(c, 'Decimal32(9, 2)'), 0.5),+ percentile_cont(DISTINCT arrow_cast(c, 'Decimal32(9, 2)'), 0.5),+ arrow_typeof(percentile_cont(DISTINCT arrow_cast(c, 'Decimal32(9, 2)'), 0.5))+FROM t+----+15 25 Decimal32(9, 2)++query IRT+SELECT g,+ percentile_cont(arrow_cast(c, 'Decimal64(18, 2)'), 0.5),+ arrow_typeof(percentile_cont(arrow_cast(c, 'Decimal64(18, 2)'), 0.5))+FROM t GROUP BY g ORDER BY g+----+1 10 Decimal64(18, 2)+2 35 Decimal64(18, 2)++query RRT+SELECT+ percentile_cont(arrow_cast(c, 'Decimal64(18, 2)'), 0.5),+ percentile_cont(DISTINCT arrow_cast(c, 'Decimal64(18, 2)'), 0.5),+ arrow_typeof(percentile_cont(DISTINCT arrow_cast(c, 'Decimal64(18, 2)'), 0.5))+FROM t+----+15 25 Decimal64(18, 2)++query IRT+SELECT g,+ percentile_cont(arrow_cast(c, 'Decimal256(54, 6)'), 0.5),+ arrow_typeof(percentile_cont(arrow_cast(c, 'Decimal256(54, 6)'), 0.5))+FROM t GROUP BY g ORDER BY g+----+1 10 Decimal256(54, 6)+2 35 Decimal256(54, 6)++query RRT+SELECT+ percentile_cont(arrow_cast(c, 'Decimal256(54, 6)'), 0.5),+ percentile_cont(DISTINCT arrow_cast(c, 'Decimal256(54, 6)'), 0.5),+ arrow_typeof(percentile_cont(DISTINCT arrow_cast(c, 'Decimal256(54, 6)'), 0.5))+FROM t+----+15 25 Decimal256(54, 6)++statement ok+DROP TABLE t;++statement ok+CREATE TABLE t (c DECIMAL(50, 2)) AS VALUES (10.00), (20.00);++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t+----+15 Decimal256(50, 2)++statement ok+DROP TABLE t;++# Decimal128 overflow cases+statement ok+CREATE TABLE t (c DECIMAL(38, 0)) AS VALUES+ (arrow_cast('-90000000000000000000000000000000000000', 'Decimal128(38,0)')),+ (arrow_cast('90000000000000000000000000000000000000', 'Decimal128(38,0)'));++query RT+SELECT percentile_cont(0.25) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.25) WITHIN GROUP (ORDER BY c)) FROM t+----+-45000000000000000000000000000000000000 Decimal128(38, 0)++statement ok+DROP TABLE t;++statement ok+CREATE TABLE t (c DECIMAL(38, 0)) AS VALUES+ (arrow_cast('-99999999999999999999999999999999999999', 'Decimal128(38,0)')),+ (arrow_cast('99999999999999999999999999999999999999', 'Decimal128(38,0)'));++query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY c), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY c)) FROM t+----+0 Decimal128(38, 0)++statement ok+DROP TABLE t;++# Decimal32 overflow cases+query RT+SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.5) WITHIN GROUP (ORDER BY v))+FROM (VALUES (arrow_cast(-999999999, 'Decimal32(9, 0)')), (arrow_cast(999999999, 'Decimal32(9, 0)'))) as t (v)+----+0 Decimal32(9, 0)++# A fraction lower than `1 / den` returns zero numerator, so the interpolation+# floors to the lower value+query RT+SELECT percentile_cont(0.0000001) WITHIN GROUP (ORDER BY v), arrow_typeof(percentile_cont(0.0000001) WITHIN GROUP (ORDER BY v))+FROM (VALUES (arrow_cast(10.00, 'Decimal64(18, 2)')), (arrow_cast(20.00, 'Decimal64(18, 2)'))) as t (v)+----+10 Decimal64(18, 2)+++# variance_single_valuequery RRRRselect var(sq.column1), var_pop(sq.column1), stddev(sq.column1), stddev_pop(sq.column1) from (values (1.0)) as sq;@@ -8875,10 +9181,11 @@ NULL NULL NULL NULLstatement okdrop table distinct_avg;-query R-select percentile_cont(null, 0.5);+# Null evaluates to null+query ?T+select percentile_cont(null, 0.5), arrow_typeof(percentile_cont(null, 0.5));-----NULL+NULL Null# Test string_agg window frame behavior (fix for issue #19612)statement ok