Apache DataFusion SQL Query Engine
feat: change approx percentile/median UDFs to return floats (#21074)
theirix29f1acd28be8
diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rsindex 014f356cd6..2ada0411f4 100644--- a/datafusion/core/tests/dataframe/dataframe_functions.rs+++ b/datafusion/core/tests/dataframe/dataframe_functions.rs@@ -402,7 +402,7 @@ async fn test_fn_approx_median() -> Result<()> {+-----------------------+| approx_median(test.b) |+-----------------------+- | 10 |+ | 10.0 |+-----------------------+");@@ -422,7 +422,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {+---------------------------------------------------------------------------+| approx_percentile_cont(Float64(0.5)) WITHIN GROUP [test.b ASC NULLS LAST] |+---------------------------------------------------------------------------+- | 10 |+ | 10.0 |+---------------------------------------------------------------------------+");@@ -437,7 +437,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {+----------------------------------------------------------------------------+| approx_percentile_cont(Float64(0.1)) WITHIN GROUP [test.b DESC NULLS LAST] |+----------------------------------------------------------------------------+- | 100 |+ | 100.0 |+----------------------------------------------------------------------------+");@@ -457,7 +457,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {+--------------------------------------------------------------------+| approx_percentile_cont(arg_2) WITHIN GROUP [test.b ASC NULLS LAST] |+--------------------------------------------------------------------+- | 10 |+ | 10.0 |+--------------------------------------------------------------------+");@@ -477,7 +477,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {+---------------------------------------------------------------------+| approx_percentile_cont(arg_2) WITHIN GROUP [test.b DESC NULLS LAST] |+---------------------------------------------------------------------+- | 100 |+ | 100.0 |+---------------------------------------------------------------------+");@@ -494,7 +494,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {+------------------------------------------------------------------------------------+| approx_percentile_cont(Float64(0.5),Int32(2)) WITHIN GROUP [test.b ASC NULLS LAST] |+------------------------------------------------------------------------------------+- | 30 |+ | 30.25 |+------------------------------------------------------------------------------------+");@@ -510,7 +510,7 @@ async fn test_fn_approx_percentile_cont() -> Result<()> {+-------------------------------------------------------------------------------------+| approx_percentile_cont(Float64(0.1),Int32(2)) WITHIN GROUP [test.b DESC NULLS LAST] |+-------------------------------------------------------------------------------------+- | 69 |+ | 69.85 |+-------------------------------------------------------------------------------------+");diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 26de1e23b3..e083075439 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -1204,26 +1204,26 @@ async fn window_using_aggregates() -> Result<()> {| first_value | last_val | approx_distinct | approx_median | median | max | min | c2 | c3 |+-------------+----------+-----------------+---------------+--------+-----+------+----+------+| | | | | | | | 1 | -85 |- | -85 | -101 | 14 | -12 | -12 | 83 | -101 | 4 | -54 |- | -85 | -101 | 17 | -25 | -25 | 83 | -101 | 5 | -31 |- | -85 | -12 | 10 | -32 | -34 | 83 | -85 | 3 | 13 |- | -85 | -25 | 3 | -56 | -56 | -25 | -85 | 1 | -5 |- | -85 | -31 | 18 | -29 | -28 | 83 | -101 | 5 | 36 |- | -85 | -38 | 16 | -25 | -25 | 83 | -101 | 4 | 65 |- | -85 | -43 | 7 | -43 | -43 | 83 | -85 | 2 | 45 |- | -85 | -48 | 6 | -35 | -36 | 83 | -85 | 2 | -43 |- | -85 | -5 | 4 | -37 | -40 | -5 | -85 | 1 | 83 |- | -85 | -54 | 15 | -17 | -18 | 83 | -101 | 4 | -38 |- | -85 | -56 | 2 | -70 | -70 | -56 | -85 | 1 | -25 |- | -85 | -72 | 9 | -43 | -43 | 83 | -85 | 3 | -12 |- | -85 | -85 | 1 | -85 | -85 | -85 | -85 | 1 | -56 |- | -85 | 13 | 11 | -17 | -18 | 83 | -85 | 3 | 14 |- | -85 | 13 | 11 | -25 | -25 | 83 | -85 | 3 | 13 |- | -85 | 14 | 12 | -12 | -12 | 83 | -85 | 3 | 17 |- | -85 | 17 | 13 | -11 | -8 | 83 | -85 | 4 | -101 |- | -85 | 45 | 8 | -34 | -34 | 83 | -85 | 3 | -72 |- | -85 | 65 | 17 | -17 | -18 | 83 | -101 | 5 | -101 |- | -85 | 83 | 5 | -25 | -25 | 83 | -85 | 2 | -48 |+ | -85 | -101 | 14 | -12.0 | -12 | 83 | -101 | 4 | -54 |+ | -85 | -101 | 17 | -25.0 | -25 | 83 | -101 | 5 | -31 |+ | -85 | -12 | 10 | -32.75 | -34 | 83 | -85 | 3 | 13 |+ | -85 | -25 | 3 | -56.0 | -56 | -25 | -85 | 1 | -5 |+ | -85 | -31 | 18 | -29.75 | -28 | 83 | -101 | 5 | 36 |+ | -85 | -38 | 16 | -25.0 | -25 | 83 | -101 | 4 | 65 |+ | -85 | -43 | 7 | -43.0 | -43 | 83 | -85 | 2 | 45 |+ | -85 | -48 | 6 | -35.75 | -36 | 83 | -85 | 2 | -43 |+ | -85 | -5 | 4 | -37.75 | -40 | -5 | -85 | 1 | 83 |+ | -85 | -54 | 15 | -17.0 | -18 | 83 | -101 | 4 | -38 |+ | -85 | -56 | 2 | -70.5 | -70 | -56 | -85 | 1 | -25 |+ | -85 | -72 | 9 | -43.0 | -43 | 83 | -85 | 3 | -12 |+ | -85 | -85 | 1 | -85.0 | -85 | -85 | -85 | 1 | -56 |+ | -85 | 13 | 11 | -17.0 | -18 | 83 | -85 | 3 | 14 |+ | -85 | 13 | 11 | -25.0 | -25 | 83 | -85 | 3 | 13 |+ | -85 | 14 | 12 | -12.0 | -12 | 83 | -85 | 3 | 17 |+ | -85 | 17 | 13 | -11.25 | -8 | 83 | -85 | 4 | -101 |+ | -85 | 45 | 8 | -34.5 | -34 | 83 | -85 | 3 | -72 |+ | -85 | 65 | 17 | -17.0 | -18 | 83 | -101 | 5 | -101 |+ | -85 | 83 | 5 | -25.0 | -25 | 83 | -85 | 2 | -48 |+-------------+----------+-----------------+---------------+--------+-----+------+----+------+");diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rsindex 42d4de939a..3e941f00c2 100644--- a/datafusion/expr-common/src/signature.rs+++ b/datafusion/expr-common/src/signature.rs@@ -21,7 +21,6 @@ use std::fmt::Display;use std::hash::Hash;use std::sync::Arc;-use crate::type_coercion::aggregates::NUMERICS;use arrow::datatypes::{DECIMAL32_MAX_PRECISION, DECIMAL64_MAX_PRECISION, DECIMAL128_MAX_PRECISION, DataType,Decimal128Type, DecimalType, Field, IntervalUnit, TimeUnit,@@ -596,6 +595,20 @@ impl Display for ArrayFunctionArgument {}}+static NUMERICS: &[DataType] = &[+ DataType::Int8,+ DataType::Int16,+ DataType::Int32,+ DataType::Int64,+ DataType::UInt8,+ DataType::UInt16,+ DataType::UInt32,+ DataType::UInt64,+ DataType::Float16,+ DataType::Float32,+ DataType::Float64,+];+impl TypeSignature {pub fn to_string_repr(&self) -> Vec<String> {match self {diff --git a/datafusion/expr-common/src/type_coercion/aggregates.rs b/datafusion/expr-common/src/type_coercion/aggregates.rsindex df86ff582d..ada0bd26b8 100644--- a/datafusion/expr-common/src/type_coercion/aggregates.rs+++ b/datafusion/expr-common/src/type_coercion/aggregates.rs@@ -20,8 +20,7 @@ use arrow::datatypes::{DataType, FieldRef};use datafusion_common::{Result, internal_err, plan_err};-// TODO: remove usage of these (INTEGERS and NUMERICS) in favour of signatures-// see https://github.com/apache/datafusion/issues/18092+#[deprecated(since = "54.0.0", note = "Use functions signatures")]pub static INTEGERS: &[DataType] = &[DataType::Int8,DataType::Int16,@@ -33,6 +32,7 @@ pub static INTEGERS: &[DataType] = &[DataType::UInt64,];+#[deprecated(since = "54.0.0", note = "Use functions signatures")]pub static NUMERICS: &[DataType] = &[DataType::Int8,DataType::Int16,diff --git a/datafusion/expr/src/test/function_stub.rs b/datafusion/expr/src/test/function_stub.rsindex 3be03260a9..a1f29b649b 100644--- a/datafusion/expr/src/test/function_stub.rs+++ b/datafusion/expr/src/test/function_stub.rs@@ -29,13 +29,14 @@ use datafusion_common::plan_err;use datafusion_common::{Result, exec_err, not_impl_err, utils::take_function_args};use crate::Volatility::Immutable;-use crate::type_coercion::aggregates::NUMERICS;use crate::{- Accumulator, AggregateUDFImpl, Expr, GroupsAccumulator, ReversedUDAF, Signature,+ Accumulator, AggregateUDFImpl, Coercion, Expr, GroupsAccumulator, ReversedUDAF,+ Signature, TypeSignature, TypeSignatureClass,expr::AggregateFunction,function::{AccumulatorArgs, StateFieldsArgs},utils::AggregateOrderSensitivity,};+use datafusion_common::types::{NativeType, logical_float64};macro_rules! create_func {($UDAF:ty, $AGGREGATE_UDF_FN:ident) => {@@ -444,9 +445,22 @@ pub struct Avg {impl Avg {pub fn new() -> Self {+ let signature = Signature::one_of(+ vec![+ TypeSignature::Coercible(vec![Coercion::new_exact(+ TypeSignatureClass::Decimal,+ )]),+ TypeSignature::Coercible(vec![Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Integer, TypeSignatureClass::Float],+ NativeType::Float64,+ )]),+ ],+ Immutable,+ );Self {aliases: vec![String::from("mean")],- signature: Signature::uniform(1, NUMERICS.to_vec(), Immutable),+ signature,}}}diff --git a/datafusion/functions-aggregate/src/approx_median.rs b/datafusion/functions-aggregate/src/approx_median.rsindex c016ada695..162dc224f2 100644--- a/datafusion/functions-aggregate/src/approx_median.rs+++ b/datafusion/functions-aggregate/src/approx_median.rs@@ -74,16 +74,11 @@ impl ApproxMedian {pub fn new() -> Self {Self {signature: Signature::one_of(- vec![- TypeSignature::Coercible(vec![Coercion::new_exact(- TypeSignatureClass::Integer,- )]),- TypeSignature::Coercible(vec![Coercion::new_implicit(- TypeSignatureClass::Float,- vec![TypeSignatureClass::Decimal],- NativeType::Float64,- )]),- ],+ vec![TypeSignature::Coercible(vec![Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ )])],Volatility::Immutable,),}diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rsindex 5e9e5beb0c..3f1adcca12 100644--- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs+++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs@@ -23,23 +23,20 @@ use arrow::array::{Array, Float16Array};use arrow::compute::{filter, is_not_null};use arrow::datatypes::FieldRef;use arrow::{- array::{- ArrayRef, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array,- Int64Array, UInt8Array, UInt16Array, UInt32Array, UInt64Array,- },+ array::{ArrayRef, Float32Array, Float64Array},datatypes::{DataType, Field},};+use datafusion_common::types::{NativeType, logical_float64};use datafusion_common::{DataFusionError, Result, ScalarValue, downcast_value, internal_err, not_impl_err,plan_err,};use datafusion_expr::expr::{AggregateFunction, Sort};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::type_coercion::aggregates::{INTEGERS, NUMERICS};use datafusion_expr::utils::format_state_name;use datafusion_expr::{- Accumulator, AggregateUDFImpl, Documentation, Expr, Signature, TypeSignature,- Volatility,+ Accumulator, AggregateUDFImpl, Coercion, Documentation, Expr, Signature,+ TypeSignature, TypeSignatureClass, Volatility,};use datafusion_functions_aggregate_common::tdigest::{DEFAULT_MAX_SIZE, TDigest};use datafusion_macros::user_doc;@@ -132,22 +129,44 @@ impl Default for ApproxPercentileCont {impl ApproxPercentileCont {/// Create a new [`ApproxPercentileCont`] aggregate function.pub fn new() -> Self {- let mut variants = Vec::with_capacity(NUMERICS.len() * (INTEGERS.len() + 1));// Accept any numeric value paired with a float64 percentile- for num in NUMERICS {- variants.push(TypeSignature::Exact(vec![num.clone(), DataType::Float64]));- // Additionally accept an integer number of centroids for T-Digest- for int in INTEGERS {- variants.push(TypeSignature::Exact(vec![- num.clone(),- DataType::Float64,- int.clone(),- ]))- }- }- Self {- signature: Signature::one_of(variants, Volatility::Immutable),- }+ let signature = Signature::one_of(+ vec![+ // 2 args - numeric, percentile (float)+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ ]),+ // 3 args - numeric, percentile (float), number of centroid for T-Digest (integer)+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Integer,+ vec![TypeSignatureClass::Numeric],+ NativeType::Int64,+ ),+ ]),+ ],+ Volatility::Immutable,+ );+ Self { signature }}pub(crate) fn create_accumulator(@@ -177,17 +196,7 @@ impl ApproxPercentileCont {let data_type = args.expr_fields[0].data_type();let accumulator: ApproxPercentileAccumulator = match data_type {- DataType::UInt8- | DataType::UInt16- | DataType::UInt32- | DataType::UInt64- | DataType::Int8- | DataType::Int16- | DataType::Int32- | DataType::Int64- | DataType::Float16- | DataType::Float32- | DataType::Float64 => {+ DataType::Float16 | DataType::Float32 | DataType::Float64 => {if let Some(max_size) = tdigest_max_size {ApproxPercentileAccumulator::new_with_max_size(percentile,@@ -374,38 +383,6 @@ impl ApproxPercentileAccumulator {.map(|v| v.to_f64()).collect::<Vec<_>>())}- DataType::Int64 => {- let array = downcast_value!(values, Int64Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::Int32 => {- let array = downcast_value!(values, Int32Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::Int16 => {- let array = downcast_value!(values, Int16Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::Int8 => {- let array = downcast_value!(values, Int8Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::UInt64 => {- let array = downcast_value!(values, UInt64Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::UInt32 => {- let array = downcast_value!(values, UInt32Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::UInt16 => {- let array = downcast_value!(values, UInt16Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }- DataType::UInt8 => {- let array = downcast_value!(values, UInt8Array);- Ok(array.values().iter().map(|v| *v as f64).collect::<Vec<_>>())- }e => internal_err!("APPROX_PERCENTILE_CONT is not expected to receive the type {e:?}"),@@ -439,14 +416,6 @@ impl Accumulator for ApproxPercentileAccumulator {// These acceptable return types MUST match the validation in// ApproxPercentile::create_accumulator.Ok(match &self.return_type {- DataType::Int8 => ScalarValue::Int8(Some(q as i8)),- DataType::Int16 => ScalarValue::Int16(Some(q as i16)),- DataType::Int32 => ScalarValue::Int32(Some(q as i32)),- DataType::Int64 => ScalarValue::Int64(Some(q as i64)),- DataType::UInt8 => ScalarValue::UInt8(Some(q as u8)),- DataType::UInt16 => ScalarValue::UInt16(Some(q as u16)),- DataType::UInt32 => ScalarValue::UInt32(Some(q as u32)),- DataType::UInt64 => ScalarValue::UInt64(Some(q as u64)),DataType::Float16 => ScalarValue::Float16(Some(half::f16::from_f64(q))),DataType::Float32 => ScalarValue::Float32(Some(q as f32)),DataType::Float64 => ScalarValue::Float64(Some(q)),diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs b/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rsindex 81a4787dd5..6ada47fb38 100644--- a/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs+++ b/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs@@ -24,13 +24,13 @@ use arrow::compute::{and, filter, is_not_null};use arrow::datatypes::FieldRef;use arrow::{array::ArrayRef, datatypes::DataType};use datafusion_common::ScalarValue;+use datafusion_common::types::{NativeType, logical_float64};use datafusion_common::{Result, not_impl_err, plan_err};-use datafusion_expr::Volatility::Immutable;use datafusion_expr::expr::{AggregateFunction, Sort};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};-use datafusion_expr::type_coercion::aggregates::{INTEGERS, NUMERICS};use datafusion_expr::{- Accumulator, AggregateUDFImpl, Documentation, Expr, Signature, TypeSignature,+ Accumulator, AggregateUDFImpl, Coercion, Documentation, Expr, Signature,+ TypeSignature, TypeSignatureClass, Volatility,};use datafusion_functions_aggregate_common::tdigest::{Centroid, TDigest};use datafusion_macros::user_doc;@@ -125,26 +125,54 @@ impl Default for ApproxPercentileContWithWeight {impl ApproxPercentileContWithWeight {/// Create a new [`ApproxPercentileContWithWeight`] aggregate function.pub fn new() -> Self {- let mut variants = Vec::with_capacity(NUMERICS.len() * (INTEGERS.len() + 1));- // Accept any numeric value paired with weight and float64 percentile- for num in NUMERICS {- variants.push(TypeSignature::Exact(vec![- num.clone(),- num.clone(),- DataType::Float64,- ]));- // Additionally accept an integer number of centroids for T-Digest- for int in INTEGERS {- variants.push(TypeSignature::Exact(vec![- num.clone(),- num.clone(),- DataType::Float64,- int.clone(),- ]));- }- }+ let signature = Signature::one_of(+ vec![+ // 3 args - numeric, weight (float), percentile (float)+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ ]),+ // 4 args - numeric, weight (float), percentile (float), centroid (integer)+ TypeSignature::Coercible(vec![+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Float,+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Native(logical_float64()),+ vec![TypeSignatureClass::Numeric],+ NativeType::Float64,+ ),+ Coercion::new_implicit(+ TypeSignatureClass::Integer,+ vec![TypeSignatureClass::Numeric],+ NativeType::Int64,+ ),+ ]),+ ],+ Volatility::Immutable,+ );Self {- signature: Signature::one_of(variants, Immutable),+ signature,approx_percentile_cont: ApproxPercentileCont::new(),}}diff --git a/datafusion/functions-aggregate/src/utils.rs b/datafusion/functions-aggregate/src/utils.rsindex 5e1925fcdb..6d816e54bd 100644--- a/datafusion/functions-aggregate/src/utils.rs+++ b/datafusion/functions-aggregate/src/utils.rs@@ -54,6 +54,11 @@ pub(crate) fn validate_percentile_expr(let percentile = match scalar_value {ScalarValue::Float32(Some(value)) => value as f64,ScalarValue::Float64(Some(value)) => value,+ ScalarValue::Float32(None) | ScalarValue::Float64(None) => {+ return plan_err!(+ "Percentile value for '{fn_name}' must be Float32 or Float64 literal (got null)"+ );+ }sv => {return plan_err!("Percentile value for '{fn_name}' must be Float32 or Float64 literal (got data type {})",diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex 253428288f..bb17d13f5c 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -1912,7 +1912,10 @@ mod test {.err().unwrap().strip_backtrace();- assert!(err.starts_with("Error during planning: Failed to coerce arguments to satisfy a call to 'avg' function: coercion from Utf8 to the signature Uniform(1, [Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64]) failed"));+ assert!(+ err.contains("Function 'avg' failed to match any signature"),+ "Err: {err:?}"+ );Ok(())}diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.sltindex 962d1d5103..00ca0482a3 100644--- a/datafusion/sqllogictest/test_files/aggregate.slt+++ b/datafusion/sqllogictest/test_files/aggregate.slt@@ -174,28 +174,22 @@ statement error DataFusion error: Schema error: Schema contains duplicate unqualSELECT approx_distinct(c9) count_c9, approx_distinct(cast(c9 as varchar)) count_c9_str FROM aggregate_test_100# csv_query_approx_percentile_cont_with_weight-statement error Failed to coerce arguments to satisfy a call to 'approx_percentile_cont_with_weight' function+statement error Function 'approx_percentile_cont_with_weight' failed to match any signatureSELECT approx_percentile_cont_with_weight(c2, 0.95) WITHIN GROUP (ORDER BY c1) FROM aggregate_test_100-statement error Failed to coerce arguments to satisfy a call to 'approx_percentile_cont_with_weight' function+statement error Function 'approx_percentile_cont_with_weight' failed to match any signatureSELECT approx_percentile_cont_with_weight(c1, 0.95) WITHIN GROUP (ORDER BY c3) FROM aggregate_test_100-statement error Failed to coerce arguments to satisfy a call to 'approx_percentile_cont_with_weight' function+statement error Function 'approx_percentile_cont_with_weight' failed to match any signatureSELECT approx_percentile_cont_with_weight(c2, c1) WITHIN GROUP (ORDER BY c3) FROM aggregate_test_100# csv_query_approx_percentile_cont_with_histogram_binsstatement error DataFusion error: Error during planning: Tdigest max_size value for 'APPROX_PERCENTILE_CONT' must be UInt > 0 literal \(got data type Int64\)\.SELECT c1, approx_percentile_cont(0.95, -1000) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-statement error Failed to coerce arguments to satisfy a call to 'approx_percentile_cont' function+statement error Function 'approx_percentile_cont' failed to match any signatureSELECT approx_percentile_cont(0.95, c1) WITHIN GROUP (ORDER BY c3) FROM aggregate_test_100-statement error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'approx_percentile_cont' function: coercion from Int16, Float64, Float64 to the signature OneOf(.*) failed(.|\n)*-SELECT approx_percentile_cont(0.95, 111.1) WITHIN GROUP (ORDER BY c3) FROM aggregate_test_100--statement error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'approx_percentile_cont' function: coercion from Float64, Float64, Float64 to the signature OneOf(.*) failed(.|\n)*-SELECT approx_percentile_cont(0.95, 111.1) WITHIN GROUP (ORDER BY c12) FROM aggregate_test_100-statement error DataFusion error: Error during planning: Percentile value for 'APPROX_PERCENTILE_CONT' must be a literalSELECT approx_percentile_cont(c12) WITHIN GROUP (ORDER BY c12) FROM aggregate_test_100@@ -948,16 +942,16 @@ SELECT c2, var_samp(CASE WHEN c12 > 0.90 THEN c12 ELSE null END) FROM aggregate_# csv_query_approx_median_1-query I+query RSELECT approx_median(c2) FROM aggregate_test_100----3# csv_query_approx_median_2-query I+query RSELECT approx_median(c6) FROM aggregate_test_100-----1146409980542786560+1146409980542786600# csv_query_approx_median_3query R@@ -1006,7 +1000,7 @@ SELECT median(col_i8), median(distinct col_i8) FROM median_table-14 100# approx_distinct_median_i8-query I+query RSELECT approx_median(distinct col_i8) FROM median_table----100@@ -2032,26 +2026,29 @@ SELECT (ABS(1 - CAST(approx_percentile_cont(0.9) WITHIN GROUP (ORDER BY c11) AStrue# percentile_cont_with_nulls-query I+query RSELECT APPROX_PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY v) FROM (VALUES (1), (2), (3), (NULL), (NULL), (NULL)) as t (v);----2# percentile_cont_with_nulls_only-query I+query RSELECT APPROX_PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY v) FROM (VALUES (CAST(NULL as INT))) as t (v);----NULL+query error DataFusion error: Error during planning: Percentile value for 'APPROX_PERCENTILE_CONT' must be Float32 or Float64 literal \(got null\)+SELECT APPROX_PERCENTILE_CONT(NULL) WITHIN GROUP (ORDER BY v) FROM (VALUES (CAST(NULL as INT))) as t (v);+# percentile_cont_with_weight_with_nulls-query I+query RSELECT APPROX_PERCENTILE_CONT_WITH_WEIGHT(w, 0.5) WITHIN GROUP (ORDER BY v)FROM (VALUES (1, 1), (2, 1), (3, 1), (4, NULL), (NULL, 1), (NULL, NULL)) as t (v, w);----2# percentile_cont_with_weight_nulls_only-query I+query RSELECT APPROX_PERCENTILE_CONT_WITH_WEIGHT(1, 0.5) WITHIN GROUP (ORDER BY v) FROM (VALUES (CAST(NULL as INT))) as t (v);----NULL@@ -2332,36 +2329,36 @@ b 5 NULL 20135.4b NULL NULL 7732.315789473684# csv_query_approx_percentile_cont_with_weight-query TI+query TRSELECT c1, approx_percentile_cont(0.95) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 73+a 73.55b 68-c 122-d 124-e 115+c 122.5+d 124.2+e 115.6# csv_query_approx_percentile_cont_with_weight (should be the same as above)-query TI+query TRSELECT c1, approx_percentile_cont(c3, 0.95) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 73+a 73.55b 68-c 122-d 124-e 115+c 122.5+d 124.2+e 115.6# using approx_percentile_cont on 2 columns with same signature-query TII+query TRRSELECT c1, approx_percentile_cont(c2, 0.95) AS c2, approx_percentile_cont(c3, 0.95) AS c3 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 5 73+a 5 73.55b 5 68-c 5 122-d 5 124-e 5 115+c 5 122.5+d 5 124.2+e 5 115.6# error is unique to this UDAFquery TRR@@ -2375,73 +2372,82 @@ e 3 40.333333333333-query TI+query TRSELECT c1, approx_percentile_cont(0.95) WITHIN GROUP (ORDER BY c3 DESC) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1----a -101-b -114-c -109-d -98-e -93+b -114.3+c -109.475+d -98.6+e -93.65# csv_query_approx_percentile_cont_with_weight (2)-query TI+query TRSELECT c1, approx_percentile_cont_with_weight(1, 0.95) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 73+a 73.55b 68-c 122-d 124-e 115+c 122.5+d 124.2+e 115.6# csv_query_approx_percentile_cont_with_weight alternate syntax-query TI+query TRSELECT c1, approx_percentile_cont_with_weight(c3, 1, 0.95) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 73+a 73.55b 68-c 122-d 124-e 115+c 122.5+d 124.2+e 115.6--query TI+query TRSELECT c1, approx_percentile_cont_with_weight(1, 0.95) WITHIN GROUP (ORDER BY c3 DESC) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1----a -101-b -114-c -109-d -98-e -93+b -114.3+c -109.475+d -98.6+e -93.65# csv_query_approx_percentile_cont_with_histogram_bins-query TI+query TRSELECT c1, approx_percentile_cont(0.95, 200) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 73+a 73.55b 68-c 122-d 124-e 115+c 122.5+d 124.2+e 115.6-query TI+query TR+SELECT c1, approx_percentile_cont(0.95, 200.1) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1+----+a 73.55+b 68+c 122.5+d 124.2+e 115.6+++query TRSELECT c1, approx_percentile_cont_with_weight(c2, 0.95) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1----a 65b 68c 122-d 123-e 110+d 123.15+e 110.266666666667# approx_percentile_cont_with_weight with centroids-query TI+query TRSELECT c1, approx_percentile_cont_with_weight(c2, 0.95, 200) WITHIN GROUP (ORDER BY c3) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1----a 65b 68c 122-d 123-e 110+d 123.15+e 110.266666666667# csv_query_sum_crossjoinquery TTI@@ -3834,10 +3840,10 @@ SELECT COUNT(DISTINCT c1) FROM test# TODO: aggregate_with_alias# test_approx_percentile_cont_decimal_support-query TI+query TRSELECT c1, approx_percentile_cont(cast(0.85 as decimal(10,2))) WITHIN GROUP (ORDER BY c2) apc FROM aggregate_test_100 GROUP BY 1 ORDER BY 1-----a 4+a 4.175b 5c 4d 4@@ -5978,7 +5984,7 @@ select median(a) from (select 1 as a where 1=0);----NULL-query I+query Rselect approx_median(a) from (select 1 as a where 1=0);----NULLdiff --git a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt b/datafusion/sqllogictest/test_files/aggregate_skip_partial.sltindex e12ac5782e..cd95426a9b 100644--- a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt+++ b/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt@@ -145,7 +145,7 @@ GROUP BY 1, 2 ORDER BY 1 LIMIT 5;-2117946883 d -2117946883 NULL NULL NULL-2098805236 c -2098805236 NULL NULL NULL-query ITIIII+query ITRRRRSELECT c5, c1,APPROX_MEDIAN(c5),APPROX_MEDIAN(CASE WHEN c1 = 'a' THEN c5 ELSE NULL END),@@ -318,14 +318,14 @@ SELECT c2, median(c5), median(c11) FROM aggregate_test_100 GROUP BY c2 ORDER BY5 604973998 0.49842384# Test approx_median for int / float-query IIR+query IRRSELECT c2, approx_median(c5), approx_median(c11) FROM aggregate_test_100 GROUP BY c2 ORDER BY c2;-----1 191655437 0.59926736+1 191655437.25 0.599267362 -587831330 0.432304863 240273900 0.401996974 762932956 0.48515016-5 593204320 0.5156586+5 593204320.5 0.5156586# Test approx_distinct for varchar / intquery III@@ -399,14 +399,14 @@ SELECT c2, median(c3), median(c11) FROM aggregate_test_100_null GROUP BY c2 ORDE5 -35 0.5536642# Test approx_median with nullable fields-query IIR+query IRRSELECT c2, approx_median(c3), approx_median(c11) FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;----1 12 0.60679442 1 0.460764883 14 0.40154034-4 -7 0.48515016-5 -39 0.5536642+4 -7.75 0.48515016+5 -39.75 0.5536642# Test approx_distinct with nullable fieldsquery II@@ -516,7 +516,7 @@ FROM aggregate_test_100 GROUP BY c2 ORDER BY c2;5 64 -59# Test approx_median with filter-query III+query IRRSELECTc2,approx_median(c3) FILTER (WHERE c3 > 0),@@ -526,7 +526,7 @@ FROM aggregate_test_100 GROUP BY c2 ORDER BY c2;1 57 -562 52 -603 71 -76-4 65 -64+4 65 -64.755 64 -59# Test count with nullable fields and filter@@ -662,20 +662,20 @@ FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;5 -22# Test approx_median with nullable fields and filter-query IIR+query IRRSELECT c2,approx_median(c3) FILTER (WHERE c5 > 0),approx_median(c11) FILTER (WHERE c5 < 0)FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;----1 -5 0.6623719-2 12 0.52930677+2 12.25 0.529306773 13 0.327925384 -38 0.49774808-5 -21 0.47652745+5 -21.75 0.47652745# Test approx_median with nullable fields and nullable filter-query II+query IRSELECT c2,approx_median(c3) FILTER (WHERE c11 > 0.5)FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;diff --git a/datafusion/sqllogictest/test_files/clickbench_extended.slt b/datafusion/sqllogictest/test_files/clickbench_extended.sltindex ee3e33551e..6b0d78cdba 100644--- a/datafusion/sqllogictest/test_files/clickbench_extended.slt+++ b/datafusion/sqllogictest/test_files/clickbench_extended.slt@@ -52,7 +52,7 @@ query IIIIIISELECT "ClientIP", "WatchID", COUNT(*) c, MIN("ResponseStartTiming") tmin, MEDIAN("ResponseStartTiming") tmed, MAX("ResponseStartTiming") tmax FROM hits WHERE "JavaEnable" = 0 GROUP BY "ClientIP", "WatchID" HAVING c > 1 ORDER BY tmed DESC LIMIT 10;-----query IIIIII+query IIIIRISELECT "ClientIP", "WatchID", COUNT(*) c, MIN("ResponseStartTiming") tmin, APPROX_PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY "ResponseStartTiming") tp95, MAX("ResponseStartTiming") tmax FROM 'hits' WHERE "JavaEnable" = 0 GROUP BY "ClientIP", "WatchID" HAVING c > 1 ORDER BY tp95 DESC LIMIT 10;----diff --git a/datafusion/sqllogictest/test_files/metadata.slt b/datafusion/sqllogictest/test_files/metadata.sltindex 42589481f9..f584e3c342 100644--- a/datafusion/sqllogictest/test_files/metadata.slt+++ b/datafusion/sqllogictest/test_files/metadata.slt@@ -83,7 +83,7 @@ select count(distinct name) from table_with_metadata;2# Regression test: prevent field metadata loss per https://github.com/apache/datafusion/issues/12687-query I+query Rselect approx_median(distinct id) from table_with_metadata;----2diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdindex d5fe197854..c277f69d0b 100644--- a/docs/source/library-user-guide/upgrading/54.0.0.md+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -321,3 +321,29 @@ clusters (e.g., ZWJ emoji sequences). For ASCII and most common Unicode text,behavior is unchanged.[#17861]: https://github.com/apache/datafusion/pull/17861++### `approx_percentile_cont`, `approx_percentile_cont_with_weight`, `approx_median` now coerce to floats++The type signatures of `approx_percentile_cont`, `approx_percentile_cont_with_weight`, and+`approx_median` now coerce integer input values to `Float64` before computing the approximation.+As a result, these functions always return a float, even when the input column is an integer type.++**Who is affected:**++- Queries or downstream code that relied on `approx_percentile_cont` / `approx_percentile_cont_with_weight` /+ `approx_median` returning an integer type when given an integer column.++**Migration guide:**++If downstream code checks or relies on the return type being an integer, add an explicit+`CAST` back to the desired integer type, or update the type assertion:++```sql+-- Before (returned Int64):+SELECT approx_percentile_cont(quantity, 0.5) FROM orders;++-- After (returns Float64); cast if an integer result is required:+SELECT CAST(approx_percentile_cont(quantity, 0.5) AS BIGINT) FROM orders;+```++[#21074]: https://github.com/apache/datafusion/pull/21074