Apache DataFusion SQL Query Engine
fix(ffi): forward distinct_handling through FFI_AggregateUDF (#25876)
Bharadwaj Pendyalaac279ebf5cee
diff --git a/datafusion/ffi/integration-tests/tests/cases/udaf.rs b/datafusion/ffi/integration-tests/tests/cases/udaf.rsindex 71aca901d0..7805f8fcc9 100644--- a/datafusion/ffi/integration-tests/tests/cases/udaf.rs+++ b/datafusion/ffi/integration-tests/tests/cases/udaf.rs@@ -20,10 +20,10 @@ mod tests {use arrow::array::{Float64Array, record_batch};use datafusion::error::Result;- use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl};+ use datafusion::logical_expr::{AggregateUDF, AggregateUDFImpl, DistinctHandling};use datafusion::prelude::{SessionContext, col};use datafusion_catalog::MemTable;- use datafusion_expr::{ScalarUDF, ScalarUDFImpl};+ use datafusion_expr::{ExprFunctionExt, ScalarUDF, ScalarUDFImpl};use datafusion_ffi::tests::utils::get_module;#[tokio::test]@@ -79,6 +79,71 @@ mod tests {Ok(())}+ #[test]+ fn test_distinct_handling() -> Result<()> {+ let module = get_module()?;++ let ffi_stddev_func = (module.create_stddev_udaf)();+ let foreign_stddev_func: Arc<dyn AggregateUDFImpl> = (&ffi_stddev_func).into();+ assert_eq!(+ foreign_stddev_func.distinct_handling(),+ DistinctHandling::Unsupported+ );++ let ffi_sum_func = (module.create_sum_udaf)();+ let foreign_sum_func: Arc<dyn AggregateUDFImpl> = (&ffi_sum_func).into();+ assert_eq!(+ foreign_sum_func.distinct_handling(),+ DistinctHandling::Sensitive+ );++ Ok(())+ }++ #[tokio::test]+ async fn test_foreign_min_distinct_drops_distinct() -> Result<()> {+ let module = get_module()?;++ let ffi_min_func = (module.create_min_udaf)();+ let foreign_min_func: Arc<dyn AggregateUDFImpl> = (&ffi_min_func).into();++ let udaf = AggregateUDF::new_from_shared_impl(foreign_min_func);++ let ctx = SessionContext::default();+ let record_batch = record_batch!(+ ("a", Int32, vec![1, 2, 2, 4, 4, 4, 4]),+ ("b", Float64, vec![3.0, 2.0, 2.0, 4.0, 1.0, 1.0, 4.0])+ )+ .unwrap();++ let df = ctx+ .read_batch(record_batch)?+ .aggregate(+ vec![col("a")],+ vec![udaf.call(vec![col("b")]).distinct().build()?.alias("min_b")],+ )?+ .sort_by(vec![col("a")])?;++ let plan = df.clone().into_optimized_plan()?;+ assert_eq!(+ plan.display_indent().to_string(),+ "Sort: ?table?.a ASC NULLS LAST\+ \n Aggregate: groupBy=[[?table?.a]], aggr=[[min(?table?.b) AS min_b]]\+ \n TableScan: ?table? projection=[a, b]"+ );++ let result = df.collect().await?;++ let expected = record_batch!(+ ("a", Int32, vec![1, 2, 4]),+ ("min_b", Float64, vec![3.0, 2.0, 1.0])+ )?;++ assert_eq!(result[0], expected);++ Ok(())+ }+#[tokio::test]async fn test_ffi_grouping_udaf() -> Result<()> {let module = get_module()?;diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex a334bd6d21..99cc611acc 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -36,9 +36,9 @@ use datafusion_physical_expr::{use datafusion_physical_plan::{ExecutionPlan, Partitioning};use sync_provider::create_sync_table_provider;use udf_udaf_udwf::{- create_ffi_abs_func, create_ffi_first_value_func, create_ffi_random_func,- create_ffi_rank_func, create_ffi_stddev_func, create_ffi_sum_func,- create_ffi_table_func,+ create_ffi_abs_func, create_ffi_first_value_func, create_ffi_min_func,+ create_ffi_random_func, create_ffi_rank_func, create_ffi_stddev_func,+ create_ffi_sum_func, create_ffi_table_func,};use crate::catalog_provider::FFI_CatalogProvider;@@ -142,6 +142,9 @@ pub struct ForeignLibraryModule {/// Create an aggregate UDAF using first_valuepub create_first_value_udaf: extern "C" fn() -> FFI_AggregateUDF,++ /// Create an aggregate UDAF using min+ pub create_min_udaf: extern "C" fn() -> FFI_AggregateUDF,}pub fn create_test_schema() -> Arc<Schema> {@@ -429,5 +432,6 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {create_query_planner: query_planner::create_query_planner,version: super::version,create_first_value_udaf: create_ffi_first_value_func,+ create_min_udaf: create_ffi_min_func,}}diff --git a/datafusion/ffi/src/tests/udf_udaf_udwf.rs b/datafusion/ffi/src/tests/udf_udaf_udwf.rsindex ea7c03b5d6..8603a85c8a 100644--- a/datafusion/ffi/src/tests/udf_udaf_udwf.rs+++ b/datafusion/ffi/src/tests/udf_udaf_udwf.rs@@ -29,6 +29,7 @@ use datafusion_expr::{use datafusion_functions::math::abs::AbsFunc;use datafusion_functions::math::random::RandomFunc;use datafusion_functions_aggregate::first_last::FirstValue;+use datafusion_functions_aggregate::min_max::Min;use datafusion_functions_aggregate::stddev::Stddev;use datafusion_functions_aggregate::sum::Sum;use datafusion_functions_table::generate_series::RangeFunc;@@ -210,6 +211,12 @@ pub(crate) extern "C" fn create_ffi_stddev_func() -> FFI_AggregateUDF {udaf.into()}+pub(crate) extern "C" fn create_ffi_min_func() -> FFI_AggregateUDF {+ let udaf: Arc<AggregateUDF> = Arc::new(Min::new().into());++ udaf.into()+}+pub(crate) extern "C" fn create_ffi_rank_func() -> FFI_WindowUDF {let udwf: Arc<WindowUDF> = Arc::new(Rank::new(diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rsindex a103b4af20..df27ec78ba 100644--- a/datafusion/ffi/src/udaf/mod.rs+++ b/datafusion/ffi/src/udaf/mod.rs@@ -24,7 +24,8 @@ use datafusion_common::{DataFusionError, Result, ffi_datafusion_err};use datafusion_expr::function::AggregateFunctionSimplification;use datafusion_expr::type_coercion::functions::fields_with_udf;use datafusion_expr::{- Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature,+ Accumulator, AggregateUDF, AggregateUDFImpl, DistinctHandling, GroupsAccumulator,+ Signature,};use datafusion_functions_aggregate_common::accumulator::{AccumulatorArgs, StateFieldsArgs,@@ -149,6 +150,10 @@ pub struct FFI_AggregateUDF {/// FFI equivalent to [`AggregateUDF::supports_null_handling_clause`]pub supports_null_handling_clause:unsafe extern "C" fn(udaf: &FFI_AggregateUDF) -> bool,++ /// FFI equivalent to [`AggregateUDF::distinct_handling`]+ pub distinct_handling:+ unsafe extern "C" fn(udaf: &FFI_AggregateUDF) -> FFI_DistinctHandling,}unsafe impl Send for FFI_AggregateUDF {}@@ -337,6 +342,12 @@ unsafe extern "C" fn supports_null_handling_clause_fn_wrapper(unsafe { udaf.inner().supports_null_handling_clause() }}+unsafe extern "C" fn distinct_handling_fn_wrapper(+ udaf: &FFI_AggregateUDF,+) -> FFI_DistinctHandling {+ unsafe { udaf.inner().distinct_handling().into() }+}+unsafe extern "C" fn coerce_types_fn_wrapper(udaf: &FFI_AggregateUDF,arg_types: SVec<WrappedSchema>,@@ -412,6 +423,7 @@ impl From<Arc<AggregateUDF>> for FFI_AggregateUDF {private_data: Box::into_raw(private_data).cast::<c_void>(),library_marker_id: crate::get_library_marker_id,supports_null_handling_clause: supports_null_handling_clause_fn_wrapper,+ distinct_handling: distinct_handling_fn_wrapper,}}}@@ -610,6 +622,10 @@ impl AggregateUDFImpl for ForeignAggregateUDF {unsafe { (self.udaf.supports_null_handling_clause)(&self.udaf) }}+ fn distinct_handling(&self) -> DistinctHandling {+ unsafe { (self.udaf.distinct_handling)(&self.udaf).into() }+ }+fn simplify(&self) -> Option<AggregateFunctionSimplification> {None}@@ -655,6 +671,37 @@ impl From<AggregateOrderSensitivity> for FFI_AggregateOrderSensitivity {}}+#[repr(C)]+#[derive(Debug)]+pub enum FFI_DistinctHandling {+ Insensitive,+ Sensitive,+ Unsupported,+}++impl From<FFI_DistinctHandling> for DistinctHandling {+ fn from(value: FFI_DistinctHandling) -> Self {+ match value {+ FFI_DistinctHandling::Insensitive => Self::Insensitive,+ FFI_DistinctHandling::Sensitive => Self::Sensitive,+ FFI_DistinctHandling::Unsupported => Self::Unsupported,+ }+ }+}++impl From<DistinctHandling> for FFI_DistinctHandling {+ fn from(value: DistinctHandling) -> Self {+ match value {+ DistinctHandling::Insensitive => Self::Insensitive,+ DistinctHandling::Sensitive => Self::Sensitive,+ DistinctHandling::Unsupported => Self::Unsupported,+ // `DistinctHandling` is non_exhaustive, so map any variant added+ // later to the default.+ _ => Self::Sensitive,+ }+ }+}+#[cfg(test)]mod tests {@@ -798,6 +845,24 @@ mod tests {Ok(())}+ #[test]+ fn test_distinct_handling() -> Result<()> {+ let min = create_test_foreign_udaf(+ datafusion::functions_aggregate::min_max::Min::new(),+ )?;+ assert_eq!(min.distinct_handling(), DistinctHandling::Insensitive);++ let sum = create_test_foreign_udaf(Sum::new())?;+ assert_eq!(sum.distinct_handling(), DistinctHandling::Sensitive);++ let stddev = create_test_foreign_udaf(+ datafusion::functions_aggregate::stddev::Stddev::new(),+ )?;+ assert_eq!(stddev.distinct_handling(), DistinctHandling::Unsupported);++ Ok(())+ }+#[test]fn test_beneficial_ordering() -> Result<()> {let foreign_udaf = create_test_foreign_udaf(@@ -870,6 +935,18 @@ mod tests {test_round_trip_order_sensitivity(AggregateOrderSensitivity::Beneficial);}+ #[test]+ fn test_round_trip_all_distinct_handlings() {+ for handling in [+ DistinctHandling::Insensitive,+ DistinctHandling::Sensitive,+ DistinctHandling::Unsupported,+ ] {+ let ffi_handling: FFI_DistinctHandling = handling.into();+ assert_eq!(DistinctHandling::from(ffi_handling), handling);+ }+ }+#[test]fn test_ffi_udaf_local_bypass() -> Result<()> {let original_udaf = Sum::new();